From 33bfa0b1918ae2069bab44e02aa12d8d7a38d9bb Mon Sep 17 00:00:00 2001 From: Marco Slot Date: Thu, 23 Dec 2021 14:07:44 +0100 Subject: [PATCH] Hide shards from application_name's with a specific prefix --- .../distributed/commands/utility_hook.c | 1 + .../distributed/commands/variableset.c | 1 + .../distributed/metadata/metadata_cache.c | 19 ++ .../distributed/planner/distributed_planner.c | 4 +- src/backend/distributed/shared_library_init.c | 124 +++++++++ .../distributed/sql/citus--10.2-4--11.0-1.sql | 2 + .../sql/downgrades/citus--11.0-1--10.2-4.sql | 34 +++ .../citus_shard_indexes_on_worker/11.0-1.sql | 39 +++ .../citus_shard_indexes_on_worker/latest.sql | 39 +++ .../udfs/citus_shards_on_worker/11.0-1.sql | 34 +++ .../udfs/citus_shards_on_worker/latest.sql | 34 +++ .../worker/worker_shard_visibility.c | 238 ++++++++++++++---- src/include/distributed/metadata_cache.h | 1 + .../distributed/worker_shard_visibility.h | 4 +- src/test/regress/expected/multi_extension.out | 108 ++++---- .../expected/multi_mx_hide_shard_names.out | 192 ++++++++++---- src/test/regress/expected/pg_dump.out | 28 +++ .../expected/upgrade_list_citus_objects.out | 4 +- src/test/regress/multi_1_schedule | 3 +- src/test/regress/pg_regress_multi.pl | 3 + .../regress/sql/multi_mx_hide_shard_names.sql | 105 +++++--- src/test/regress/sql/pg_dump.sql | 25 ++ 22 files changed, 844 insertions(+), 198 deletions(-) create mode 100644 src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-1.sql create mode 100644 src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql create mode 100644 src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-1.sql create mode 100644 src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 4c79ec3e1..b7f52e871 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -66,6 +66,7 @@ #include "distributed/resource_lock.h" #include "distributed/transmit.h" #include "distributed/version_compat.h" +#include "distributed/worker_shard_visibility.h" #include "distributed/worker_transaction.h" #include "foreign/foreign.h" #include "lib/stringinfo.h" diff --git a/src/backend/distributed/commands/variableset.c b/src/backend/distributed/commands/variableset.c index c7ad22df2..ca17856f1 100644 --- a/src/backend/distributed/commands/variableset.c +++ b/src/backend/distributed/commands/variableset.c @@ -92,6 +92,7 @@ IsSettingSafeToPropagate(char *name) { /* if this list grows considerably we should switch to bsearch */ const char *skipSettings[] = { + "application_name", "citus.propagate_set_commands", "client_encoding", "exit_on_error", diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index ce05f7c28..fa8c6f8c5 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -166,6 +166,7 @@ typedef struct MetadataCacheData Oid secondaryNodeRoleId; Oid pgTableIsVisibleFuncId; Oid citusTableIsVisibleFuncId; + Oid relationIsAKnownShardFuncId; Oid jsonbExtractPathFuncId; bool databaseNameValid; char databaseName[NAMEDATALEN]; @@ -2619,6 +2620,24 @@ CitusTableVisibleFuncId(void) } +/* + * RelationIsAKnownShardFuncId returns oid of the relation_is_a_known_shard function. + */ +Oid +RelationIsAKnownShardFuncId(void) +{ + if (MetadataCache.relationIsAKnownShardFuncId == InvalidOid) + { + const int argCount = 1; + + MetadataCache.relationIsAKnownShardFuncId = + FunctionOid("pg_catalog", "relation_is_a_known_shard", argCount); + } + + return MetadataCache.relationIsAKnownShardFuncId; +} + + /* * JsonbExtractPathFuncId returns oid of the jsonb_extract_path function. */ diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index b318a3f3c..d5b71e505 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -202,9 +202,9 @@ distributed_planner(Query *parse, /* * Make sure that we hide shard names on the Citus MX worker nodes. See comments in - * ReplaceTableVisibleFunction() for the details. + * HideShardsFromSomeApplications() for the details. */ - ReplaceTableVisibleFunction((Node *) parse); + HideShardsFromSomeApplications(parse); /* create a restriction context and put it at the end if context list */ planContext.plannerRestrictionContext = CreateAndPushPlannerRestrictionContext(); diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 8d82e4d69..f74a3f43b 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -25,6 +25,7 @@ #include "citus_version.h" #include "commands/explain.h" +#include "common/string.h" #include "executor/executor.h" #include "distributed/backend_data.h" #include "distributed/citus_nodefuncs.h" @@ -102,6 +103,9 @@ static char *CitusVersion = CITUS_VERSION; /* deprecated GUC value that should not be used anywhere outside this file */ static int ReplicationModel = REPLICATION_MODEL_STREAMING; +/* we override the application_name assign_hook and keep a pointer to the old one */ +static GucStringAssignHook OldApplicationNameAssignHook = NULL; + void _PG_init(void); void _PG_fini(void); @@ -115,11 +119,16 @@ static void CitusCleanupConnectionsAtExit(int code, Datum arg); static void DecrementClientBackendCounterAtExit(int code, Datum arg); static void CreateRequiredDirectories(void); static void RegisterCitusConfigVariables(void); +static void OverridePostgresConfigAssignHooks(void); static bool ErrorIfNotASuitableDeadlockFactor(double *newval, void **extra, GucSource source); static bool WarnIfDeprecatedExecutorUsed(int *newval, void **extra, GucSource source); static bool WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source); static bool NoticeIfSubqueryPushdownEnabled(bool *newval, void **extra, GucSource source); +static bool HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, + GucSource source); +static void HideShardsFromAppNamePrefixesAssignHook(const char *newval, void *extra); +static void ApplicationNameAssignHook(const char *newval, void *extra); static bool NodeConninfoGucCheckHook(char **newval, void **extra, GucSource source); static void NodeConninfoGucAssignHook(const char *newval, void *extra); static const char * MaxSharedPoolSizeGucShowHook(void); @@ -1106,6 +1115,24 @@ RegisterCitusConfigVariables(void) GUC_NO_SHOW_ALL, NULL, NULL, NULL); + DefineCustomStringVariable( + "citus.hide_shards_from_app_name_prefixes", + gettext_noop("If application_name starts with one of these values, hide shards"), + gettext_noop("Citus places distributed tables and shards in the same schema. " + "That can cause confusion when inspecting the list of tables on " + "a node with shards. This GUC can be used to hide the shards from " + "pg_class for certain applications based on the application_name " + "of the connection. The default is *, which hides shards from all " + "applications. This behaviour can be overridden using the " + "citus.override_table_visibility setting"), + &HideShardsFromAppNamePrefixes, + "*", + PGC_USERSET, + GUC_STANDARD, + HideShardsFromAppNamePrefixesCheckHook, + HideShardsFromAppNamePrefixesAssignHook, + NULL); + DefineCustomIntVariable( "citus.isolation_test_session_process_id", NULL, @@ -1782,6 +1809,33 @@ RegisterCitusConfigVariables(void) /* warn about config items in the citus namespace that are not registered above */ EmitWarningsOnPlaceholders("citus"); + + OverridePostgresConfigAssignHooks(); +} + + +/* + * OverridePostgresConfigAssignHooks overrides GUC assign hooks where we want + * custom behaviour. + */ +static void +OverridePostgresConfigAssignHooks(void) +{ + struct config_generic **guc_vars = get_guc_variables(); + int gucCount = GetNumConfigOptions(); + + for (int gucIndex = 0; gucIndex < gucCount; gucIndex++) + { + struct config_generic *var = (struct config_generic *) guc_vars[gucIndex]; + + if (strcmp(var->name, "application_name") == 0) + { + struct config_string *stringVar = (struct config_string *) var; + + OldApplicationNameAssignHook = stringVar->assign_hook; + stringVar->assign_hook = ApplicationNameAssignHook; + } + } } @@ -1884,6 +1938,76 @@ WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source) } +/* + * HideShardsFromAppNamePrefixesCheckHook ensures that the + * citus.hide_shards_from_app_name_prefixes holds a valid list of application_name + * values. + */ +static bool +HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, GucSource source) +{ + List *prefixList = NIL; + + /* SplitGUCList scribbles on the input */ + char *splitCopy = pstrdup(*newval); + + /* check whether we can split into a list of identifiers */ + if (!SplitGUCList(splitCopy, ',', &prefixList)) + { + GUC_check_errdetail("not a valid list of identifiers"); + return false; + } + + char *appNamePrefix = NULL; + foreach_ptr(appNamePrefix, prefixList) + { + int prefixLength = strlen(appNamePrefix); + if (prefixLength >= NAMEDATALEN) + { + GUC_check_errdetail("prefix %s is more than %d characters", appNamePrefix, + NAMEDATALEN); + return false; + } + + char *prefixAscii = pstrdup(appNamePrefix); + pg_clean_ascii(prefixAscii); + + if (strcmp(prefixAscii, appNamePrefix) != 0) + { + GUC_check_errdetail("prefix %s in citus.hide_shards_from_app_name_prefixes " + "contains non-ascii characters", appNamePrefix); + return false; + } + } + + return true; +} + + +/* + * HideShardsFromAppNamePrefixesAssignHook ensures changes to + * citus.hide_shards_from_app_name_prefixes are reflected in the decision + * whether or not to show shards. + */ +static void +HideShardsFromAppNamePrefixesAssignHook(const char *newval, void *extra) +{ + ResetHideShardsDecision(); +} + + +/* + * ApplicationNameAssignHook is called whenever application_name changes + * to allow us to reset our hide shards decision. + */ +static void +ApplicationNameAssignHook(const char *newval, void *extra) +{ + ResetHideShardsDecision(); + OldApplicationNameAssignHook(newval, extra); +} + + /* * NodeConninfoGucCheckHook ensures conninfo settings are in the expected form * and that the keywords of all non-null settings are on a allowlist devised to diff --git a/src/backend/distributed/sql/citus--10.2-4--11.0-1.sql b/src/backend/distributed/sql/citus--10.2-4--11.0-1.sql index b4b6cc532..484a3e5f7 100644 --- a/src/backend/distributed/sql/citus--10.2-4--11.0-1.sql +++ b/src/backend/distributed/sql/citus--10.2-4--11.0-1.sql @@ -5,6 +5,8 @@ #include "udfs/citus_check_connection_to_node/11.0-1.sql" #include "udfs/citus_check_cluster_node_health/11.0-1.sql" +#include "udfs/citus_shards_on_worker/11.0-1.sql" +#include "udfs/citus_shard_indexes_on_worker/11.0-1.sql" #include "udfs/citus_internal_add_object_metadata/11.0-1.sql" #include "udfs/citus_run_local_command/11.0-1.sql" diff --git a/src/backend/distributed/sql/downgrades/citus--11.0-1--10.2-4.sql b/src/backend/distributed/sql/downgrades/citus--11.0-1--10.2-4.sql index df823be93..81d1944e9 100644 --- a/src/backend/distributed/sql/downgrades/citus--11.0-1--10.2-4.sql +++ b/src/backend/distributed/sql/downgrades/citus--11.0-1--10.2-4.sql @@ -46,3 +46,37 @@ DROP FUNCTION pg_catalog.citus_check_cluster_node_health (); DROP FUNCTION pg_catalog.citus_internal_add_object_metadata(text, text[], text[], integer, integer); DROP FUNCTION pg_catalog.citus_run_local_command(text); DROP FUNCTION pg_catalog.worker_drop_sequence_dependency(text); + +CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS + SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r','p','v','m','S','f','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.relation_is_a_known_shard(c.oid) + ORDER BY 1,2; + +CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS +SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner", + c2.relname as "Table" +FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid + LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid +WHERE c.relkind IN ('i','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.relation_is_a_known_shard(c.oid) +ORDER BY 1,2; + +DROP FUNCTION pg_catalog.citus_shards_on_worker(); +DROP FUNCTION pg_catalog.citus_shard_indexes_on_worker(); diff --git a/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-1.sql b/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-1.sql new file mode 100644 index 000000000..d98cdafe5 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-1.sql @@ -0,0 +1,39 @@ +CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker( + OUT schema_name name, + OUT index_name name, + OUT table_type text, + OUT owner_name name, + OUT shard_name name) + RETURNS SETOF record + LANGUAGE plpgsql + SET citus.hide_shards_from_app_name_prefixes = '' + AS $$ +BEGIN + -- this is the query that \di produces, except pg_table_is_visible + -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) + RETURN QUERY + SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner", + c2.relname as "Table" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid + LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid + WHERE c.relkind IN ('i','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.relation_is_a_known_shard(c.oid) + ORDER BY 1,2; +END; +$$; + +CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS + SELECT schema_name as "Schema", + index_name as "Name", + table_type as "Type", + owner_name as "Owner", + shard_name as "Table" + FROM pg_catalog.citus_shard_indexes_on_worker() s; diff --git a/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql b/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql new file mode 100644 index 000000000..d98cdafe5 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql @@ -0,0 +1,39 @@ +CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker( + OUT schema_name name, + OUT index_name name, + OUT table_type text, + OUT owner_name name, + OUT shard_name name) + RETURNS SETOF record + LANGUAGE plpgsql + SET citus.hide_shards_from_app_name_prefixes = '' + AS $$ +BEGIN + -- this is the query that \di produces, except pg_table_is_visible + -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) + RETURN QUERY + SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner", + c2.relname as "Table" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid + LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid + WHERE c.relkind IN ('i','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.relation_is_a_known_shard(c.oid) + ORDER BY 1,2; +END; +$$; + +CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS + SELECT schema_name as "Schema", + index_name as "Name", + table_type as "Type", + owner_name as "Owner", + shard_name as "Table" + FROM pg_catalog.citus_shard_indexes_on_worker() s; diff --git a/src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-1.sql b/src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-1.sql new file mode 100644 index 000000000..895c92ae8 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-1.sql @@ -0,0 +1,34 @@ +CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker( + OUT schema_name name, + OUT shard_name name, + OUT table_type text, + OUT owner_name name) + RETURNS SETOF record + LANGUAGE plpgsql + SET citus.hide_shards_from_app_name_prefixes = '' + AS $$ +BEGIN + -- this is the query that \d produces, except pg_table_is_visible + -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) + RETURN QUERY + SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r','p','v','m','S','f','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.relation_is_a_known_shard(c.oid) + ORDER BY 1,2; +END; +$$; + +CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS + SELECT schema_name as "Schema", + shard_name as "Name", + table_type as "Type", + owner_name as "Owner" + FROM pg_catalog.citus_shards_on_worker() s; diff --git a/src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql b/src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql new file mode 100644 index 000000000..895c92ae8 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql @@ -0,0 +1,34 @@ +CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker( + OUT schema_name name, + OUT shard_name name, + OUT table_type text, + OUT owner_name name) + RETURNS SETOF record + LANGUAGE plpgsql + SET citus.hide_shards_from_app_name_prefixes = '' + AS $$ +BEGIN + -- this is the query that \d produces, except pg_table_is_visible + -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) + RETURN QUERY + SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r','p','v','m','S','f','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.relation_is_a_known_shard(c.oid) + ORDER BY 1,2; +END; +$$; + +CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS + SELECT schema_name as "Schema", + shard_name as "Name", + table_type as "Type", + owner_name as "Owner" + FROM pg_catalog.citus_shards_on_worker() s; diff --git a/src/backend/distributed/worker/worker_shard_visibility.c b/src/backend/distributed/worker/worker_shard_visibility.c index 22dfcac65..a9edaf382 100644 --- a/src/backend/distributed/worker/worker_shard_visibility.c +++ b/src/backend/distributed/worker/worker_shard_visibility.c @@ -12,21 +12,43 @@ #include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" +#include "catalog/pg_type.h" #include "distributed/metadata_cache.h" #include "distributed/coordinator_protocol.h" +#include "distributed/listutils.h" #include "distributed/local_executor.h" +#include "distributed/query_colocation_checker.h" #include "distributed/worker_protocol.h" #include "distributed/worker_shard_visibility.h" +#include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "utils/lsyscache.h" #include "utils/syscache.h" +#include "utils/varlena.h" +/* HideShardsMode is used to determine whether to hide shards */ +typedef enum HideShardsMode +{ + CHECK_APPLICATION_NAME, + HIDE_SHARDS_FROM_APPLICATION, + DO_NOT_HIDE_SHARDS +} HideShardsMode; + /* Config variable managed via guc.c */ bool OverrideTableVisibility = true; bool EnableManualChangesToShards = false; -static bool ReplaceTableVisibleFunctionWalker(Node *inputNode); +/* hide shards when the application_name starts with one of: */ +char *HideShardsFromAppNamePrefixes = "*"; + +/* cache of whether or not to hide shards */ +static HideShardsMode HideShards = CHECK_APPLICATION_NAME; + +static bool ShouldHideShards(void); +static bool ShouldHideShardsInternal(void); +static bool FilterShardsFromPgclass(Node *node, void *context); +static Node * CreateRelationIsAKnownShardFilter(int pgClassVarno); PG_FUNCTION_INFO_V1(citus_table_is_visible); PG_FUNCTION_INFO_V1(relation_is_a_known_shard); @@ -43,18 +65,6 @@ relation_is_a_known_shard(PG_FUNCTION_ARGS) CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); - - if (!RelationIsVisible(relationId)) - { - /* - * Relation is not on the search path. - * - * TODO: it might be nicer to add a separate check in the - * citus_shards_on_worker views where this UDF is used. - */ - PG_RETURN_BOOL(false); - } - PG_RETURN_BOOL(RelationIsAKnownShard(relationId)); } @@ -257,72 +267,190 @@ RelationIsAKnownShard(Oid shardRelationId) /* - * ReplaceTableVisibleFunction is a wrapper around ReplaceTableVisibleFunctionWalker. - * The replace functionality can be enabled/disable via a GUC. This function also - * ensures that the extension is loaded and the version is compatible. + * HideShardsFromSomeApplications transforms queries to pg_class to + * filter out known shards if the application_name matches any of + * the prefixes in citus.hide_shards_from_app_name_prefixes. */ void -ReplaceTableVisibleFunction(Node *inputNode) +HideShardsFromSomeApplications(Query *query) { - if (!OverrideTableVisibility || + if (!OverrideTableVisibility || HideShards == DO_NOT_HIDE_SHARDS || !CitusHasBeenLoaded() || !CheckCitusVersion(DEBUG2)) { return; } - ReplaceTableVisibleFunctionWalker(inputNode); + if (ShouldHideShards()) + { + FilterShardsFromPgclass((Node *) query, NULL); + } } /* - * ReplaceTableVisibleFunction replaces all occurrences of - * pg_catalog.pg_table_visible() to - * pg_catalog.citus_table_visible() in the given input node. - * - * Note that the only difference between the functions is that - * the latter filters the tables that are known to be shards on - * Citus MX worker (data) nodes. - * - * Note that although the function mutates the input node, we - * prefer to use query_tree_walker/expression_tree_walker over - * their mutator equivalents. This is safe because we ensure that - * the replaced function has the exact same input/output values with - * its precedent. + * ShouldHideShards returns whether we should hide shards in the current + * session. It only checks the application_name once and then uses a + * cached response unless either the application_name or + * citus.hide_shards_from_app_name_prefixes changes. */ static bool -ReplaceTableVisibleFunctionWalker(Node *inputNode) +ShouldHideShards(void) { - if (inputNode == NULL) + if (HideShards == CHECK_APPLICATION_NAME) + { + if (ShouldHideShardsInternal()) + { + HideShards = HIDE_SHARDS_FROM_APPLICATION; + return true; + } + else + { + HideShards = DO_NOT_HIDE_SHARDS; + return false; + } + } + else + { + return HideShards == HIDE_SHARDS_FROM_APPLICATION; + } +} + + +/* + * ResetHideShardsDecision resets the decision whether to hide shards. + */ +void +ResetHideShardsDecision(void) +{ + HideShards = CHECK_APPLICATION_NAME; +} + + +/* + * ShouldHideShardsInternal determines whether we should hide shards based on + * the current application name. + */ +static bool +ShouldHideShardsInternal(void) +{ + if (IsCitusInitiatedRemoteBackend()) + { + /* we never hide shards from Citus */ + return false; + } + + List *prefixList = NIL; + + /* SplitGUCList scribbles on the input */ + char *splitCopy = pstrdup(HideShardsFromAppNamePrefixes); + + if (!SplitGUCList(splitCopy, ',', &prefixList)) + { + /* invalid GUC value, ignore */ + return false; + } + + char *appNamePrefix = NULL; + foreach_ptr(appNamePrefix, prefixList) + { + /* always hide shards when one of the prefixes is * */ + if (strcmp(appNamePrefix, "*") == 0) + { + return true; + } + + /* compare only the first first characters */ + int prefixLength = strlen(appNamePrefix); + if (strncmp(application_name, appNamePrefix, prefixLength) == 0) + { + return true; + } + } + + return false; +} + + +/* + * FilterShardsFromPgclass adds a NOT relation_is_a_known_shard(oid) filter + * to the security quals of pg_class RTEs. + */ +static bool +FilterShardsFromPgclass(Node *node, void *context) +{ + if (node == NULL) { return false; } - if (IsA(inputNode, FuncExpr)) + if (IsA(node, Query)) { - FuncExpr *functionToProcess = (FuncExpr *) inputNode; - Oid functionId = functionToProcess->funcid; + Query *query = (Query *) node; + MemoryContext queryContext = GetMemoryChunkContext(query); - if (functionId == PgTableVisibleFuncId()) + /* + * We process the whole rtable rather than visiting individual RangeTblEntry's + * in the walker, since we need to know the varno to generate the right + * fiter. + */ + int varno = 0; + RangeTblEntry *rangeTableEntry = NULL; + + foreach_ptr(rangeTableEntry, query->rtable) { - /* - * We simply update the function id of the FuncExpr for - * two reasons: (i) We don't want to interfere with the - * memory contexts so don't want to deal with allocating - * a new functionExpr (ii) We already know that both - * functions have the exact same signature. - */ - functionToProcess->funcid = CitusTableVisibleFuncId(); + varno++; - /* although not very likely, we could have nested calls to pg_table_is_visible */ - return expression_tree_walker(inputNode, ReplaceTableVisibleFunctionWalker, - NULL); + if (rangeTableEntry->rtekind != RTE_RELATION || + rangeTableEntry->relid != RelationRelationId) + { + /* not pg_class */ + continue; + } + + /* make sure the expression is in the right memory context */ + MemoryContext originalContext = MemoryContextSwitchTo(queryContext); + + /* add NOT relation_is_a_known_shard(oid) to the security quals of the RTE */ + rangeTableEntry->securityQuals = + list_make1(CreateRelationIsAKnownShardFilter(varno)); + + MemoryContextSwitchTo(originalContext); } - } - else if (IsA(inputNode, Query)) - { - return query_tree_walker((Query *) inputNode, ReplaceTableVisibleFunctionWalker, - NULL, 0); + + return query_tree_walker((Query *) node, FilterShardsFromPgclass, context, 0); } - return expression_tree_walker(inputNode, ReplaceTableVisibleFunctionWalker, NULL); + return expression_tree_walker(node, FilterShardsFromPgclass, context); +} + + +/* + * CreateRelationIsAKnownShardFilter constructs an expression of the form: + * NOT pg_catalog.relation_is_a_known_shard(oid) + */ +static Node * +CreateRelationIsAKnownShardFilter(int pgClassVarno) +{ + /* oid is always the first column */ + AttrNumber oidAttNum = 1; + + Var *oidVar = makeVar(pgClassVarno, oidAttNum, OIDOID, -1, InvalidOid, 0); + + /* build the call to read_intermediate_result */ + FuncExpr *funcExpr = makeNode(FuncExpr); + funcExpr->funcid = RelationIsAKnownShardFuncId(); + funcExpr->funcretset = false; + funcExpr->funcvariadic = false; + funcExpr->funcformat = 0; + funcExpr->funccollid = 0; + funcExpr->inputcollid = 0; + funcExpr->location = -1; + funcExpr->args = list_make1(oidVar); + + BoolExpr *notExpr = makeNode(BoolExpr); + notExpr->boolop = NOT_EXPR; + notExpr->args = list_make1(funcExpr); + notExpr->location = -1; + + return (Node *) notExpr; } diff --git a/src/include/distributed/metadata_cache.h b/src/include/distributed/metadata_cache.h index 4461cb1e9..132bd59ae 100644 --- a/src/include/distributed/metadata_cache.h +++ b/src/include/distributed/metadata_cache.h @@ -252,6 +252,7 @@ extern Oid CitusExtraDataContainerFuncId(void); extern Oid CitusAnyValueFunctionId(void); extern Oid PgTableVisibleFuncId(void); extern Oid CitusTableVisibleFuncId(void); +extern Oid RelationIsAKnownShardFuncId(void); extern Oid JsonbExtractPathFuncId(void); /* enum oids */ diff --git a/src/include/distributed/worker_shard_visibility.h b/src/include/distributed/worker_shard_visibility.h index c5c58d712..957992fed 100644 --- a/src/include/distributed/worker_shard_visibility.h +++ b/src/include/distributed/worker_shard_visibility.h @@ -15,9 +15,11 @@ extern bool OverrideTableVisibility; extern bool EnableManualChangesToShards; +extern char *HideShardsFromAppNamePrefixes; -extern void ReplaceTableVisibleFunction(Node *inputNode); +extern void HideShardsFromSomeApplications(Query *query); +extern void ResetHideShardsDecision(void); extern void ErrorIfRelationIsAKnownShard(Oid relationId); extern void ErrorIfIllegallyChangingKnownShard(Oid relationId); extern bool RelationIsAKnownShard(Oid shardRelationId); diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 701b5e26f..b0c37d16a 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -425,20 +425,20 @@ SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDE ALTER EXTENSION citus UPDATE TO '9.4-2'; -- should see the old source code SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1; - prosrc + prosrc --------------------------------------------------------------------- - + - DECLARE + - colocated_tables regclass[]; + - BEGIN + - SELECT get_colocated_table_array(relation) INTO colocated_tables;+ - PERFORM + - master_update_shard_statistics(shardid) + - FROM + - pg_dist_shard + - WHERE + - logicalrelid = ANY (colocated_tables); + - END; + + + + DECLARE + + colocated_tables regclass[]; + + BEGIN + + SELECT get_colocated_table_array(relation) INTO colocated_tables;+ + PERFORM + + master_update_shard_statistics(shardid) + + FROM + + pg_dist_shard + + WHERE + + logicalrelid = ANY (colocated_tables); + + END; + (1 row) @@ -466,20 +466,20 @@ SELECT * FROM multi_extension.print_extension_changes(); ALTER EXTENSION citus UPDATE TO '9.4-1'; -- should see the old source code SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1; - prosrc + prosrc --------------------------------------------------------------------- - + - DECLARE + - colocated_tables regclass[]; + - BEGIN + - SELECT get_colocated_table_array(relation) INTO colocated_tables;+ - PERFORM + - master_update_shard_statistics(shardid) + - FROM + - pg_dist_shard + - WHERE + - logicalrelid = ANY (colocated_tables); + - END; + + + + DECLARE + + colocated_tables regclass[]; + + BEGIN + + SELECT get_colocated_table_array(relation) INTO colocated_tables;+ + PERFORM + + master_update_shard_statistics(shardid) + + FROM + + pg_dist_shard + + WHERE + + logicalrelid = ANY (colocated_tables); + + END; + (1 row) @@ -573,20 +573,20 @@ SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDE ALTER EXTENSION citus UPDATE TO '9.5-2'; -- should see the old source code SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1; - prosrc + prosrc --------------------------------------------------------------------- - + - DECLARE + - colocated_tables regclass[]; + - BEGIN + - SELECT get_colocated_table_array(relation) INTO colocated_tables;+ - PERFORM + - master_update_shard_statistics(shardid) + - FROM + - pg_dist_shard + - WHERE + - logicalrelid = ANY (colocated_tables); + - END; + + + + DECLARE + + colocated_tables regclass[]; + + BEGIN + + SELECT get_colocated_table_array(relation) INTO colocated_tables;+ + PERFORM + + master_update_shard_statistics(shardid) + + FROM + + pg_dist_shard + + WHERE + + logicalrelid = ANY (colocated_tables); + + END; + (1 row) @@ -614,20 +614,20 @@ SELECT * FROM multi_extension.print_extension_changes(); ALTER EXTENSION citus UPDATE TO '9.5-1'; -- should see the old source code SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1; - prosrc + prosrc --------------------------------------------------------------------- - + - DECLARE + - colocated_tables regclass[]; + - BEGIN + - SELECT get_colocated_table_array(relation) INTO colocated_tables;+ - PERFORM + - master_update_shard_statistics(shardid) + - FROM + - pg_dist_shard + - WHERE + - logicalrelid = ANY (colocated_tables); + - END; + + + + DECLARE + + colocated_tables regclass[]; + + BEGIN + + SELECT get_colocated_table_array(relation) INTO colocated_tables;+ + PERFORM + + master_update_shard_statistics(shardid) + + FROM + + pg_dist_shard + + WHERE + + logicalrelid = ANY (colocated_tables); + + END; + (1 row) @@ -1005,8 +1005,10 @@ SELECT * FROM multi_extension.print_extension_changes(); | function citus_disable_node(text,integer,boolean) void | function citus_internal_add_object_metadata(text,text[],text[],integer,integer) void | function citus_run_local_command(text) void + | function citus_shard_indexes_on_worker() SETOF record + | function citus_shards_on_worker() SETOF record | function worker_drop_sequence_dependency(text) void -(10 rows) +(12 rows) DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; -- show running version diff --git a/src/test/regress/expected/multi_mx_hide_shard_names.out b/src/test/regress/expected/multi_mx_hide_shard_names.out index b2965fba5..469677c10 100644 --- a/src/test/regress/expected/multi_mx_hide_shard_names.out +++ b/src/test/regress/expected/multi_mx_hide_shard_names.out @@ -45,47 +45,37 @@ SELECT create_distributed_table('test_table', 'id'); -- first show that the views does not show -- any shards on the coordinator as expected -SELECT * FROM citus_shards_on_worker; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names'; Schema | Name | Type | Owner --------------------------------------------------------------------- (0 rows) -SELECT * FROM citus_shard_indexes_on_worker; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names'; Schema | Name | Type | Owner | Table --------------------------------------------------------------------- (0 rows) -- now show that we see the shards, but not the -- indexes as there are no indexes -\c - - - :worker_1_port +\c postgresql://postgres@localhost::worker_1_port/regression?application_name=psql SET search_path TO 'mx_hide_shard_names'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; Schema | Name | Type | Owner --------------------------------------------------------------------- mx_hide_shard_names | test_table_1130000 | table | postgres mx_hide_shard_names | test_table_1130002 | table | postgres (2 rows) -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; Schema | Name | Type | Owner | Table --------------------------------------------------------------------- (0 rows) --- also show that nested calls to pg_table_is_visible works fine --- if both of the calls to the pg_table_is_visible haven't been --- replaced, we would get 0 rows in the output -SELECT - pg_table_is_visible((SELECT - "t1"."Name"::regclass - FROM - citus_shards_on_worker as t1 - WHERE - NOT pg_table_is_visible("t1"."Name"::regclass) - LIMIT - 1)); - pg_table_is_visible +-- shards are hidden when using psql as application_name +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname --------------------------------------------------------------------- - f + test_table (1 row) -- now create an index @@ -94,22 +84,136 @@ SET search_path TO 'mx_hide_shard_names'; CREATE INDEX test_index ON mx_hide_shard_names.test_table(id); -- now show that we see the shards, and the -- indexes as well -\c - - - :worker_1_port +\c postgresql://postgres@localhost::worker_1_port/regression?application_name=psql SET search_path TO 'mx_hide_shard_names'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; Schema | Name | Type | Owner --------------------------------------------------------------------- mx_hide_shard_names | test_table_1130000 | table | postgres mx_hide_shard_names | test_table_1130002 | table | postgres (2 rows) -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; Schema | Name | Type | Owner | Table --------------------------------------------------------------------- mx_hide_shard_names | test_index_1130000 | index | postgres | test_table_1130000 mx_hide_shard_names | test_index_1130002 | index | postgres | test_table_1130002 (2 rows) +-- shards are hidden when using psql as application_name +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname +--------------------------------------------------------------------- + test_index + test_table +(2 rows) + +-- changing application_name reveals the shards +SET application_name TO ''; +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname +--------------------------------------------------------------------- + test_index + test_index_1130000 + test_index_1130002 + test_table + test_table_1130000 + test_table_1130002 +(6 rows) + +RESET application_name; +-- shards are hidden again after GUCs are reset +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname +--------------------------------------------------------------------- + test_index + test_table +(2 rows) + +-- changing application_name in transaction reveals the shards +BEGIN; +SET LOCAL application_name TO ''; +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname +--------------------------------------------------------------------- + test_index + test_index_1130000 + test_index_1130002 + test_table + test_table_1130000 + test_table_1130002 +(6 rows) + +ROLLBACK; +-- shards are hidden again after GUCs are reset +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname +--------------------------------------------------------------------- + test_index + test_table +(2 rows) + +-- now with session-level GUC, but ROLLBACK; +BEGIN; +SET application_name TO ''; +ROLLBACK; +-- shards are hidden again after GUCs are reset +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname +--------------------------------------------------------------------- + test_index + test_table +(2 rows) + +-- we should hide correctly based on application_name with savepoints +BEGIN; +SAVEPOINT s1; +SET application_name TO ''; +-- changing application_name reveals the shards +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname +--------------------------------------------------------------------- + test_index + test_index_1130000 + test_index_1130002 + test_table + test_table_1130000 + test_table_1130002 +(6 rows) + +ROLLBACK TO SAVEPOINT s1; +-- shards are hidden again after GUCs are reset +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname +--------------------------------------------------------------------- + test_index + test_table +(2 rows) + +ROLLBACK; +-- changing citus.hide_shards_from_app_name_prefixes reveals the shards +BEGIN; +SET LOCAL citus.hide_shards_from_app_name_prefixes TO 'notpsql'; +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname +--------------------------------------------------------------------- + test_index + test_index_1130000 + test_index_1130002 + test_table + test_table_1130000 + test_table_1130002 +(6 rows) + +ROLLBACK; +-- shards are hidden again after GUCs are reset +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname +--------------------------------------------------------------------- + test_index + test_table +(2 rows) + -- we should be able to select from the shards directly if we -- know the name of the tables SELECT count(*) FROM test_table_1130000; @@ -118,20 +222,20 @@ SELECT count(*) FROM test_table_1130000; 0 (1 row) --- disable the config so that table becomes visible -SELECT pg_table_is_visible('test_table_1130000'::regclass); - pg_table_is_visible ---------------------------------------------------------------------- - f -(1 row) - -SET citus.override_table_visibility TO FALSE; +-- shards on the search_path still match pg_table_is_visible SELECT pg_table_is_visible('test_table_1130000'::regclass); pg_table_is_visible --------------------------------------------------------------------- t (1 row) +-- shards on the search_path do not match citus_table_is_visible +SELECT citus_table_is_visible('test_table_1130000'::regclass); + citus_table_is_visible +--------------------------------------------------------------------- + f +(1 row) + \c - - - :master_port -- make sure that we're resilient to the edge cases -- such that the table name includes the shard number @@ -153,7 +257,7 @@ SET search_path TO 'mx_hide_shard_names'; -- with the same name since a table with the same -- name already exists :) CREATE TABLE test_table_2_1130000(id int, time date); -SELECT * FROM citus_shards_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; Schema | Name | Type | Owner --------------------------------------------------------------------- mx_hide_shard_names | test_table_102008_1130004 | table | postgres @@ -187,7 +291,7 @@ SELECT create_distributed_table('test_table', 'id'); CREATE INDEX test_index ON mx_hide_shard_names_2.test_table(id); \c - - - :worker_1_port SET search_path TO 'mx_hide_shard_names'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; Schema | Name | Type | Owner --------------------------------------------------------------------- mx_hide_shard_names | test_table_102008_1130004 | table | postgres @@ -196,39 +300,27 @@ SELECT * FROM citus_shards_on_worker ORDER BY 2; mx_hide_shard_names | test_table_1130002 | table | postgres (4 rows) -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; Schema | Name | Type | Owner | Table --------------------------------------------------------------------- mx_hide_shard_names | test_index_1130000 | index | postgres | test_table_1130000 mx_hide_shard_names | test_index_1130002 | index | postgres | test_table_1130002 (2 rows) -SET search_path TO 'mx_hide_shard_names_2'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names_2' ORDER BY 2; Schema | Name | Type | Owner --------------------------------------------------------------------- mx_hide_shard_names_2 | test_table_1130008 | table | postgres mx_hide_shard_names_2 | test_table_1130010 | table | postgres (2 rows) -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names_2' ORDER BY 2; Schema | Name | Type | Owner | Table --------------------------------------------------------------------- mx_hide_shard_names_2 | test_index_1130008 | index | postgres | test_table_1130008 mx_hide_shard_names_2 | test_index_1130010 | index | postgres | test_table_1130010 (2 rows) -SET search_path TO 'mx_hide_shard_names_2, mx_hide_shard_names'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; - Schema | Name | Type | Owner ---------------------------------------------------------------------- -(0 rows) - -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; - Schema | Name | Type | Owner | Table ---------------------------------------------------------------------- -(0 rows) - -- now try very long table names \c - - - :master_port SET citus.shard_count TO 4; @@ -247,7 +339,7 @@ SELECT create_distributed_table('too_long_12345678901234567890123456789012345678 \c - - - :worker_1_port SET search_path TO 'mx_hide_shard_names_3'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names_3' ORDER BY 2; Schema | Name | Type | Owner --------------------------------------------------------------------- mx_hide_shard_names_3 | too_long_1234567890123456789012345678901234567_e0119164_1130012 | table | postgres @@ -278,14 +370,14 @@ SELECT create_distributed_table('"CiTuS.TeeN"."TeeNTabLE.1!?!"', 'TeNANt_Id'); \c - - - :worker_1_port SET search_path TO "CiTuS.TeeN"; -SELECT * FROM citus_shards_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'CiTuS.TeeN' ORDER BY 2; Schema | Name | Type | Owner --------------------------------------------------------------------- CiTuS.TeeN | TeeNTabLE.1!?!_1130016 | table | postgres CiTuS.TeeN | TeeNTabLE.1!?!_1130018 | table | postgres (2 rows) -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'CiTuS.TeeN' ORDER BY 2; Schema | Name | Type | Owner | Table --------------------------------------------------------------------- CiTuS.TeeN | MyTenantIndex_1130016 | index | postgres | TeeNTabLE.1!?!_1130016 diff --git a/src/test/regress/expected/pg_dump.out b/src/test/regress/expected/pg_dump.out index 750652e38..b3c5dab41 100644 --- a/src/test/regress/expected/pg_dump.out +++ b/src/test/regress/expected/pg_dump.out @@ -161,3 +161,31 @@ DETAIL: drop cascades to table data drop cascades to table dist_columnar drop cascades to table simple_columnar drop cascades to table "weird.table" +CREATE SCHEMA dumper; +CREATE TABLE data ( + key int, + value text +); +SELECT create_distributed_table('data', 'key'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +COPY data FROM STDIN WITH (format csv, delimiter '|', escape '\'); +-- run pg_dump on worker (which has shards) +\COPY output FROM PROGRAM 'PGAPPNAME=pg_dump pg_dump -f results/pg_dump.tmp -h localhost -p 57637 -U postgres -d regression -n dumper --quote-all-identifiers' +-- restore pg_dump from worker via coordinator +DROP SCHEMA dumper CASCADE; +NOTICE: drop cascades to table data +\COPY (SELECT line FROM output WHERE line IS NOT NULL) TO PROGRAM 'psql -qtAX -h localhost -p 57636 -U postgres -d regression -f results/pg_dump.tmp' + +-- check the tables (should not include shards) +SELECT tablename FROM pg_tables WHERE schemaname = 'dumper' ORDER BY 1; + tablename +--------------------------------------------------------------------- + data +(1 row) + +DROP SCHEMA dumper CASCADE; +NOTICE: drop cascades to table data diff --git a/src/test/regress/expected/upgrade_list_citus_objects.out b/src/test/regress/expected/upgrade_list_citus_objects.out index e38818594..6daee122e 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects.out +++ b/src/test/regress/expected/upgrade_list_citus_objects.out @@ -97,7 +97,9 @@ ORDER BY 1; function citus_shard_allowed_on_node_true(bigint,integer) function citus_shard_cost_1(bigint) function citus_shard_cost_by_disk_size(bigint) + function citus_shard_indexes_on_worker() function citus_shard_sizes() + function citus_shards_on_worker() function citus_stat_statements() function citus_stat_statements_reset() function citus_table_is_visible(oid) @@ -264,5 +266,5 @@ ORDER BY 1; view citus_worker_stat_activity view pg_dist_shard_placement view time_partitions -(248 rows) +(250 rows) diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index fbcc34196..4bdbe25fe 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -196,12 +196,13 @@ test: local_dist_join_modifications test: local_table_join test: local_dist_join_mixed test: citus_local_dist_joins +test: pg_dump # --------- # multi_copy creates hash and range-partitioned tables and performs COPY # multi_router_planner creates hash partitioned tables. # --------- -test: multi_copy fast_path_router_modify pg_dump +test: multi_copy fast_path_router_modify test: multi_router_planner # These 2 tests have prepared statements which sometimes get invalidated by concurrent tests, # changing the debug output. We should not run them in parallel with others diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index d18b63539..ebbb15371 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -465,6 +465,9 @@ push(@pgOptions, "citus.node_connection_timeout=${connectionTimeout}"); push(@pgOptions, "citus.explain_analyze_sort_method='taskId'"); push(@pgOptions, "citus.enable_manual_changes_to_shards=on"); +# Some tests look at shards in pg_class, make sure we can usually see them: +push(@pgOptions, "citus.hide_shards_from_app_name_prefixes='psql,pg_dump'"); + # we disable slow start by default to encourage parallelism within tests push(@pgOptions, "citus.executor_slow_start_interval=0ms"); diff --git a/src/test/regress/sql/multi_mx_hide_shard_names.sql b/src/test/regress/sql/multi_mx_hide_shard_names.sql index 3cc191c78..57017c90c 100644 --- a/src/test/regress/sql/multi_mx_hide_shard_names.sql +++ b/src/test/regress/sql/multi_mx_hide_shard_names.sql @@ -31,28 +31,18 @@ SELECT create_distributed_table('test_table', 'id'); -- first show that the views does not show -- any shards on the coordinator as expected -SELECT * FROM citus_shards_on_worker; -SELECT * FROM citus_shard_indexes_on_worker; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names'; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names'; -- now show that we see the shards, but not the -- indexes as there are no indexes -\c - - - :worker_1_port +\c postgresql://postgres@localhost::worker_1_port/regression?application_name=psql SET search_path TO 'mx_hide_shard_names'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; --- also show that nested calls to pg_table_is_visible works fine --- if both of the calls to the pg_table_is_visible haven't been --- replaced, we would get 0 rows in the output -SELECT - pg_table_is_visible((SELECT - "t1"."Name"::regclass - FROM - citus_shards_on_worker as t1 - WHERE - NOT pg_table_is_visible("t1"."Name"::regclass) - LIMIT - 1)); +-- shards are hidden when using psql as application_name +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; -- now create an index \c - - - :master_port @@ -61,20 +51,69 @@ CREATE INDEX test_index ON mx_hide_shard_names.test_table(id); -- now show that we see the shards, and the -- indexes as well -\c - - - :worker_1_port +\c postgresql://postgres@localhost::worker_1_port/regression?application_name=psql SET search_path TO 'mx_hide_shard_names'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; + +-- shards are hidden when using psql as application_name +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + +-- changing application_name reveals the shards +SET application_name TO ''; +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; +RESET application_name; + +-- shards are hidden again after GUCs are reset +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + +-- changing application_name in transaction reveals the shards +BEGIN; +SET LOCAL application_name TO ''; +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; +ROLLBACK; + +-- shards are hidden again after GUCs are reset +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + +-- now with session-level GUC, but ROLLBACK; +BEGIN; +SET application_name TO ''; +ROLLBACK; + +-- shards are hidden again after GUCs are reset +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + +-- we should hide correctly based on application_name with savepoints +BEGIN; +SAVEPOINT s1; +SET application_name TO ''; +-- changing application_name reveals the shards +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; +ROLLBACK TO SAVEPOINT s1; +-- shards are hidden again after GUCs are reset +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; +ROLLBACK; + +-- changing citus.hide_shards_from_app_name_prefixes reveals the shards +BEGIN; +SET LOCAL citus.hide_shards_from_app_name_prefixes TO 'notpsql'; +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; +ROLLBACK; + +-- shards are hidden again after GUCs are reset +SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; -- we should be able to select from the shards directly if we -- know the name of the tables SELECT count(*) FROM test_table_1130000; --- disable the config so that table becomes visible -SELECT pg_table_is_visible('test_table_1130000'::regclass); -SET citus.override_table_visibility TO FALSE; +-- shards on the search_path still match pg_table_is_visible SELECT pg_table_is_visible('test_table_1130000'::regclass); +-- shards on the search_path do not match citus_table_is_visible +SELECT citus_table_is_visible('test_table_1130000'::regclass); + \c - - - :master_port -- make sure that we're resilient to the edge cases -- such that the table name includes the shard number @@ -95,7 +134,7 @@ SET search_path TO 'mx_hide_shard_names'; -- name already exists :) CREATE TABLE test_table_2_1130000(id int, time date); -SELECT * FROM citus_shards_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; \d @@ -111,14 +150,10 @@ CREATE INDEX test_index ON mx_hide_shard_names_2.test_table(id); \c - - - :worker_1_port SET search_path TO 'mx_hide_shard_names'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; -SET search_path TO 'mx_hide_shard_names_2'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; -SET search_path TO 'mx_hide_shard_names_2, mx_hide_shard_names'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names_2' ORDER BY 2; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names_2' ORDER BY 2; -- now try very long table names \c - - - :master_port @@ -137,7 +172,7 @@ SELECT create_distributed_table('too_long_12345678901234567890123456789012345678 \c - - - :worker_1_port SET search_path TO 'mx_hide_shard_names_3'; -SELECT * FROM citus_shards_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names_3' ORDER BY 2; \d @@ -159,8 +194,8 @@ SELECT create_distributed_table('"CiTuS.TeeN"."TeeNTabLE.1!?!"', 'TeNANt_Id'); \c - - - :worker_1_port SET search_path TO "CiTuS.TeeN"; -SELECT * FROM citus_shards_on_worker ORDER BY 2; -SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2; +SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'CiTuS.TeeN' ORDER BY 2; +SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'CiTuS.TeeN' ORDER BY 2; \d \di diff --git a/src/test/regress/sql/pg_dump.sql b/src/test/regress/sql/pg_dump.sql index bf45566cc..c8c7f45fd 100644 --- a/src/test/regress/sql/pg_dump.sql +++ b/src/test/regress/sql/pg_dump.sql @@ -101,3 +101,28 @@ COPY dist_columnar TO STDOUT; SELECT indexname FROM pg_indexes WHERE tablename = 'weird.table' ORDER BY indexname; DROP SCHEMA dumper CASCADE; + +CREATE SCHEMA dumper; +CREATE TABLE data ( + key int, + value text +); +SELECT create_distributed_table('data', 'key'); +COPY data FROM STDIN WITH (format csv, delimiter '|', escape '\'); +1|{"this":"is","json":1} +2|{"$\"":9} +3|{"{}":" "} +4|{} +\. + +-- run pg_dump on worker (which has shards) +\COPY output FROM PROGRAM 'PGAPPNAME=pg_dump pg_dump -f results/pg_dump.tmp -h localhost -p 57637 -U postgres -d regression -n dumper --quote-all-identifiers' + +-- restore pg_dump from worker via coordinator +DROP SCHEMA dumper CASCADE; +\COPY (SELECT line FROM output WHERE line IS NOT NULL) TO PROGRAM 'psql -qtAX -h localhost -p 57636 -U postgres -d regression -f results/pg_dump.tmp' + +-- check the tables (should not include shards) +SELECT tablename FROM pg_tables WHERE schemaname = 'dumper' ORDER BY 1; + +DROP SCHEMA dumper CASCADE;