external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Hide shards from application_name's with a specific prefix

pull/5567/head
Marco Slot 2021-12-23 14:07:44 +01:00
parent d98500ac22
commit 33bfa0b191
22 changed files with 844 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/backend/distributed/commands/utility_hook.c
View File

@ -66,6 +66,7 @@
#include "distributed/resource_lock.h"
#include "distributed/transmit.h"
#include "distributed/version_compat.h"
#include "distributed/worker_shard_visibility.h"
#include "distributed/worker_transaction.h"
#include "foreign/foreign.h"
#include "lib/stringinfo.h"

1
src/backend/distributed/commands/variableset.c
View File

@ -92,6 +92,7 @@ IsSettingSafeToPropagate(char *name)
{
/* if this list grows considerably we should switch to bsearch */
const char *skipSettings[] = {
"application_name",
"citus.propagate_set_commands",
"client_encoding",
"exit_on_error",

19
src/backend/distributed/metadata/metadata_cache.c
View File

@ -166,6 +166,7 @@ typedef struct MetadataCacheData
Oid secondaryNodeRoleId;
Oid pgTableIsVisibleFuncId;
Oid citusTableIsVisibleFuncId;
Oid relationIsAKnownShardFuncId;
Oid jsonbExtractPathFuncId;
bool databaseNameValid;
char databaseName[NAMEDATALEN];
@ -2619,6 +2620,24 @@ CitusTableVisibleFuncId(void)
}
/*
* RelationIsAKnownShardFuncId returns oid of the relation_is_a_known_shard function.
*/
Oid
RelationIsAKnownShardFuncId(void)
{
if (MetadataCache.relationIsAKnownShardFuncId == InvalidOid)
{
const int argCount = 1;
MetadataCache.relationIsAKnownShardFuncId =
FunctionOid("pg_catalog", "relation_is_a_known_shard", argCount);
}
return MetadataCache.relationIsAKnownShardFuncId;
}
/*
* JsonbExtractPathFuncId returns oid of the jsonb_extract_path function.
*/

4
src/backend/distributed/planner/distributed_planner.c
View File

@ -202,9 +202,9 @@ distributed_planner(Query *parse,
/*
* Make sure that we hide shard names on the Citus MX worker nodes. See comments in
* ReplaceTableVisibleFunction() for the details.
* HideShardsFromSomeApplications() for the details.
*/
ReplaceTableVisibleFunction((Node *) parse);
HideShardsFromSomeApplications(parse);
/* create a restriction context and put it at the end if context list */
planContext.plannerRestrictionContext = CreateAndPushPlannerRestrictionContext();

124
src/backend/distributed/shared_library_init.c
View File

@ -25,6 +25,7 @@
#include "citus_version.h"
#include "commands/explain.h"
#include "common/string.h"
#include "executor/executor.h"
#include "distributed/backend_data.h"
#include "distributed/citus_nodefuncs.h"
@ -102,6 +103,9 @@ static char *CitusVersion = CITUS_VERSION;
/* deprecated GUC value that should not be used anywhere outside this file */
static int ReplicationModel = REPLICATION_MODEL_STREAMING;
/* we override the application_name assign_hook and keep a pointer to the old one */
static GucStringAssignHook OldApplicationNameAssignHook = NULL;
void _PG_init(void);
void _PG_fini(void);
@ -115,11 +119,16 @@ static void CitusCleanupConnectionsAtExit(int code, Datum arg);
static void DecrementClientBackendCounterAtExit(int code, Datum arg);
static void CreateRequiredDirectories(void);
static void RegisterCitusConfigVariables(void);
static void OverridePostgresConfigAssignHooks(void);
static bool ErrorIfNotASuitableDeadlockFactor(double *newval, void **extra,
GucSource source);
static bool WarnIfDeprecatedExecutorUsed(int *newval, void **extra, GucSource source);
static bool WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source);
static bool NoticeIfSubqueryPushdownEnabled(bool *newval, void **extra, GucSource source);
static bool HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra,
GucSource source);
static void HideShardsFromAppNamePrefixesAssignHook(const char *newval, void *extra);
static void ApplicationNameAssignHook(const char *newval, void *extra);
static bool NodeConninfoGucCheckHook(char **newval, void **extra, GucSource source);
static void NodeConninfoGucAssignHook(const char *newval, void *extra);
static const char * MaxSharedPoolSizeGucShowHook(void);
@ -1106,6 +1115,24 @@ RegisterCitusConfigVariables(void)
GUC_NO_SHOW_ALL,
NULL, NULL, NULL);
DefineCustomStringVariable(
"citus.hide_shards_from_app_name_prefixes",
gettext_noop("If application_name starts with one of these values, hide shards"),
gettext_noop("Citus places distributed tables and shards in the same schema. "
"That can cause confusion when inspecting the list of tables on "
"a node with shards. This GUC can be used to hide the shards from "
"pg_class for certain applications based on the application_name "
"of the connection. The default is *, which hides shards from all "
"applications. This behaviour can be overridden using the "
"citus.override_table_visibility setting"),
&HideShardsFromAppNamePrefixes,
"*",
PGC_USERSET,
GUC_STANDARD,
HideShardsFromAppNamePrefixesCheckHook,
HideShardsFromAppNamePrefixesAssignHook,
NULL);
DefineCustomIntVariable(
"citus.isolation_test_session_process_id",
NULL,
@ -1782,6 +1809,33 @@ RegisterCitusConfigVariables(void)
/* warn about config items in the citus namespace that are not registered above */
EmitWarningsOnPlaceholders("citus");
OverridePostgresConfigAssignHooks();
}
/*
* OverridePostgresConfigAssignHooks overrides GUC assign hooks where we want
* custom behaviour.
*/
static void
OverridePostgresConfigAssignHooks(void)
{
struct config_generic **guc_vars = get_guc_variables();
int gucCount = GetNumConfigOptions();
for (int gucIndex = 0; gucIndex < gucCount; gucIndex++)
{
struct config_generic *var = (struct config_generic *) guc_vars[gucIndex];
if (strcmp(var->name, "application_name") == 0)
{
struct config_string *stringVar = (struct config_string *) var;
OldApplicationNameAssignHook = stringVar->assign_hook;
stringVar->assign_hook = ApplicationNameAssignHook;
}
}
}
@ -1884,6 +1938,76 @@ WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source)
}
/*
* HideShardsFromAppNamePrefixesCheckHook ensures that the
* citus.hide_shards_from_app_name_prefixes holds a valid list of application_name
* values.
*/
static bool
HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, GucSource source)
{
List *prefixList = NIL;
/* SplitGUCList scribbles on the input */
char *splitCopy = pstrdup(*newval);
/* check whether we can split into a list of identifiers */
if (!SplitGUCList(splitCopy, ',', &prefixList))
{
GUC_check_errdetail("not a valid list of identifiers");
return false;
}
char *appNamePrefix = NULL;
foreach_ptr(appNamePrefix, prefixList)
{
int prefixLength = strlen(appNamePrefix);
if (prefixLength >= NAMEDATALEN)
{
GUC_check_errdetail("prefix %s is more than %d characters", appNamePrefix,
NAMEDATALEN);
return false;
}
char *prefixAscii = pstrdup(appNamePrefix);
pg_clean_ascii(prefixAscii);
if (strcmp(prefixAscii, appNamePrefix) != 0)
{
GUC_check_errdetail("prefix %s in citus.hide_shards_from_app_name_prefixes "
"contains non-ascii characters", appNamePrefix);
return false;
}
}
return true;
}
/*
* HideShardsFromAppNamePrefixesAssignHook ensures changes to
* citus.hide_shards_from_app_name_prefixes are reflected in the decision
* whether or not to show shards.
*/
static void
HideShardsFromAppNamePrefixesAssignHook(const char *newval, void *extra)
{
ResetHideShardsDecision();
}
/*
* ApplicationNameAssignHook is called whenever application_name changes
* to allow us to reset our hide shards decision.
*/
static void
ApplicationNameAssignHook(const char *newval, void *extra)
{
ResetHideShardsDecision();
OldApplicationNameAssignHook(newval, extra);
}
/*
* NodeConninfoGucCheckHook ensures conninfo settings are in the expected form
* and that the keywords of all non-null settings are on a allowlist devised to

2
src/backend/distributed/sql/citus--10.2-4--11.0-1.sql
View File

@ -5,6 +5,8 @@
#include "udfs/citus_check_connection_to_node/11.0-1.sql"
#include "udfs/citus_check_cluster_node_health/11.0-1.sql"
#include "udfs/citus_shards_on_worker/11.0-1.sql"
#include "udfs/citus_shard_indexes_on_worker/11.0-1.sql"
#include "udfs/citus_internal_add_object_metadata/11.0-1.sql"
#include "udfs/citus_run_local_command/11.0-1.sql"

34
src/backend/distributed/sql/downgrades/citus--11.0-1--10.2-4.sql
View File

@ -46,3 +46,37 @@ DROP FUNCTION pg_catalog.citus_check_cluster_node_health ();
DROP FUNCTION pg_catalog.citus_internal_add_object_metadata(text, text[], text[], integer, integer);
DROP FUNCTION pg_catalog.citus_run_local_command(text);
DROP FUNCTION pg_catalog.worker_drop_sequence_dependency(text);
CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS
SELECT n.nspname as "Schema",
c.relname as "Name",
CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type",
pg_catalog.pg_get_userbyid(c.relowner) as "Owner"
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind IN ('r','p','v','m','S','f','')
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
AND pg_catalog.relation_is_a_known_shard(c.oid)
ORDER BY 1,2;
CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS
SELECT n.nspname as "Schema",
c.relname as "Name",
CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type",
pg_catalog.pg_get_userbyid(c.relowner) as "Owner",
c2.relname as "Table"
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid
LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid
WHERE c.relkind IN ('i','')
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
AND pg_catalog.relation_is_a_known_shard(c.oid)
ORDER BY 1,2;
DROP FUNCTION pg_catalog.citus_shards_on_worker();
DROP FUNCTION pg_catalog.citus_shard_indexes_on_worker();

39
src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-1.sql generated Normal file
View File

@ -0,0 +1,39 @@
CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker(
OUT schema_name name,
OUT index_name name,
OUT table_type text,
OUT owner_name name,
OUT shard_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.hide_shards_from_app_name_prefixes = ''
AS $$
BEGIN
-- this is the query that \di produces, except pg_table_is_visible
-- is replaced with pg_catalog.relation_is_a_known_shard(c.oid)
RETURN QUERY
SELECT n.nspname as "Schema",
c.relname as "Name",
CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type",
pg_catalog.pg_get_userbyid(c.relowner) as "Owner",
c2.relname as "Table"
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid
LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid
WHERE c.relkind IN ('i','')
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
AND pg_catalog.relation_is_a_known_shard(c.oid)
ORDER BY 1,2;
END;
$$;
CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS
SELECT schema_name as "Schema",
index_name as "Name",
table_type as "Type",
owner_name as "Owner",
shard_name as "Table"
FROM pg_catalog.citus_shard_indexes_on_worker() s;

39
src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql Normal file
View File

@ -0,0 +1,39 @@
CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker(
OUT schema_name name,
OUT index_name name,
OUT table_type text,
OUT owner_name name,
OUT shard_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.hide_shards_from_app_name_prefixes = ''
AS $$
BEGIN
-- this is the query that \di produces, except pg_table_is_visible
-- is replaced with pg_catalog.relation_is_a_known_shard(c.oid)
RETURN QUERY
SELECT n.nspname as "Schema",
c.relname as "Name",
CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type",
pg_catalog.pg_get_userbyid(c.relowner) as "Owner",
c2.relname as "Table"
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid
LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid
WHERE c.relkind IN ('i','')
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
AND pg_catalog.relation_is_a_known_shard(c.oid)
ORDER BY 1,2;
END;
$$;
CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS
SELECT schema_name as "Schema",
index_name as "Name",
table_type as "Type",
owner_name as "Owner",
shard_name as "Table"
FROM pg_catalog.citus_shard_indexes_on_worker() s;

34
src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-1.sql generated Normal file
View File

@ -0,0 +1,34 @@
CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker(
OUT schema_name name,
OUT shard_name name,
OUT table_type text,
OUT owner_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.hide_shards_from_app_name_prefixes = ''
AS $$
BEGIN
-- this is the query that \d produces, except pg_table_is_visible
-- is replaced with pg_catalog.relation_is_a_known_shard(c.oid)
RETURN QUERY
SELECT n.nspname as "Schema",
c.relname as "Name",
CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type",
pg_catalog.pg_get_userbyid(c.relowner) as "Owner"
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind IN ('r','p','v','m','S','f','')
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
AND pg_catalog.relation_is_a_known_shard(c.oid)
ORDER BY 1,2;
END;
$$;
CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS
SELECT schema_name as "Schema",
shard_name as "Name",
table_type as "Type",
owner_name as "Owner"
FROM pg_catalog.citus_shards_on_worker() s;

34
src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql Normal file
View File

@ -0,0 +1,34 @@
CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker(
OUT schema_name name,
OUT shard_name name,
OUT table_type text,
OUT owner_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.hide_shards_from_app_name_prefixes = ''
AS $$
BEGIN
-- this is the query that \d produces, except pg_table_is_visible
-- is replaced with pg_catalog.relation_is_a_known_shard(c.oid)
RETURN QUERY
SELECT n.nspname as "Schema",
c.relname as "Name",
CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type",
pg_catalog.pg_get_userbyid(c.relowner) as "Owner"
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind IN ('r','p','v','m','S','f','')
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
AND pg_catalog.relation_is_a_known_shard(c.oid)
ORDER BY 1,2;
END;
$$;
CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS
SELECT schema_name as "Schema",
shard_name as "Name",
table_type as "Type",
owner_name as "Owner"
FROM pg_catalog.citus_shards_on_worker() s;

238
src/backend/distributed/worker/worker_shard_visibility.c
View File

@ -12,21 +12,43 @@
#include "catalog/index.h"
#include "catalog/namespace.h"
#include "catalog/pg_class.h"
#include "catalog/pg_type.h"
#include "distributed/metadata_cache.h"
#include "distributed/coordinator_protocol.h"
#include "distributed/listutils.h"
#include "distributed/local_executor.h"
#include "distributed/query_colocation_checker.h"
#include "distributed/worker_protocol.h"
#include "distributed/worker_shard_visibility.h"
#include "nodes/makefuncs.h"
#include "nodes/nodeFuncs.h"
#include "utils/lsyscache.h"
#include "utils/syscache.h"
#include "utils/varlena.h"
/* HideShardsMode is used to determine whether to hide shards */
typedef enum HideShardsMode
{
CHECK_APPLICATION_NAME,
HIDE_SHARDS_FROM_APPLICATION,
DO_NOT_HIDE_SHARDS
} HideShardsMode;
/* Config variable managed via guc.c */
bool OverrideTableVisibility = true;
bool EnableManualChangesToShards = false;
static bool ReplaceTableVisibleFunctionWalker(Node *inputNode);
/* hide shards when the application_name starts with one of: */
char *HideShardsFromAppNamePrefixes = "*";
/* cache of whether or not to hide shards */
static HideShardsMode HideShards = CHECK_APPLICATION_NAME;
static bool ShouldHideShards(void);
static bool ShouldHideShardsInternal(void);
static bool FilterShardsFromPgclass(Node *node, void *context);
static Node * CreateRelationIsAKnownShardFilter(int pgClassVarno);
PG_FUNCTION_INFO_V1(citus_table_is_visible);
PG_FUNCTION_INFO_V1(relation_is_a_known_shard);
@ -43,18 +65,6 @@ relation_is_a_known_shard(PG_FUNCTION_ARGS)
CheckCitusVersion(ERROR);
Oid relationId = PG_GETARG_OID(0);
if (!RelationIsVisible(relationId))
{
/*
* Relation is not on the search path.
*
* TODO: it might be nicer to add a separate check in the
* citus_shards_on_worker views where this UDF is used.
*/
PG_RETURN_BOOL(false);
}
PG_RETURN_BOOL(RelationIsAKnownShard(relationId));
}
@ -257,72 +267,190 @@ RelationIsAKnownShard(Oid shardRelationId)
/*
* ReplaceTableVisibleFunction is a wrapper around ReplaceTableVisibleFunctionWalker.
* The replace functionality can be enabled/disable via a GUC. This function also
* ensures that the extension is loaded and the version is compatible.
* HideShardsFromSomeApplications transforms queries to pg_class to
* filter out known shards if the application_name matches any of
* the prefixes in citus.hide_shards_from_app_name_prefixes.
*/
void
ReplaceTableVisibleFunction(Node *inputNode)
HideShardsFromSomeApplications(Query *query)
{
if (!OverrideTableVisibility ||
if (!OverrideTableVisibility || HideShards == DO_NOT_HIDE_SHARDS ||
!CitusHasBeenLoaded() || !CheckCitusVersion(DEBUG2))
{
return;
}
ReplaceTableVisibleFunctionWalker(inputNode);
if (ShouldHideShards())
{
FilterShardsFromPgclass((Node *) query, NULL);
}
}
/*
* ReplaceTableVisibleFunction replaces all occurrences of
* pg_catalog.pg_table_visible() to
* pg_catalog.citus_table_visible() in the given input node.
*
* Note that the only difference between the functions is that
* the latter filters the tables that are known to be shards on
* Citus MX worker (data) nodes.
*
* Note that although the function mutates the input node, we
* prefer to use query_tree_walker/expression_tree_walker over
* their mutator equivalents. This is safe because we ensure that
* the replaced function has the exact same input/output values with
* its precedent.
* ShouldHideShards returns whether we should hide shards in the current
* session. It only checks the application_name once and then uses a
* cached response unless either the application_name or
* citus.hide_shards_from_app_name_prefixes changes.
*/
static bool
ReplaceTableVisibleFunctionWalker(Node *inputNode)
ShouldHideShards(void)
{
if (inputNode == NULL)
if (HideShards == CHECK_APPLICATION_NAME)
{
if (ShouldHideShardsInternal())
{
HideShards = HIDE_SHARDS_FROM_APPLICATION;
return true;
}
else
{
HideShards = DO_NOT_HIDE_SHARDS;
return false;
}
}
else
{
return HideShards == HIDE_SHARDS_FROM_APPLICATION;
}
}
/*
* ResetHideShardsDecision resets the decision whether to hide shards.
*/
void
ResetHideShardsDecision(void)
{
HideShards = CHECK_APPLICATION_NAME;
}
/*
* ShouldHideShardsInternal determines whether we should hide shards based on
* the current application name.
*/
static bool
ShouldHideShardsInternal(void)
{
if (IsCitusInitiatedRemoteBackend())
{
/* we never hide shards from Citus */
return false;
}
List *prefixList = NIL;
/* SplitGUCList scribbles on the input */
char *splitCopy = pstrdup(HideShardsFromAppNamePrefixes);
if (!SplitGUCList(splitCopy, ',', &prefixList))
{
/* invalid GUC value, ignore */
return false;
}
char *appNamePrefix = NULL;
foreach_ptr(appNamePrefix, prefixList)
{
/* always hide shards when one of the prefixes is * */
if (strcmp(appNamePrefix, "*") == 0)
{
return true;
}
/* compare only the first first <prefixLength> characters */
int prefixLength = strlen(appNamePrefix);
if (strncmp(application_name, appNamePrefix, prefixLength) == 0)
{
return true;
}
}
return false;
}
/*
* FilterShardsFromPgclass adds a NOT relation_is_a_known_shard(oid) filter
* to the security quals of pg_class RTEs.
*/
static bool
FilterShardsFromPgclass(Node *node, void *context)
{
if (node == NULL)
{
return false;
}
if (IsA(inputNode, FuncExpr))
if (IsA(node, Query))
{
FuncExpr *functionToProcess = (FuncExpr *) inputNode;
Oid functionId = functionToProcess->funcid;
Query *query = (Query *) node;
MemoryContext queryContext = GetMemoryChunkContext(query);
if (functionId == PgTableVisibleFuncId())
/*
* We process the whole rtable rather than visiting individual RangeTblEntry's
* in the walker, since we need to know the varno to generate the right
* fiter.
*/
int varno = 0;
RangeTblEntry *rangeTableEntry = NULL;
foreach_ptr(rangeTableEntry, query->rtable)
{
/*
* We simply update the function id of the FuncExpr for
* two reasons: (i) We don't want to interfere with the
* memory contexts so don't want to deal with allocating
* a new functionExpr (ii) We already know that both
* functions have the exact same signature.
*/
functionToProcess->funcid = CitusTableVisibleFuncId();
varno++;
/* although not very likely, we could have nested calls to pg_table_is_visible */
return expression_tree_walker(inputNode, ReplaceTableVisibleFunctionWalker,
NULL);
if (rangeTableEntry->rtekind != RTE_RELATION ||
rangeTableEntry->relid != RelationRelationId)
{
/* not pg_class */
continue;
}
/* make sure the expression is in the right memory context */
MemoryContext originalContext = MemoryContextSwitchTo(queryContext);
/* add NOT relation_is_a_known_shard(oid) to the security quals of the RTE */
rangeTableEntry->securityQuals =
list_make1(CreateRelationIsAKnownShardFilter(varno));
MemoryContextSwitchTo(originalContext);
}
}
else if (IsA(inputNode, Query))
{
return query_tree_walker((Query *) inputNode, ReplaceTableVisibleFunctionWalker,
NULL, 0);
return query_tree_walker((Query *) node, FilterShardsFromPgclass, context, 0);
}
return expression_tree_walker(inputNode, ReplaceTableVisibleFunctionWalker, NULL);
return expression_tree_walker(node, FilterShardsFromPgclass, context);
}
/*
* CreateRelationIsAKnownShardFilter constructs an expression of the form:
* NOT pg_catalog.relation_is_a_known_shard(oid)
*/
static Node *
CreateRelationIsAKnownShardFilter(int pgClassVarno)
{
/* oid is always the first column */
AttrNumber oidAttNum = 1;
Var *oidVar = makeVar(pgClassVarno, oidAttNum, OIDOID, -1, InvalidOid, 0);
/* build the call to read_intermediate_result */
FuncExpr *funcExpr = makeNode(FuncExpr);
funcExpr->funcid = RelationIsAKnownShardFuncId();
funcExpr->funcretset = false;
funcExpr->funcvariadic = false;
funcExpr->funcformat = 0;
funcExpr->funccollid = 0;
funcExpr->inputcollid = 0;
funcExpr->location = -1;
funcExpr->args = list_make1(oidVar);
BoolExpr *notExpr = makeNode(BoolExpr);
notExpr->boolop = NOT_EXPR;
notExpr->args = list_make1(funcExpr);
notExpr->location = -1;
return (Node *) notExpr;
}

1
src/include/distributed/metadata_cache.h
View File

@ -252,6 +252,7 @@ extern Oid CitusExtraDataContainerFuncId(void);
extern Oid CitusAnyValueFunctionId(void);
extern Oid PgTableVisibleFuncId(void);
extern Oid CitusTableVisibleFuncId(void);
extern Oid RelationIsAKnownShardFuncId(void);
extern Oid JsonbExtractPathFuncId(void);
/* enum oids */

4
src/include/distributed/worker_shard_visibility.h
View File

@ -15,9 +15,11 @@
extern bool OverrideTableVisibility;
extern bool EnableManualChangesToShards;
extern char *HideShardsFromAppNamePrefixes;
extern void ReplaceTableVisibleFunction(Node *inputNode);
extern void HideShardsFromSomeApplications(Query *query);
extern void ResetHideShardsDecision(void);
extern void ErrorIfRelationIsAKnownShard(Oid relationId);
extern void ErrorIfIllegallyChangingKnownShard(Oid relationId);
extern bool RelationIsAKnownShard(Oid shardRelationId);

108
src/test/regress/expected/multi_extension.out
View File

@ -425,20 +425,20 @@ SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDE
ALTER EXTENSION citus UPDATE TO '9.4-2';
-- should see the old source code
SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
prosrc
prosrc
---------------------------------------------------------------------
+
DECLARE +
colocated_tables regclass[]; +
BEGIN +
SELECT get_colocated_table_array(relation) INTO colocated_tables;+
PERFORM +
master_update_shard_statistics(shardid) +
FROM +
pg_dist_shard +
WHERE +
logicalrelid = ANY (colocated_tables); +
END; +
+
DECLARE +
colocated_tables regclass[]; +
BEGIN +
SELECT get_colocated_table_array(relation) INTO colocated_tables;+
PERFORM +
master_update_shard_statistics(shardid) +
FROM +
pg_dist_shard +
WHERE +
logicalrelid = ANY (colocated_tables); +
END; +
(1 row)
@ -466,20 +466,20 @@ SELECT * FROM multi_extension.print_extension_changes();
ALTER EXTENSION citus UPDATE TO '9.4-1';
-- should see the old source code
SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
prosrc
prosrc
---------------------------------------------------------------------
+
DECLARE +
colocated_tables regclass[]; +
BEGIN +
SELECT get_colocated_table_array(relation) INTO colocated_tables;+
PERFORM +
master_update_shard_statistics(shardid) +
FROM +
pg_dist_shard +
WHERE +
logicalrelid = ANY (colocated_tables); +
END; +
+
DECLARE +
colocated_tables regclass[]; +
BEGIN +
SELECT get_colocated_table_array(relation) INTO colocated_tables;+
PERFORM +
master_update_shard_statistics(shardid) +
FROM +
pg_dist_shard +
WHERE +
logicalrelid = ANY (colocated_tables); +
END; +
(1 row)
@ -573,20 +573,20 @@ SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDE
ALTER EXTENSION citus UPDATE TO '9.5-2';
-- should see the old source code
SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
prosrc
prosrc
---------------------------------------------------------------------
+
DECLARE +
colocated_tables regclass[]; +
BEGIN +
SELECT get_colocated_table_array(relation) INTO colocated_tables;+
PERFORM +
master_update_shard_statistics(shardid) +
FROM +
pg_dist_shard +
WHERE +
logicalrelid = ANY (colocated_tables); +
END; +
+
DECLARE +
colocated_tables regclass[]; +
BEGIN +
SELECT get_colocated_table_array(relation) INTO colocated_tables;+
PERFORM +
master_update_shard_statistics(shardid) +
FROM +
pg_dist_shard +
WHERE +
logicalrelid = ANY (colocated_tables); +
END; +
(1 row)
@ -614,20 +614,20 @@ SELECT * FROM multi_extension.print_extension_changes();
ALTER EXTENSION citus UPDATE TO '9.5-1';
-- should see the old source code
SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
prosrc
prosrc
---------------------------------------------------------------------
+
DECLARE +
colocated_tables regclass[]; +
BEGIN +
SELECT get_colocated_table_array(relation) INTO colocated_tables;+
PERFORM +
master_update_shard_statistics(shardid) +
FROM +
pg_dist_shard +
WHERE +
logicalrelid = ANY (colocated_tables); +
END; +
+
DECLARE +
colocated_tables regclass[]; +
BEGIN +
SELECT get_colocated_table_array(relation) INTO colocated_tables;+
PERFORM +
master_update_shard_statistics(shardid) +
FROM +
pg_dist_shard +
WHERE +
logicalrelid = ANY (colocated_tables); +
END; +
(1 row)
@ -1005,8 +1005,10 @@ SELECT * FROM multi_extension.print_extension_changes();
| function citus_disable_node(text,integer,boolean) void
| function citus_internal_add_object_metadata(text,text[],text[],integer,integer) void
| function citus_run_local_command(text) void
| function citus_shard_indexes_on_worker() SETOF record
| function citus_shards_on_worker() SETOF record
| function worker_drop_sequence_dependency(text) void
(10 rows)
(12 rows)
DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff;
-- show running version

192
src/test/regress/expected/multi_mx_hide_shard_names.out
View File

@ -45,47 +45,37 @@ SELECT create_distributed_table('test_table', 'id');
-- first show that the views does not show
-- any shards on the coordinator as expected
SELECT * FROM citus_shards_on_worker;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names';
Schema | Name | Type | Owner
---------------------------------------------------------------------
(0 rows)
SELECT * FROM citus_shard_indexes_on_worker;
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names';
Schema | Name | Type | Owner | Table
---------------------------------------------------------------------
(0 rows)
-- now show that we see the shards, but not the
-- indexes as there are no indexes
\c - - - :worker_1_port
\c postgresql://postgres@localhost::worker_1_port/regression?application_name=psql
SET search_path TO 'mx_hide_shard_names';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
Schema | Name | Type | Owner
---------------------------------------------------------------------
mx_hide_shard_names | test_table_1130000 | table | postgres
mx_hide_shard_names | test_table_1130002 | table | postgres
(2 rows)
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
Schema | Name | Type | Owner | Table
---------------------------------------------------------------------
(0 rows)
-- also show that nested calls to pg_table_is_visible works fine
-- if both of the calls to the pg_table_is_visible haven't been
-- replaced, we would get 0 rows in the output
SELECT
pg_table_is_visible((SELECT
"t1"."Name"::regclass
FROM
citus_shards_on_worker as t1
WHERE
NOT pg_table_is_visible("t1"."Name"::regclass)
LIMIT
1));
pg_table_is_visible
-- shards are hidden when using psql as application_name
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
f
test_table
(1 row)
-- now create an index
@ -94,22 +84,136 @@ SET search_path TO 'mx_hide_shard_names';
CREATE INDEX test_index ON mx_hide_shard_names.test_table(id);
-- now show that we see the shards, and the
-- indexes as well
\c - - - :worker_1_port
\c postgresql://postgres@localhost::worker_1_port/regression?application_name=psql
SET search_path TO 'mx_hide_shard_names';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
Schema | Name | Type | Owner
---------------------------------------------------------------------
mx_hide_shard_names | test_table_1130000 | table | postgres
mx_hide_shard_names | test_table_1130002 | table | postgres
(2 rows)
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
Schema | Name | Type | Owner | Table
---------------------------------------------------------------------
mx_hide_shard_names | test_index_1130000 | index | postgres | test_table_1130000
mx_hide_shard_names | test_index_1130002 | index | postgres | test_table_1130002
(2 rows)
-- shards are hidden when using psql as application_name
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
test_index
test_table
(2 rows)
-- changing application_name reveals the shards
SET application_name TO '';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
test_index
test_index_1130000
test_index_1130002
test_table
test_table_1130000
test_table_1130002
(6 rows)
RESET application_name;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
test_index
test_table
(2 rows)
-- changing application_name in transaction reveals the shards
BEGIN;
SET LOCAL application_name TO '';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
test_index
test_index_1130000
test_index_1130002
test_table
test_table_1130000
test_table_1130002
(6 rows)
ROLLBACK;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
test_index
test_table
(2 rows)
-- now with session-level GUC, but ROLLBACK;
BEGIN;
SET application_name TO '';
ROLLBACK;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
test_index
test_table
(2 rows)
-- we should hide correctly based on application_name with savepoints
BEGIN;
SAVEPOINT s1;
SET application_name TO '';
-- changing application_name reveals the shards
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
test_index
test_index_1130000
test_index_1130002
test_table
test_table_1130000
test_table_1130002
(6 rows)
ROLLBACK TO SAVEPOINT s1;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
test_index
test_table
(2 rows)
ROLLBACK;
-- changing citus.hide_shards_from_app_name_prefixes reveals the shards
BEGIN;
SET LOCAL citus.hide_shards_from_app_name_prefixes TO 'notpsql';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
test_index
test_index_1130000
test_index_1130002
test_table
test_table_1130000
test_table_1130002
(6 rows)
ROLLBACK;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
test_index
test_table
(2 rows)
-- we should be able to select from the shards directly if we
-- know the name of the tables
SELECT count(*) FROM test_table_1130000;
@ -118,20 +222,20 @@ SELECT count(*) FROM test_table_1130000;
0
(1 row)
-- disable the config so that table becomes visible
SELECT pg_table_is_visible('test_table_1130000'::regclass);
pg_table_is_visible
---------------------------------------------------------------------
f
(1 row)
SET citus.override_table_visibility TO FALSE;
-- shards on the search_path still match pg_table_is_visible
SELECT pg_table_is_visible('test_table_1130000'::regclass);
pg_table_is_visible
---------------------------------------------------------------------
t
(1 row)
-- shards on the search_path do not match citus_table_is_visible
SELECT citus_table_is_visible('test_table_1130000'::regclass);
citus_table_is_visible
---------------------------------------------------------------------
f
(1 row)
\c - - - :master_port
-- make sure that we're resilient to the edge cases
-- such that the table name includes the shard number
@ -153,7 +257,7 @@ SET search_path TO 'mx_hide_shard_names';
-- with the same name since a table with the same
-- name already exists :)
CREATE TABLE test_table_2_1130000(id int, time date);
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
Schema | Name | Type | Owner
---------------------------------------------------------------------
mx_hide_shard_names | test_table_102008_1130004 | table | postgres
@ -187,7 +291,7 @@ SELECT create_distributed_table('test_table', 'id');
CREATE INDEX test_index ON mx_hide_shard_names_2.test_table(id);
\c - - - :worker_1_port
SET search_path TO 'mx_hide_shard_names';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
Schema | Name | Type | Owner
---------------------------------------------------------------------
mx_hide_shard_names | test_table_102008_1130004 | table | postgres
@ -196,39 +300,27 @@ SELECT * FROM citus_shards_on_worker ORDER BY 2;
mx_hide_shard_names | test_table_1130002 | table | postgres
(4 rows)
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
Schema | Name | Type | Owner | Table
---------------------------------------------------------------------
mx_hide_shard_names | test_index_1130000 | index | postgres | test_table_1130000
mx_hide_shard_names | test_index_1130002 | index | postgres | test_table_1130002
(2 rows)
SET search_path TO 'mx_hide_shard_names_2';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names_2' ORDER BY 2;
Schema | Name | Type | Owner
---------------------------------------------------------------------
mx_hide_shard_names_2 | test_table_1130008 | table | postgres
mx_hide_shard_names_2 | test_table_1130010 | table | postgres
(2 rows)
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names_2' ORDER BY 2;
Schema | Name | Type | Owner | Table
---------------------------------------------------------------------
mx_hide_shard_names_2 | test_index_1130008 | index | postgres | test_table_1130008
mx_hide_shard_names_2 | test_index_1130010 | index | postgres | test_table_1130010
(2 rows)
SET search_path TO 'mx_hide_shard_names_2, mx_hide_shard_names';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
Schema | Name | Type | Owner
---------------------------------------------------------------------
(0 rows)
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
Schema | Name | Type | Owner | Table
---------------------------------------------------------------------
(0 rows)
-- now try very long table names
\c - - - :master_port
SET citus.shard_count TO 4;
@ -247,7 +339,7 @@ SELECT create_distributed_table('too_long_12345678901234567890123456789012345678
\c - - - :worker_1_port
SET search_path TO 'mx_hide_shard_names_3';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names_3' ORDER BY 2;
Schema | Name | Type | Owner
---------------------------------------------------------------------
mx_hide_shard_names_3 | too_long_1234567890123456789012345678901234567_e0119164_1130012 | table | postgres
@ -278,14 +370,14 @@ SELECT create_distributed_table('"CiTuS.TeeN"."TeeNTabLE.1!?!"', 'TeNANt_Id');
\c - - - :worker_1_port
SET search_path TO "CiTuS.TeeN";
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'CiTuS.TeeN' ORDER BY 2;
Schema | Name | Type | Owner
---------------------------------------------------------------------
CiTuS.TeeN | TeeNTabLE.1!?!_1130016 | table | postgres
CiTuS.TeeN | TeeNTabLE.1!?!_1130018 | table | postgres
(2 rows)
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'CiTuS.TeeN' ORDER BY 2;
Schema | Name | Type | Owner | Table
---------------------------------------------------------------------
CiTuS.TeeN | MyTenantIndex_1130016 | index | postgres | TeeNTabLE.1!?!_1130016

28
src/test/regress/expected/pg_dump.out
View File

@ -161,3 +161,31 @@ DETAIL: drop cascades to table data
drop cascades to table dist_columnar
drop cascades to table simple_columnar
drop cascades to table "weird.table"
CREATE SCHEMA dumper;
CREATE TABLE data (
key int,
value text
);
SELECT create_distributed_table('data', 'key');
create_distributed_table
---------------------------------------------------------------------
(1 row)
COPY data FROM STDIN WITH (format csv, delimiter '|', escape '\');
-- run pg_dump on worker (which has shards)
\COPY output FROM PROGRAM 'PGAPPNAME=pg_dump pg_dump -f results/pg_dump.tmp -h localhost -p 57637 -U postgres -d regression -n dumper --quote-all-identifiers'
-- restore pg_dump from worker via coordinator
DROP SCHEMA dumper CASCADE;
NOTICE: drop cascades to table data
\COPY (SELECT line FROM output WHERE line IS NOT NULL) TO PROGRAM 'psql -qtAX -h localhost -p 57636 -U postgres -d regression -f results/pg_dump.tmp'
-- check the tables (should not include shards)
SELECT tablename FROM pg_tables WHERE schemaname = 'dumper' ORDER BY 1;
tablename
---------------------------------------------------------------------
data
(1 row)
DROP SCHEMA dumper CASCADE;
NOTICE: drop cascades to table data

4
src/test/regress/expected/upgrade_list_citus_objects.out
View File

@ -97,7 +97,9 @@ ORDER BY 1;
function citus_shard_allowed_on_node_true(bigint,integer)
function citus_shard_cost_1(bigint)
function citus_shard_cost_by_disk_size(bigint)
function citus_shard_indexes_on_worker()
function citus_shard_sizes()
function citus_shards_on_worker()
function citus_stat_statements()
function citus_stat_statements_reset()
function citus_table_is_visible(oid)
@ -264,5 +266,5 @@ ORDER BY 1;
view citus_worker_stat_activity
view pg_dist_shard_placement
view time_partitions
(248 rows)
(250 rows)

3
src/test/regress/multi_1_schedule
View File

@ -196,12 +196,13 @@ test: local_dist_join_modifications
test: local_table_join
test: local_dist_join_mixed
test: citus_local_dist_joins
test: pg_dump
# ---------
# multi_copy creates hash and range-partitioned tables and performs COPY
# multi_router_planner creates hash partitioned tables.
# ---------
test: multi_copy fast_path_router_modify pg_dump
test: multi_copy fast_path_router_modify
test: multi_router_planner
# These 2 tests have prepared statements which sometimes get invalidated by concurrent tests,
# changing the debug output. We should not run them in parallel with others

3
src/test/regress/pg_regress_multi.pl
View File

@ -465,6 +465,9 @@ push(@pgOptions, "citus.node_connection_timeout=${connectionTimeout}");
push(@pgOptions, "citus.explain_analyze_sort_method='taskId'");
push(@pgOptions, "citus.enable_manual_changes_to_shards=on");
# Some tests look at shards in pg_class, make sure we can usually see them:
push(@pgOptions, "citus.hide_shards_from_app_name_prefixes='psql,pg_dump'");
# we disable slow start by default to encourage parallelism within tests
push(@pgOptions, "citus.executor_slow_start_interval=0ms");

105
src/test/regress/sql/multi_mx_hide_shard_names.sql
View File

@ -31,28 +31,18 @@ SELECT create_distributed_table('test_table', 'id');
-- first show that the views does not show
-- any shards on the coordinator as expected
SELECT * FROM citus_shards_on_worker;
SELECT * FROM citus_shard_indexes_on_worker;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names';
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names';
-- now show that we see the shards, but not the
-- indexes as there are no indexes
\c - - - :worker_1_port
\c postgresql://postgres@localhost::worker_1_port/regression?application_name=psql
SET search_path TO 'mx_hide_shard_names';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
-- also show that nested calls to pg_table_is_visible works fine
-- if both of the calls to the pg_table_is_visible haven't been
-- replaced, we would get 0 rows in the output
SELECT
pg_table_is_visible((SELECT
"t1"."Name"::regclass
FROM
citus_shards_on_worker as t1
WHERE
NOT pg_table_is_visible("t1"."Name"::regclass)
LIMIT
1));
-- shards are hidden when using psql as application_name
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
-- now create an index
\c - - - :master_port
@ -61,20 +51,69 @@ CREATE INDEX test_index ON mx_hide_shard_names.test_table(id);
-- now show that we see the shards, and the
-- indexes as well
\c - - - :worker_1_port
\c postgresql://postgres@localhost::worker_1_port/regression?application_name=psql
SET search_path TO 'mx_hide_shard_names';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
-- shards are hidden when using psql as application_name
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
-- changing application_name reveals the shards
SET application_name TO '';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
RESET application_name;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
-- changing application_name in transaction reveals the shards
BEGIN;
SET LOCAL application_name TO '';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
-- now with session-level GUC, but ROLLBACK;
BEGIN;
SET application_name TO '';
ROLLBACK;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
-- we should hide correctly based on application_name with savepoints
BEGIN;
SAVEPOINT s1;
SET application_name TO '';
-- changing application_name reveals the shards
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK TO SAVEPOINT s1;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK;
-- changing citus.hide_shards_from_app_name_prefixes reveals the shards
BEGIN;
SET LOCAL citus.hide_shards_from_app_name_prefixes TO 'notpsql';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
-- we should be able to select from the shards directly if we
-- know the name of the tables
SELECT count(*) FROM test_table_1130000;
-- disable the config so that table becomes visible
SELECT pg_table_is_visible('test_table_1130000'::regclass);
SET citus.override_table_visibility TO FALSE;
-- shards on the search_path still match pg_table_is_visible
SELECT pg_table_is_visible('test_table_1130000'::regclass);
-- shards on the search_path do not match citus_table_is_visible
SELECT citus_table_is_visible('test_table_1130000'::regclass);
\c - - - :master_port
-- make sure that we're resilient to the edge cases
-- such that the table name includes the shard number
@ -95,7 +134,7 @@ SET search_path TO 'mx_hide_shard_names';
-- name already exists :)
CREATE TABLE test_table_2_1130000(id int, time date);
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
\d
@ -111,14 +150,10 @@ CREATE INDEX test_index ON mx_hide_shard_names_2.test_table(id);
\c - - - :worker_1_port
SET search_path TO 'mx_hide_shard_names';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
SET search_path TO 'mx_hide_shard_names_2';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
SET search_path TO 'mx_hide_shard_names_2, mx_hide_shard_names';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names' ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names_2' ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_names_2' ORDER BY 2;
-- now try very long table names
\c - - - :master_port
@ -137,7 +172,7 @@ SELECT create_distributed_table('too_long_12345678901234567890123456789012345678
\c - - - :worker_1_port
SET search_path TO 'mx_hide_shard_names_3';
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'mx_hide_shard_names_3' ORDER BY 2;
\d
@ -159,8 +194,8 @@ SELECT create_distributed_table('"CiTuS.TeeN"."TeeNTabLE.1!?!"', 'TeNANt_Id');
\c - - - :worker_1_port
SET search_path TO "CiTuS.TeeN";
SELECT * FROM citus_shards_on_worker ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker ORDER BY 2;
SELECT * FROM citus_shards_on_worker WHERE "Schema" = 'CiTuS.TeeN' ORDER BY 2;
SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'CiTuS.TeeN' ORDER BY 2;
\d
\di

25
src/test/regress/sql/pg_dump.sql
View File

@ -101,3 +101,28 @@ COPY dist_columnar TO STDOUT;
SELECT indexname FROM pg_indexes WHERE tablename = 'weird.table' ORDER BY indexname;
DROP SCHEMA dumper CASCADE;
CREATE SCHEMA dumper;
CREATE TABLE data (
key int,
value text
);
SELECT create_distributed_table('data', 'key');
COPY data FROM STDIN WITH (format csv, delimiter '|', escape '\');
1|{"this":"is","json":1}
2|{"$\"":9}
3|{"{}":" "}
4|{}
\.
-- run pg_dump on worker (which has shards)
\COPY output FROM PROGRAM 'PGAPPNAME=pg_dump pg_dump -f results/pg_dump.tmp -h localhost -p 57637 -U postgres -d regression -n dumper --quote-all-identifiers'
-- restore pg_dump from worker via coordinator
DROP SCHEMA dumper CASCADE;
\COPY (SELECT line FROM output WHERE line IS NOT NULL) TO PROGRAM 'psql -qtAX -h localhost -p 57636 -U postgres -d regression -f results/pg_dump.tmp'
-- check the tables (should not include shards)
SELECT tablename FROM pg_tables WHERE schemaname = 'dumper' ORDER BY 1;
DROP SCHEMA dumper CASCADE;