From 31c8f279acf82bc6931e675b0fec617a1e8fbf1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96nder=20Kalac=C4=B1?= Date: Mon, 18 Oct 2021 13:46:49 +0200 Subject: [PATCH] Add helper UDFs to inspect object dependencies (#5293) - citus_get_all_dependencies_for_object: emulate what Citus would qualify as dependency when adding a new node - citus_get_dependencies_for_object: emulate what Citus would qualify as dependency when creating an object Example use: ```SQL -- find all the depedencies of table test SELECT pg_identify_object(t.classid, t.objid, t.objsubid) FROM (SELECT * FROM pg_get_object_address('table', '{test}', '{}')) as addr JOIN LATERAL citus_get_all_dependencies_for_object(addr.classid, addr.objid, addr.objsubid) as t(classid oid, objid oid, objsubid int) ON TRUE ORDER BY 1; ``` --- src/backend/distributed/metadata/dependency.c | 25 ++++ src/backend/distributed/test/dependency.c | 113 ++++++++++++++++++ src/include/distributed/metadata/dependency.h | 1 + .../expected/object_propagation_debug.out | 102 ++++++++++++++++ src/test/regress/multi_schedule | 2 + .../regress/sql/object_propagation_debug.sql | 77 ++++++++++++ 6 files changed, 320 insertions(+) create mode 100644 src/backend/distributed/test/dependency.c create mode 100644 src/test/regress/expected/object_propagation_debug.out create mode 100644 src/test/regress/sql/object_propagation_debug.sql diff --git a/src/backend/distributed/metadata/dependency.c b/src/backend/distributed/metadata/dependency.c index d092cbfd3..340f5ea41 100644 --- a/src/backend/distributed/metadata/dependency.c +++ b/src/backend/distributed/metadata/dependency.c @@ -207,6 +207,31 @@ GetDependenciesForObject(const ObjectAddress *target) } +/* + * GetAllDependenciesForObject returns a list of all the ObjectAddresses to be + * created in order before the target object could safely be created on a + * worker. As a caller, you probably need GetDependenciesForObject() which + * eliminates already distributed objects from the returned list. + * + * Some of the object might already be created on a worker. It should be created + * in an idempotent way. + */ +List * +GetAllDependenciesForObject(const ObjectAddress *target) +{ + ObjectAddressCollector collector = { 0 }; + InitObjectAddressCollector(&collector); + + RecurseObjectDependencies(*target, + &ExpandCitusSupportedTypes, + &FollowAllSupportedDependencies, + &ApplyAddToDependencyList, + &collector); + + return collector.dependencyList; +} + + /* * OrderObjectAddressListInDependencyOrder given a list of ObjectAddresses return a new * list of the same ObjectAddresses ordered on dependency order where dependencies diff --git a/src/backend/distributed/test/dependency.c b/src/backend/distributed/test/dependency.c new file mode 100644 index 000000000..79ad1c139 --- /dev/null +++ b/src/backend/distributed/test/dependency.c @@ -0,0 +1,113 @@ +/*------------------------------------------------------------------------- + * + * test/src/dependency.c + * + * This file contains functions to exercise dependency resolution for objects. + * + * Copyright (c) Citus Data, Inc. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "c.h" +#include "fmgr.h" + +#include "distributed/listutils.h" +#include "distributed/metadata/dependency.h" +#include "distributed/metadata_cache.h" +#include "distributed/tuplestore.h" + + +PG_FUNCTION_INFO_V1(citus_get_all_dependencies_for_object); +PG_FUNCTION_INFO_V1(citus_get_dependencies_for_object); + + +/* + * citus_get_all_dependencies_for_object(classid oid, objid oid, objsubid int) + * + * citus_get_all_dependencies_for_object gets an object and returns all of its + * dependencies irrespective of whether the dependencies are already distributed + * or not. + * + * This is to emulate what Citus would qualify as dependency when adding a new + * node. + */ +Datum +citus_get_all_dependencies_for_object(PG_FUNCTION_ARGS) +{ + CheckCitusVersion(ERROR); + + Oid classid = PG_GETARG_OID(0); + Oid objid = PG_GETARG_OID(1); + int32 objsubid = PG_GETARG_INT32(2); + + TupleDesc tupleDescriptor = NULL; + Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); + + ObjectAddress address = { 0 }; + ObjectAddressSubSet(address, classid, objid, objsubid); + + List *dependencies = GetAllDependenciesForObject(&address); + ObjectAddress *dependency = NULL; + foreach_ptr(dependency, dependencies) + { + Datum values[3]; + bool isNulls[3]; + + memset(values, 0, sizeof(values)); + memset(isNulls, 0, sizeof(isNulls)); + + values[0] = ObjectIdGetDatum(dependency->classId); + values[1] = ObjectIdGetDatum(dependency->objectId); + values[2] = Int32GetDatum(dependency->objectSubId); + + tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); + } + + PG_RETURN_VOID(); +} + + +/* + * citus_get_dependencies_for_object(classid oid, objid oid, objsubid int) + * + * citus_get_dependencies_for_object gets an object and returns all of its + * dependencies that are not already distributed. + * + * This is to emulate what Citus would qualify as dependency when creating + * a new object. + */ +Datum +citus_get_dependencies_for_object(PG_FUNCTION_ARGS) +{ + CheckCitusVersion(ERROR); + + Oid classid = PG_GETARG_OID(0); + Oid objid = PG_GETARG_OID(1); + int32 objsubid = PG_GETARG_INT32(2); + + TupleDesc tupleDescriptor = NULL; + Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); + + ObjectAddress address = { 0 }; + ObjectAddressSubSet(address, classid, objid, objsubid); + + List *dependencies = GetDependenciesForObject(&address); + ObjectAddress *dependency = NULL; + foreach_ptr(dependency, dependencies) + { + Datum values[3]; + bool isNulls[3]; + + memset(values, 0, sizeof(values)); + memset(isNulls, 0, sizeof(isNulls)); + + values[0] = ObjectIdGetDatum(dependency->classId); + values[1] = ObjectIdGetDatum(dependency->objectId); + values[2] = Int32GetDatum(dependency->objectSubId); + + tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); + } + + PG_RETURN_VOID(); +} diff --git a/src/include/distributed/metadata/dependency.h b/src/include/distributed/metadata/dependency.h index 66ae30fd3..d20103011 100644 --- a/src/include/distributed/metadata/dependency.h +++ b/src/include/distributed/metadata/dependency.h @@ -19,6 +19,7 @@ extern List * GetUniqueDependenciesList(List *objectAddressesList); extern List * GetDependenciesForObject(const ObjectAddress *target); +extern List * GetAllDependenciesForObject(const ObjectAddress *target); extern List * OrderObjectAddressListInDependencyOrder(List *objectAddressList); extern bool SupportedDependencyByCitus(const ObjectAddress *address); extern List * GetPgDependTuplesForDependingObjects(Oid targetObjectClassId, diff --git a/src/test/regress/expected/object_propagation_debug.out b/src/test/regress/expected/object_propagation_debug.out new file mode 100644 index 000000000..248ec4fd4 --- /dev/null +++ b/src/test/regress/expected/object_propagation_debug.out @@ -0,0 +1,102 @@ +CREATE SCHEMA "object prop"; +SET search_path TO "object prop"; +CREATE OR REPLACE FUNCTION citus_get_all_dependencies_for_object(classid oid, objid oid, objsubid int) + RETURNS SETOF RECORD + LANGUAGE C STRICT + AS 'citus', $$citus_get_all_dependencies_for_object$$; +COMMENT ON FUNCTION citus_get_all_dependencies_for_object(classid oid, objid oid, objsubid int) +IS 'emulate what Citus would qualify as dependency when adding a new node'; +CREATE OR REPLACE FUNCTION citus_get_dependencies_for_object(classid oid, objid oid, objsubid int) + RETURNS SETOF RECORD + LANGUAGE C STRICT + AS 'citus', $$citus_get_dependencies_for_object$$; +COMMENT ON FUNCTION citus_get_dependencies_for_object(classid oid, objid oid, objsubid int) +IS 'emulate what Citus would qualify as dependency when creating the object'; +create type t1 as (a int); +create table test(a int, b t1); +SELECT create_distributed_table('test', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE VIEW v1 AS SELECT * FROM test; +-- find all the dependencies of table test +SELECT + pg_identify_object(t.classid, t.objid, t.objsubid) +FROM + (SELECT * FROM pg_get_object_address('table', '{test}', '{}')) as addr +JOIN LATERAL + citus_get_all_dependencies_for_object(addr.classid, addr.objid, addr.objsubid) as t(classid oid, objid oid, objsubid int) +ON TRUE + ORDER BY 1; + pg_identify_object +--------------------------------------------------------------------- + ("composite type","""object prop""",t1,"""object prop"".t1") + (schema,,"""object prop""","""object prop""") + (type,"""object prop""",t1,"""object prop"".t1") +(3 rows) + +-- find all the dependencies of view v1 +SELECT + pg_identify_object(t.classid, t.objid, t.objsubid) +FROM + (SELECT * FROM pg_get_object_address('view', '{v1}', '{}')) as addr +JOIN LATERAL + citus_get_all_dependencies_for_object(addr.classid, addr.objid, addr.objsubid) as t(classid oid, objid oid, objsubid int) +ON TRUE + ORDER BY 1; + pg_identify_object +--------------------------------------------------------------------- + ("composite type","""object prop""",t1,"""object prop"".t1") + (schema,,"""object prop""","""object prop""") + (type,"""object prop""",t1,"""object prop"".t1") +(3 rows) + +-- find all the dependencies of type t1 +SELECT + pg_identify_object(t.classid, t.objid, t.objsubid) +FROM + (SELECT * FROM pg_get_object_address('type', '{t1}', '{}')) as addr +JOIN LATERAL + citus_get_all_dependencies_for_object(addr.classid, addr.objid, addr.objsubid) as t(classid oid, objid oid, objsubid int) +ON TRUE + ORDER BY 1; + pg_identify_object +--------------------------------------------------------------------- + ("composite type","""object prop""",t1,"""object prop"".t1") + (schema,,"""object prop""","""object prop""") +(2 rows) + +-- find non-distributed dependencies of table test +SELECT + pg_identify_object(t.classid, t.objid, t.objsubid) +FROM + (SELECT * FROM pg_get_object_address('table', '{test}', '{}')) as addr +JOIN LATERAL + citus_get_dependencies_for_object(addr.classid, addr.objid, addr.objsubid) as t(classid oid, objid oid, objsubid int) +ON TRUE + ORDER BY 1; + pg_identify_object +--------------------------------------------------------------------- +(0 rows) + +-- find non-distributed dependencies of the local table test2 +CREATE SCHEMA objectprop2; +create table objectprop2.test2(a int, b t1); +SET search_path TO objectprop2; +SELECT + pg_identify_object(t.classid, t.objid, t.objsubid) +FROM + (SELECT * FROM pg_get_object_address('table', '{test2}', '{}')) as addr +JOIN LATERAL + "object prop".citus_get_dependencies_for_object(addr.classid, addr.objid, addr.objsubid) as t(classid oid, objid oid, objsubid int) +ON TRUE + ORDER BY 1; + pg_identify_object +--------------------------------------------------------------------- + (schema,,objectprop2,objectprop2) +(1 row) + +SET client_min_messages TO ERROR; +DROP SCHEMA "object prop", objectprop2 CASCADE; diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index c44aa6b62..f45224301 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -98,6 +98,8 @@ test: binary_protocol test: alter_table_set_access_method test: alter_distributed_table test: issue_5248 +test: object_propagation_debug + # --------- # test that no tests leaked intermediate results. This should always be last diff --git a/src/test/regress/sql/object_propagation_debug.sql b/src/test/regress/sql/object_propagation_debug.sql new file mode 100644 index 000000000..0b676bd38 --- /dev/null +++ b/src/test/regress/sql/object_propagation_debug.sql @@ -0,0 +1,77 @@ +CREATE SCHEMA "object prop"; +SET search_path TO "object prop"; + +CREATE OR REPLACE FUNCTION citus_get_all_dependencies_for_object(classid oid, objid oid, objsubid int) + RETURNS SETOF RECORD + LANGUAGE C STRICT + AS 'citus', $$citus_get_all_dependencies_for_object$$; +COMMENT ON FUNCTION citus_get_all_dependencies_for_object(classid oid, objid oid, objsubid int) +IS 'emulate what Citus would qualify as dependency when adding a new node'; + +CREATE OR REPLACE FUNCTION citus_get_dependencies_for_object(classid oid, objid oid, objsubid int) + RETURNS SETOF RECORD + LANGUAGE C STRICT + AS 'citus', $$citus_get_dependencies_for_object$$; +COMMENT ON FUNCTION citus_get_dependencies_for_object(classid oid, objid oid, objsubid int) +IS 'emulate what Citus would qualify as dependency when creating the object'; + +create type t1 as (a int); +create table test(a int, b t1); +SELECT create_distributed_table('test', 'a'); +CREATE VIEW v1 AS SELECT * FROM test; + +-- find all the dependencies of table test +SELECT + pg_identify_object(t.classid, t.objid, t.objsubid) +FROM + (SELECT * FROM pg_get_object_address('table', '{test}', '{}')) as addr +JOIN LATERAL + citus_get_all_dependencies_for_object(addr.classid, addr.objid, addr.objsubid) as t(classid oid, objid oid, objsubid int) +ON TRUE + ORDER BY 1; + +-- find all the dependencies of view v1 +SELECT + pg_identify_object(t.classid, t.objid, t.objsubid) +FROM + (SELECT * FROM pg_get_object_address('view', '{v1}', '{}')) as addr +JOIN LATERAL + citus_get_all_dependencies_for_object(addr.classid, addr.objid, addr.objsubid) as t(classid oid, objid oid, objsubid int) +ON TRUE + ORDER BY 1; + +-- find all the dependencies of type t1 +SELECT + pg_identify_object(t.classid, t.objid, t.objsubid) +FROM + (SELECT * FROM pg_get_object_address('type', '{t1}', '{}')) as addr +JOIN LATERAL + citus_get_all_dependencies_for_object(addr.classid, addr.objid, addr.objsubid) as t(classid oid, objid oid, objsubid int) +ON TRUE + ORDER BY 1; + +-- find non-distributed dependencies of table test +SELECT + pg_identify_object(t.classid, t.objid, t.objsubid) +FROM + (SELECT * FROM pg_get_object_address('table', '{test}', '{}')) as addr +JOIN LATERAL + citus_get_dependencies_for_object(addr.classid, addr.objid, addr.objsubid) as t(classid oid, objid oid, objsubid int) +ON TRUE + ORDER BY 1; + +-- find non-distributed dependencies of the local table test2 +CREATE SCHEMA objectprop2; +create table objectprop2.test2(a int, b t1); +SET search_path TO objectprop2; +SELECT + pg_identify_object(t.classid, t.objid, t.objsubid) +FROM + (SELECT * FROM pg_get_object_address('table', '{test2}', '{}')) as addr +JOIN LATERAL + "object prop".citus_get_dependencies_for_object(addr.classid, addr.objid, addr.objsubid) as t(classid oid, objid oid, objsubid int) +ON TRUE + ORDER BY 1; + +SET client_min_messages TO ERROR; +DROP SCHEMA "object prop", objectprop2 CASCADE;