From 7cff8719c2166ba43aca36559b377f643f4f6665 Mon Sep 17 00:00:00 2001 From: Metin Doslu Date: Thu, 19 Jan 2017 17:02:24 +0200 Subject: [PATCH] Add worker_hash() and a stub for isolate_tenant_to_new_shard() --- src/backend/distributed/Makefile | 4 +- .../distributed/citus--6.1-16--6.1-17.sql | 19 ++++ src/backend/distributed/citus.control | 2 +- .../distributed/master/master_split_shards.c | 89 +++++++++++++++++++ src/test/regress/expected/multi_extension.out | 1 + src/test/regress/expected/multi_utilities.out | 23 +++++ src/test/regress/multi_schedule | 2 +- .../regress/multi_task_tracker_extra_schedule | 2 +- src/test/regress/sql/multi_extension.sql | 1 + src/test/regress/sql/multi_utilities.sql | 8 ++ 10 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 src/backend/distributed/citus--6.1-16--6.1-17.sql create mode 100644 src/backend/distributed/master/master_split_shards.c diff --git a/src/backend/distributed/Makefile b/src/backend/distributed/Makefile index d7065afc8..541358ca5 100644 --- a/src/backend/distributed/Makefile +++ b/src/backend/distributed/Makefile @@ -9,7 +9,7 @@ EXTVERSIONS = 5.0 5.0-1 5.0-2 \ 5.1-1 5.1-2 5.1-3 5.1-4 5.1-5 5.1-6 5.1-7 5.1-8 \ 5.2-1 5.2-2 5.2-3 5.2-4 \ 6.0-1 6.0-2 6.0-3 6.0-4 6.0-5 6.0-6 6.0-7 6.0-8 6.0-9 6.0-10 6.0-11 6.0-12 6.0-13 6.0-14 6.0-15 6.0-16 6.0-17 6.0-18 \ - 6.1-1 6.1-2 6.1-3 6.1-4 6.1-5 6.1-6 6.1-7 6.1-8 6.1-9 6.1-10 6.1-11 6.1-12 6.1-13 6.1-14 6.1-15 6.1-16 + 6.1-1 6.1-2 6.1-3 6.1-4 6.1-5 6.1-6 6.1-7 6.1-8 6.1-9 6.1-10 6.1-11 6.1-12 6.1-13 6.1-14 6.1-15 6.1-16 6.1-17 # All citus--*.sql files in the source directory DATA = $(patsubst $(citus_abs_srcdir)/%.sql,%.sql,$(wildcard $(citus_abs_srcdir)/$(EXTENSION)--*--*.sql)) @@ -127,6 +127,8 @@ $(EXTENSION)--6.1-15.sql: $(EXTENSION)--6.1-14.sql $(EXTENSION)--6.1-14--6.1-15. cat $^ > $@ $(EXTENSION)--6.1-16.sql: $(EXTENSION)--6.1-15.sql $(EXTENSION)--6.1-15--6.1-16.sql cat $^ > $@ +$(EXTENSION)--6.1-17.sql: $(EXTENSION)--6.1-16.sql $(EXTENSION)--6.1-16--6.1-17.sql + cat $^ > $@ NO_PGXS = 1 diff --git a/src/backend/distributed/citus--6.1-16--6.1-17.sql b/src/backend/distributed/citus--6.1-16--6.1-17.sql new file mode 100644 index 000000000..5a4c2d834 --- /dev/null +++ b/src/backend/distributed/citus--6.1-16--6.1-17.sql @@ -0,0 +1,19 @@ +/* citus--6.1-16--6.1-17.sql */ + +SET search_path = 'pg_catalog'; + +CREATE FUNCTION isolate_tenant_to_new_shard(table_name regclass, tenant_id "any", cascade_option text DEFAULT '') + RETURNS bigint + LANGUAGE C STRICT + AS 'MODULE_PATHNAME', $$isolate_tenant_to_new_shard$$; +COMMENT ON FUNCTION isolate_tenant_to_new_shard(table_name regclass, tenant_id "any", cascade_option text) + IS 'isolate a tenant to its own shard and return the new shard id'; + +CREATE FUNCTION worker_hash(value "any") + RETURNS integer + LANGUAGE C STRICT + AS 'MODULE_PATHNAME', $$worker_hash$$; +COMMENT ON FUNCTION worker_hash(value "any") + IS 'calculate hashed value and return it'; + +RESET search_path; diff --git a/src/backend/distributed/citus.control b/src/backend/distributed/citus.control index f0e1fb685..57d2534ec 100644 --- a/src/backend/distributed/citus.control +++ b/src/backend/distributed/citus.control @@ -1,6 +1,6 @@ # Citus extension comment = 'Citus distributed database' -default_version = '6.1-16' +default_version = '6.1-17' module_pathname = '$libdir/citus' relocatable = false schema = pg_catalog diff --git a/src/backend/distributed/master/master_split_shards.c b/src/backend/distributed/master/master_split_shards.c new file mode 100644 index 000000000..51c6a0543 --- /dev/null +++ b/src/backend/distributed/master/master_split_shards.c @@ -0,0 +1,89 @@ +/*------------------------------------------------------------------------- + * + * master_split_shards.c + * + * This file contains functions to split a shard according to a given + * distribution column value. + * + * Copyright (c) 2014-2017, Citus Data, Inc. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "c.h" +#include "fmgr.h" + +#include "catalog/pg_class.h" +#include "distributed/colocation_utils.h" +#include "distributed/connection_cache.h" +#include "distributed/master_protocol.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_router_planner.h" +#include "distributed/pg_dist_partition.h" +#include "distributed/pg_dist_shard.h" +#include "distributed/remote_commands.h" +#include "distributed/resource_lock.h" +#include "distributed/worker_manager.h" +#include "distributed/worker_protocol.h" +#include "distributed/worker_transaction.h" +#include "nodes/pg_list.h" +#include "storage/lock.h" +#include "utils/builtins.h" +#include "utils/elog.h" +#include "utils/errcodes.h" +#include "utils/lsyscache.h" +#include "utils/typcache.h" + + +/* declarations for dynamic loading */ +PG_FUNCTION_INFO_V1(isolate_tenant_to_new_shard); +PG_FUNCTION_INFO_V1(worker_hash); + + +/* + * isolate_tenant_to_new_shard isolates a tenant to its own shard by spliting + * the current matching shard. + */ +Datum +isolate_tenant_to_new_shard(PG_FUNCTION_ARGS) +{ + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("isolate_tenant_to_new_shard() is only supported on " + "Citus Enterprise"))); +} + + +/* + * worker_hash returns the hashed value of the given value. + */ +Datum +worker_hash(PG_FUNCTION_ARGS) +{ + Datum valueDatum = PG_GETARG_DATUM(0); + Datum hashedValueDatum = 0; + TypeCacheEntry *typeEntry = NULL; + FmgrInfo *hashFunction = NULL; + Oid valueDataType = InvalidOid; + + /* figure out hash function from the data type */ + valueDataType = get_fn_expr_argtype(fcinfo->flinfo, 0); + typeEntry = lookup_type_cache(valueDataType, TYPECACHE_HASH_PROC_FINFO); + + if (typeEntry->hash_proc_finfo.fn_oid == InvalidOid) + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot find a hash function for the input type"), + errhint("Cast input to a data type with a hash function."))); + } + + hashFunction = palloc0(sizeof(FmgrInfo)); + fmgr_info_copy(hashFunction, &(typeEntry->hash_proc_finfo), CurrentMemoryContext); + + /* calculate hash value */ + hashedValueDatum = FunctionCall1(hashFunction, valueDatum); + + PG_RETURN_INT32(hashedValueDatum); +} diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 38b310c5d..294f70af7 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -74,6 +74,7 @@ ALTER EXTENSION citus UPDATE TO '6.1-13'; ALTER EXTENSION citus UPDATE TO '6.1-14'; ALTER EXTENSION citus UPDATE TO '6.1-15'; ALTER EXTENSION citus UPDATE TO '6.1-16'; +ALTER EXTENSION citus UPDATE TO '6.1-17'; -- ensure no objects were created outside pg_catalog SELECT COUNT(*) FROM pg_depend AS pgd, diff --git a/src/test/regress/expected/multi_utilities.out b/src/test/regress/expected/multi_utilities.out index 09e1e5491..737017745 100644 --- a/src/test/regress/expected/multi_utilities.out +++ b/src/test/regress/expected/multi_utilities.out @@ -319,6 +319,29 @@ VACUUM dustbunnies; WARNING: not propagating VACUUM command to worker nodes HINT: Set citus.enable_ddl_propagation to true in order to send targeted VACUUM commands to worker nodes. SET citus.enable_ddl_propagation to DEFAULT; +-- test worker_hash +SELECT worker_hash(123); + worker_hash +------------- + -205084363 +(1 row) + +SELECT worker_hash('1997-08-08'::date); + worker_hash +------------- + -499701663 +(1 row) + +-- test a custom type (this test should run after multi_data_types) +SELECT worker_hash('(1, 2)'); +ERROR: cannot find a hash function for the input type +HINT: Cast input to a data type with a hash function. +SELECT worker_hash('(1, 2)'::test_composite_type); + worker_hash +------------- + -1895345704 +(1 row) + -- TODO: support VERBOSE -- VACUUM VERBOSE dustbunnies; -- VACUUM (FULL, VERBOSE) dustbunnies; diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 8e2982a92..6651ab562 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -135,9 +135,9 @@ test: multi_repair_shards test: multi_modifications test: multi_upsert test: multi_simple_queries -test: multi_utilities test: multi_create_insert_proxy test: multi_data_types +test: multi_utilities test: multi_repartition_udt test: multi_repartitioned_subquery_udf test: multi_modifying_xacts diff --git a/src/test/regress/multi_task_tracker_extra_schedule b/src/test/regress/multi_task_tracker_extra_schedule index 30c0a3653..c647e4eb2 100644 --- a/src/test/regress/multi_task_tracker_extra_schedule +++ b/src/test/regress/multi_task_tracker_extra_schedule @@ -90,9 +90,9 @@ test: multi_repair_shards test: multi_modifications test: multi_upsert test: multi_simple_queries -test: multi_utilities test: multi_create_insert_proxy test: multi_data_types +test: multi_utilities # --------- # multi_copy creates hash and range-partitioned tables and performs COPY diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql index 38a7560e1..647cedde4 100644 --- a/src/test/regress/sql/multi_extension.sql +++ b/src/test/regress/sql/multi_extension.sql @@ -74,6 +74,7 @@ ALTER EXTENSION citus UPDATE TO '6.1-13'; ALTER EXTENSION citus UPDATE TO '6.1-14'; ALTER EXTENSION citus UPDATE TO '6.1-15'; ALTER EXTENSION citus UPDATE TO '6.1-16'; +ALTER EXTENSION citus UPDATE TO '6.1-17'; -- ensure no objects were created outside pg_catalog SELECT COUNT(*) diff --git a/src/test/regress/sql/multi_utilities.sql b/src/test/regress/sql/multi_utilities.sql index e0a078cc9..e85d7ee85 100644 --- a/src/test/regress/sql/multi_utilities.sql +++ b/src/test/regress/sql/multi_utilities.sql @@ -209,6 +209,14 @@ SET citus.enable_ddl_propagation to false; VACUUM dustbunnies; SET citus.enable_ddl_propagation to DEFAULT; +-- test worker_hash +SELECT worker_hash(123); +SELECT worker_hash('1997-08-08'::date); + +-- test a custom type (this test should run after multi_data_types) +SELECT worker_hash('(1, 2)'); +SELECT worker_hash('(1, 2)'::test_composite_type); + -- TODO: support VERBOSE -- VACUUM VERBOSE dustbunnies; -- VACUUM (FULL, VERBOSE) dustbunnies;