diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index c3c21e72c..ea33cd898 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -71,23 +71,6 @@ CitusExecutorStart(QueryDesc *queryDesc, int eflags) { PlannedStmt *plannedStmt = queryDesc->plannedstmt; - if (CitusHasBeenLoaded()) - { - if (IsLocalReferenceTableJoinPlan(plannedStmt) && - IsMultiStatementTransaction()) - { - /* - * Currently we don't support this to avoid problems with tuple - * visibility, locking, etc. For example, change to the reference - * table can go through a MultiConnection, which won't be visible - * to the locally planned queries. - */ - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot join local tables and reference tables in " - "a transaction block"))); - } - } - /* * We cannot modify XactReadOnly on Windows because it is not * declared with PGDLLIMPORT. @@ -143,6 +126,23 @@ CitusExecutorRun(QueryDesc *queryDesc, FunctionCallLevel++; } + if (CitusHasBeenLoaded()) + { + if (IsLocalReferenceTableJoinPlan(queryDesc->plannedstmt) && + IsMultiStatementTransaction()) + { + /* + * Currently we don't support this to avoid problems with tuple + * visibility, locking, etc. For example, change to the reference + * table can go through a MultiConnection, which won't be visible + * to the locally planned queries. + */ + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot join local tables and reference tables in " + "a transaction block"))); + } + } + /* * Disable execution of ALTER TABLE constraint validation queries. These * constraints will be validated in worker nodes, so running these queries diff --git a/src/test/regress/base_schedule b/src/test/regress/base_schedule index fae2a8aab..3b94b1d23 100644 --- a/src/test/regress/base_schedule +++ b/src/test/regress/base_schedule @@ -3,5 +3,6 @@ # ---------- test: multi_cluster_management test: multi_test_helpers multi_create_fdw +test: multi_test_catalog_views test: multi_create_table multi_behavioral_analytics_create_table test: multi_load_data diff --git a/src/test/regress/before_pg_upgrade_schedule b/src/test/regress/before_pg_upgrade_schedule index b527ff257..dd966dccd 100644 --- a/src/test/regress/before_pg_upgrade_schedule +++ b/src/test/regress/before_pg_upgrade_schedule @@ -1,3 +1,5 @@ # The basic tests runs analyze which depends on shard numbers +test: multi_test_helpers +test: multi_test_catalog_views test: upgrade_basic_before test: upgrade_type_before upgrade_ref2ref_before upgrade_distributed_function_before diff --git a/src/test/regress/expected/distributed_functions.out b/src/test/regress/expected/distributed_functions.out index 84ebb350c..4b9fd2794 100644 --- a/src/test/regress/expected/distributed_functions.out +++ b/src/test/regress/expected/distributed_functions.out @@ -13,19 +13,6 @@ CREATE SCHEMA function_tests AUTHORIZATION functionuser; CREATE SCHEMA function_tests2 AUTHORIZATION functionuser; SET search_path TO function_tests; SET citus.shard_count TO 4; --- set sync intervals to less than 15s so wait_until_metadata_sync never times out -ALTER SYSTEM SET citus.metadata_sync_interval TO 3000; -ALTER SYSTEM SET citus.metadata_sync_retry_interval TO 500; -SELECT pg_reload_conf(); - pg_reload_conf ----------------- - t -(1 row) - -CREATE OR REPLACE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) - RETURNS void - LANGUAGE C STRICT - AS 'citus'; -- Create and distribute a simple function CREATE FUNCTION add(integer, integer) RETURNS integer AS 'select $1 + $2;' @@ -624,7 +611,7 @@ SELECT create_distributed_function('add_with_param_names(int, int)', '$1', coloc ERROR: cannot colocate function "add_with_param_names" and table "replicated_table_func_test" DETAIL: Citus currently only supports colocating function with distributed tables that are created using streaming replication model. HINT: When distributing tables make sure that citus.replication_model = 'streaming' -SELECT wait_until_metadata_sync(); +SELECT public.wait_until_metadata_sync(); wait_until_metadata_sync -------------------------- @@ -752,7 +739,7 @@ SELECT create_distributed_function('add_with_param_names(int, int)', 'val1'); ERROR: cannot distribute the function "add_with_param_names" since there is no table to colocate with HINT: Provide a distributed table via "colocate_with" option to create_distributed_function() -- sync metadata to workers for consistent results when clearing objects -SELECT wait_until_metadata_sync(); +SELECT public.wait_until_metadata_sync(); wait_until_metadata_sync -------------------------- diff --git a/src/test/regress/expected/multi_multiuser.out b/src/test/regress/expected/multi_multiuser.out index e16915c73..28d7f114b 100644 --- a/src/test/regress/expected/multi_multiuser.out +++ b/src/test/regress/expected/multi_multiuser.out @@ -5,18 +5,6 @@ -- SET citus.next_shard_id TO 1420000; SET citus.shard_replication_factor TO 1; -ALTER SYSTEM SET citus.metadata_sync_interval TO 3000; -ALTER SYSTEM SET citus.metadata_sync_retry_interval TO 500; -SELECT pg_reload_conf(); - pg_reload_conf ----------------- - t -(1 row) - -CREATE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) - RETURNS void - LANGUAGE C STRICT - AS 'citus'; CREATE TABLE test (id integer, val integer); SELECT create_distributed_table('test', 'id'); create_distributed_table @@ -433,7 +421,7 @@ INSERT INTO full_access_user_schema.t1 VALUES (1),(2),(3); -- not allowed to create a table SELECT create_distributed_table('full_access_user_schema.t1', 'id'); ERROR: permission denied for schema full_access_user_schema -CONTEXT: while executing command on localhost:57637 +CONTEXT: while executing command on localhost:57638 RESET ROLE; SET ROLE usage_access; CREATE TYPE usage_access_type AS ENUM ('a', 'b'); @@ -684,7 +672,7 @@ ERROR: could not receive file "base/pgsql_job_cache/job_0042/task_000001/p_0000 -- different user should not be able to fetch partition file SET ROLE usage_access; SELECT worker_fetch_partition_file(42, 1, 1, 1, 'localhost', :worker_1_port); -WARNING: could not open file "base/pgsql_job_cache/job_0042/task_000001/p_00001.37457": No such file or directory +WARNING: could not open file "base/pgsql_job_cache/job_0042/task_000001/p_00001.44518": No such file or directory CONTEXT: while executing command on localhost:57637 ERROR: could not receive file "base/pgsql_job_cache/job_0042/task_000001/p_00001" from localhost:57637 -- only the user whom created the files should be able to fetch @@ -723,7 +711,7 @@ RESET ROLE; -- test that the super user is unable to read the contents of the intermediate file, -- although it does create the table SELECT worker_merge_files_into_table(42, 1, ARRAY['a'], ARRAY['integer']); -WARNING: Task file "task_000001.36145" does not have expected suffix ".10" +WARNING: Task file "task_000001.43115" does not have expected suffix ".10" worker_merge_files_into_table ------------------------------- @@ -765,7 +753,7 @@ SELECT worker_merge_files_and_run_query(42, 1, 'CREATE TABLE task_000001_merge(merge_column_0 int)', 'CREATE TABLE task_000001 (a) AS SELECT sum(merge_column_0) FROM task_000001_merge' ); -WARNING: Task file "task_000001.36145" does not have expected suffix ".10" +WARNING: Task file "task_000001.43115" does not have expected suffix ".10" worker_merge_files_and_run_query ---------------------------------- diff --git a/src/test/regress/expected/multi_mx_add_coordinator.out b/src/test/regress/expected/multi_mx_add_coordinator.out index 2c700dc05..3b41a445b 100644 --- a/src/test/regress/expected/multi_mx_add_coordinator.out +++ b/src/test/regress/expected/multi_mx_add_coordinator.out @@ -6,6 +6,16 @@ SET citus.next_shard_id TO 7000000; SET citus.next_placement_id TO 7000000; SET citus.replication_model TO streaming; SET client_min_messages TO WARNING; +CREATE USER reprefuser WITH LOGIN; +SELECT run_command_on_workers('CREATE USER reprefuser WITH LOGIN'); + run_command_on_workers +----------------------------------- + (localhost,57637,t,"CREATE ROLE") + (localhost,57638,t,"CREATE ROLE") +(2 rows) + +SET citus.enable_alter_role_propagation TO ON; +ALTER ROLE reprefuser WITH CREATEDB; SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0); ?column? ---------- @@ -33,6 +43,32 @@ SELECT create_reference_table('ref'); (1 row) +-- alter role from mx worker isn't propagated +\c - - - :worker_1_port +SET citus.enable_alter_role_propagation TO ON; +ALTER ROLE reprefuser WITH CREATEROLE; +select rolcreatedb, rolcreaterole from pg_roles where rolname = 'reprefuser'; + rolcreatedb | rolcreaterole +-------------+--------------- + t | t +(1 row) + +\c - - - :worker_2_port +select rolcreatedb, rolcreaterole from pg_roles where rolname = 'reprefuser'; + rolcreatedb | rolcreaterole +-------------+--------------- + t | f +(1 row) + +\c - - - :master_port +SET search_path TO mx_add_coordinator,public; +SET client_min_messages TO WARNING; +select rolcreatedb, rolcreaterole from pg_roles where rolname = 'reprefuser'; + rolcreatedb | rolcreaterole +-------------+--------------- + t | f +(1 row) + SET citus.log_local_commands TO ON; SET client_min_messages TO DEBUG; -- if the placement policy is not round-robin, SELECTs on the reference @@ -99,6 +135,11 @@ SET search_path TO mx_add_coordinator,public; INSERT INTO ref VALUES (1), (2), (3); UPDATE ref SET a = a + 1; DELETE FROM ref WHERE a > 3; +-- Test we don't allow reference/local joins on mx workers +CREATE TABLE local_table (a int); +INSERT INTO local_table VALUES (2), (4); +SELECT r.a FROM ref r JOIN local_table lt on r.a = lt.a; +ERROR: relation local_table is not distributed \c - - - :master_port SET search_path TO mx_add_coordinator,public; SELECT * FROM ref ORDER BY a; diff --git a/src/test/regress/expected/multi_mx_node_metadata.out b/src/test/regress/expected/multi_mx_node_metadata.out index 03a07814e..c5b5718a1 100644 --- a/src/test/regress/expected/multi_mx_node_metadata.out +++ b/src/test/regress/expected/multi_mx_node_metadata.out @@ -8,45 +8,6 @@ SELECT nextval('pg_catalog.pg_dist_shardid_seq') AS last_shard_id \gset SET citus.replication_model TO streaming; SET citus.shard_count TO 8; SET citus.shard_replication_factor TO 1; --- set sync intervals to less than 15s so wait_until_metadata_sync never times out -ALTER SYSTEM SET citus.metadata_sync_interval TO 3000; -ALTER SYSTEM SET citus.metadata_sync_retry_interval TO 500; -SELECT pg_reload_conf(); - pg_reload_conf ----------------- - t -(1 row) - -CREATE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) - RETURNS void - LANGUAGE C STRICT - AS 'citus'; --- Verifies pg_dist_node and pg_dist_palcement in the given worker matches the ones in coordinator -CREATE FUNCTION verify_metadata(hostname TEXT, port INTEGER, master_port INTEGER DEFAULT 57636) - RETURNS BOOLEAN - LANGUAGE sql - AS $$ -SELECT wait_until_metadata_sync(); -WITH dist_node_summary AS ( - SELECT 'SELECT jsonb_agg(ROW(nodeid, groupid, nodename, nodeport, isactive) ORDER BY nodeid) FROM pg_dist_node' as query -), dist_node_check AS ( - SELECT count(distinct result) = 1 AS matches - FROM dist_node_summary CROSS JOIN LATERAL - master_run_on_worker(ARRAY[hostname, 'localhost'], ARRAY[port, master_port], - ARRAY[dist_node_summary.query, dist_node_summary.query], - false) -), dist_placement_summary AS ( - SELECT 'SELECT jsonb_agg(pg_dist_placement ORDER BY shardid) FROM pg_dist_placement)' AS query -), dist_placement_check AS ( - SELECT count(distinct result) = 1 AS matches - FROM dist_placement_summary CROSS JOIN LATERAL - master_run_on_worker(ARRAY[hostname, 'localhost'], ARRAY[port, master_port], - ARRAY[dist_placement_summary.query, dist_placement_summary.query], - false) -) -SELECT dist_node_check.matches AND dist_placement_check.matches -FROM dist_node_check CROSS JOIN dist_placement_check -$$; -- Simulates a readonly node by setting default_transaction_read_only. CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN) RETURNS TEXT diff --git a/src/test/regress/expected/multi_test_catalog_views.out b/src/test/regress/expected/multi_test_catalog_views.out new file mode 100644 index 000000000..80dab1b3b --- /dev/null +++ b/src/test/regress/expected/multi_test_catalog_views.out @@ -0,0 +1,104 @@ +-- The following views are intended as alternatives to \d commands, whose +-- output changed in PostgreSQL 10. In particular, they must be used any time +-- a test wishes to print out the structure of a relation, which previously +-- was safely accomplished by a \d invocation. +SELECT run_command_on_master_and_workers( +$desc_views$ +CREATE VIEW table_fkey_cols AS +SELECT rc.constraint_name AS "name", + kcu.column_name AS "column_name", + uc_kcu.column_name AS "refd_column_name", + format('%I.%I', kcu.table_schema, kcu.table_name)::regclass::oid AS relid, + format('%I.%I', uc_kcu.table_schema, uc_kcu.table_name)::regclass::oid AS refd_relid, + rc.constraint_schema AS "schema" +FROM information_schema.referential_constraints rc, + information_schema.key_column_usage kcu, + information_schema.key_column_usage uc_kcu +WHERE rc.constraint_schema = kcu.constraint_schema AND + rc.constraint_name = kcu.constraint_name AND + rc.unique_constraint_schema = uc_kcu.constraint_schema AND + rc.unique_constraint_name = uc_kcu.constraint_name; + +CREATE VIEW table_fkeys AS +SELECT name AS "Constraint", + format('FOREIGN KEY (%s) REFERENCES %s(%s)', + string_agg(DISTINCT quote_ident(column_name), ', '), + string_agg(DISTINCT refd_relid::regclass::text, ', '), + string_agg(DISTINCT quote_ident(refd_column_name), ', ')) AS "Definition", + "relid" +FROM table_fkey_cols +GROUP BY (name, relid); + +CREATE VIEW table_attrs AS +SELECT c.column_name AS "name", + c.data_type AS "type", + CASE + WHEN character_maximum_length IS NOT NULL THEN + format('(%s)', character_maximum_length) + WHEN data_type = 'numeric' AND numeric_precision IS NOT NULL THEN + format('(%s,%s)', numeric_precision, numeric_scale) + ELSE '' + END AS "modifier", + c.column_default AS "default", + (NOT c.is_nullable::boolean) AS "notnull", + format('%I.%I', c.table_schema, c.table_name)::regclass::oid AS "relid" +FROM information_schema.columns AS c +ORDER BY ordinal_position; + +CREATE VIEW table_desc AS +SELECT "name" AS "Column", + "type" || "modifier" AS "Type", + rtrim(( + CASE "notnull" + WHEN true THEN 'not null ' + ELSE '' + END + ) || ( + CASE WHEN "default" IS NULL THEN '' + ELSE 'default ' || "default" + END + )) AS "Modifiers", + "relid" +FROM table_attrs; + +CREATE VIEW table_checks AS +SELECT cc.constraint_name AS "Constraint", + ('CHECK ' || regexp_replace(check_clause, '^\((.*)\)$', '\1')) AS "Definition", + format('%I.%I', ccu.table_schema, ccu.table_name)::regclass::oid AS relid +FROM information_schema.check_constraints cc, + information_schema.constraint_column_usage ccu +WHERE cc.constraint_schema = ccu.constraint_schema AND + cc.constraint_name = ccu.constraint_name +ORDER BY cc.constraint_name ASC; + +CREATE VIEW index_attrs AS +WITH indexoid AS ( + SELECT c.oid, + n.nspname, + c.relname + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE pg_catalog.pg_table_is_visible(c.oid) + ORDER BY 2, 3 +) +SELECT + indexoid.nspname AS "nspname", + indexoid.relname AS "relname", + a.attrelid AS "relid", + a.attname AS "Column", + pg_catalog.format_type(a.atttypid, a.atttypmod) AS "Type", + pg_catalog.pg_get_indexdef(a.attrelid, a.attnum, TRUE) AS "Definition" +FROM pg_catalog.pg_attribute a +LEFT JOIN indexoid ON (a.attrelid = indexoid.oid) +WHERE true + AND a.attnum > 0 + AND NOT a.attisdropped +ORDER BY a.attrelid, a.attnum; + +$desc_views$ +); + run_command_on_master_and_workers +----------------------------------- + +(1 row) + diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index ce648077f..33651f00a 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -6,110 +6,6 @@ BEGIN EXECUTE p_sql; PERFORM run_command_on_workers(p_sql); END;$$; --- The following views are intended as alternatives to \d commands, whose --- output changed in PostgreSQL 10. In particular, they must be used any time --- a test wishes to print out the structure of a relation, which previously --- was safely accomplished by a \d invocation. -SELECT run_command_on_master_and_workers( -$desc_views$ -CREATE VIEW table_fkey_cols AS -SELECT rc.constraint_name AS "name", - kcu.column_name AS "column_name", - uc_kcu.column_name AS "refd_column_name", - format('%I.%I', kcu.table_schema, kcu.table_name)::regclass::oid AS relid, - format('%I.%I', uc_kcu.table_schema, uc_kcu.table_name)::regclass::oid AS refd_relid, - rc.constraint_schema AS "schema" -FROM information_schema.referential_constraints rc, - information_schema.key_column_usage kcu, - information_schema.key_column_usage uc_kcu -WHERE rc.constraint_schema = kcu.constraint_schema AND - rc.constraint_name = kcu.constraint_name AND - rc.unique_constraint_schema = uc_kcu.constraint_schema AND - rc.unique_constraint_name = uc_kcu.constraint_name; - -CREATE VIEW table_fkeys AS -SELECT name AS "Constraint", - format('FOREIGN KEY (%s) REFERENCES %s(%s)', - string_agg(DISTINCT quote_ident(column_name), ', '), - string_agg(DISTINCT refd_relid::regclass::text, ', '), - string_agg(DISTINCT quote_ident(refd_column_name), ', ')) AS "Definition", - "relid" -FROM table_fkey_cols -GROUP BY (name, relid); - -CREATE VIEW table_attrs AS -SELECT c.column_name AS "name", - c.data_type AS "type", - CASE - WHEN character_maximum_length IS NOT NULL THEN - format('(%s)', character_maximum_length) - WHEN data_type = 'numeric' AND numeric_precision IS NOT NULL THEN - format('(%s,%s)', numeric_precision, numeric_scale) - ELSE '' - END AS "modifier", - c.column_default AS "default", - (NOT c.is_nullable::boolean) AS "notnull", - format('%I.%I', c.table_schema, c.table_name)::regclass::oid AS "relid" -FROM information_schema.columns AS c -ORDER BY ordinal_position; - -CREATE VIEW table_desc AS -SELECT "name" AS "Column", - "type" || "modifier" AS "Type", - rtrim(( - CASE "notnull" - WHEN true THEN 'not null ' - ELSE '' - END - ) || ( - CASE WHEN "default" IS NULL THEN '' - ELSE 'default ' || "default" - END - )) AS "Modifiers", - "relid" -FROM table_attrs; - -CREATE VIEW table_checks AS -SELECT cc.constraint_name AS "Constraint", - ('CHECK ' || regexp_replace(check_clause, '^\((.*)\)$', '\1')) AS "Definition", - format('%I.%I', ccu.table_schema, ccu.table_name)::regclass::oid AS relid -FROM information_schema.check_constraints cc, - information_schema.constraint_column_usage ccu -WHERE cc.constraint_schema = ccu.constraint_schema AND - cc.constraint_name = ccu.constraint_name -ORDER BY cc.constraint_name ASC; - -CREATE VIEW index_attrs AS -WITH indexoid AS ( - SELECT c.oid, - n.nspname, - c.relname - FROM pg_catalog.pg_class c - LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - WHERE pg_catalog.pg_table_is_visible(c.oid) - ORDER BY 2, 3 -) -SELECT - indexoid.nspname AS "nspname", - indexoid.relname AS "relname", - a.attrelid AS "relid", - a.attname AS "Column", - pg_catalog.format_type(a.atttypid, a.atttypmod) AS "Type", - pg_catalog.pg_get_indexdef(a.attrelid, a.attnum, TRUE) AS "Definition" -FROM pg_catalog.pg_attribute a -LEFT JOIN indexoid ON (a.attrelid = indexoid.oid) -WHERE true - AND a.attnum > 0 - AND NOT a.attisdropped -ORDER BY a.attrelid, a.attnum; - -$desc_views$ -); - run_command_on_master_and_workers ------------------------------------ - -(1 row) - -- Create a function to make sure that queries returning the same result CREATE FUNCTION raise_failed_execution(query text) RETURNS void AS $$ BEGIN @@ -170,3 +66,42 @@ BEGIN RETURN true; END; $func$; +CREATE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) + RETURNS void + LANGUAGE C STRICT + AS 'citus'; +-- set sync intervals to less than 15s so wait_until_metadata_sync never times out +ALTER SYSTEM SET citus.metadata_sync_interval TO 3000; +ALTER SYSTEM SET citus.metadata_sync_retry_interval TO 500; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +-- Verifies pg_dist_node and pg_dist_palcement in the given worker matches the ones in coordinator +CREATE FUNCTION verify_metadata(hostname TEXT, port INTEGER, master_port INTEGER DEFAULT 57636) + RETURNS BOOLEAN + LANGUAGE sql + AS $$ +SELECT wait_until_metadata_sync(); +WITH dist_node_summary AS ( + SELECT 'SELECT jsonb_agg(ROW(nodeid, groupid, nodename, nodeport, isactive) ORDER BY nodeid) FROM pg_dist_node' as query +), dist_node_check AS ( + SELECT count(distinct result) = 1 AS matches + FROM dist_node_summary CROSS JOIN LATERAL + master_run_on_worker(ARRAY[hostname, 'localhost'], ARRAY[port, master_port], + ARRAY[dist_node_summary.query, dist_node_summary.query], + false) +), dist_placement_summary AS ( + SELECT 'SELECT jsonb_agg(pg_dist_placement ORDER BY shardid) FROM pg_dist_placement)' AS query +), dist_placement_check AS ( + SELECT count(distinct result) = 1 AS matches + FROM dist_placement_summary CROSS JOIN LATERAL + master_run_on_worker(ARRAY[hostname, 'localhost'], ARRAY[port, master_port], + ARRAY[dist_placement_summary.query, dist_placement_summary.query], + false) +) +SELECT dist_node_check.matches AND dist_placement_check.matches +FROM dist_node_check CROSS JOIN dist_placement_check +$$; diff --git a/src/test/regress/expected/replicate_reference_tables_to_coordinator.out b/src/test/regress/expected/replicate_reference_tables_to_coordinator.out index dca81da47..08b9323d1 100644 --- a/src/test/regress/expected/replicate_reference_tables_to_coordinator.out +++ b/src/test/regress/expected/replicate_reference_tables_to_coordinator.out @@ -113,11 +113,70 @@ SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers ORDER BY 1 20 | 20 (1 row) +-- test non equijoin +SELECT lt.a, sq.a, sq.b +FROM local_table lt +JOIN squares sq ON sq.a > lt.a and sq.b > 90 +ORDER BY 1,2,3; + a | a | b +---+----+----- + 2 | 10 | 100 + 4 | 10 | 100 + 7 | 10 | 100 +(3 rows) + -- error if in transaction block BEGIN; SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers ORDER BY 1; ERROR: cannot join local tables and reference tables in a transaction block ROLLBACK; +-- error if in a DO block +DO $$ +BEGIN + PERFORM local_table.a, numbers.a FROM local_table NATURAL JOIN numbers; +END; +$$; +ERROR: cannot join local tables and reference tables in a transaction block +CONTEXT: SQL statement "SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers" +PL/pgSQL function inline_code_block line 3 at PERFORM +-- test plpgsql function +CREATE FUNCTION test_reference_local_join_plpgsql_func() +RETURNS void AS $$ +BEGIN + INSERT INTO local_table VALUES (21); + INSERT INTO numbers VALUES (4); + PERFORM local_table.a, numbers.a FROM local_table NATURAL JOIN numbers ORDER BY 1; + RAISE EXCEPTION ''; + PERFORM local_table.a, numbers.a FROM local_table NATURAL JOIN numbers ORDER BY 1; +END; +$$ LANGUAGE plpgsql; +SELECT test_reference_local_join_plpgsql_func(); +LOG: executing the command locally: INSERT INTO replicate_ref_to_coordinator.numbers_8000001 (a) VALUES (4) +CONTEXT: SQL statement "INSERT INTO numbers VALUES (4)" +PL/pgSQL function test_reference_local_join_plpgsql_func() line 4 at SQL statement +ERROR: cannot join local tables and reference tables in a transaction block +CONTEXT: SQL statement "SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers ORDER BY 1" +PL/pgSQL function test_reference_local_join_plpgsql_func() line 5 at PERFORM +SELECT sum(a) FROM local_table; + sum +----- + 33 +(1 row) + +SELECT sum(a) FROM numbers; +LOG: executing the command locally: SELECT sum(a) AS sum FROM replicate_ref_to_coordinator.numbers_8000001 numbers + sum +----- + 41 +(1 row) + +-- error if in procedure's subtransaction +CREATE PROCEDURE test_reference_local_join_proc() AS $$ +SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers ORDER BY 1; +$$ LANGUAGE sql; +CALL test_reference_local_join_proc(); +ERROR: cannot join local tables and reference tables in a transaction block +CONTEXT: SQL function "test_reference_local_join_proc" statement 1 -- error if in a transaction block even if reference table is not in search path CREATE SCHEMA s1; CREATE TABLE s1.ref(a int); @@ -136,14 +195,17 @@ NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table s1.ref drop cascades to table s1.ref_8000002 -- shouldn't plan locally if modifications happen in CTEs, ... -WITH ins AS (INSERT INTO numbers VALUES (1) RETURNING *) SELECT * FROM numbers, local_table; +WITH ins AS (INSERT INTO numbers VALUES (1) RETURNING *) +SELECT * FROM numbers, local_table; ERROR: relation local_table is not distributed -WITH t AS (SELECT *, random() x FROM numbers FOR UPDATE) SELECT * FROM numbers, local_table - WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); +WITH t AS (SELECT *, random() x FROM numbers FOR UPDATE) +SELECT * FROM numbers, local_table +WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); ERROR: relation local_table is not distributed -- but this should be fine -WITH t AS (SELECT *, random() x FROM numbers) SELECT * FROM numbers, local_table - WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); +WITH t AS (SELECT *, random() x FROM numbers) +SELECT * FROM numbers, local_table +WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); a | a ---+--- (0 rows) @@ -156,21 +218,31 @@ SELECT create_distributed_table('dist', 'a'); (1 row) -WITH t AS (SELECT *, random() x FROM dist) SELECT * FROM numbers, local_table - WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); +INSERT INTO dist VALUES (20),(30); +WITH t AS (SELECT *, random() x FROM dist) +SELECT * FROM numbers, local_table +WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); ERROR: relation local_table is not distributed +-- test CTE being reference/local join for distributed query +WITH t as (SELECT n.a, random() x FROM numbers n NATURAL JOIN local_table l) +SELECT a FROM t NATURAL JOIN dist; + a +---- + 20 +(1 row) + -- error if FOR UPDATE/FOR SHARE - SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers FOR SHARE; +SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers FOR SHARE; ERROR: could not run distributed query with FOR UPDATE/SHARE commands HINT: Consider using an equality filter on the distributed table's partition column. - SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers FOR UPDATE; +SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers FOR UPDATE; ERROR: could not run distributed query with FOR UPDATE/SHARE commands HINT: Consider using an equality filter on the distributed table's partition column. -- clean-up SET client_min_messages TO ERROR; DROP SCHEMA replicate_ref_to_coordinator CASCADE; -- Make sure the shard was dropped - SELECT 'numbers_8000001'::regclass::oid; +SELECT 'numbers_8000001'::regclass::oid; ERROR: relation "numbers_8000001" does not exist LINE 1: SELECT 'numbers_8000001'::regclass::oid; ^ diff --git a/src/test/regress/expected/upgrade_distributed_function_before.out b/src/test/regress/expected/upgrade_distributed_function_before.out index 7273196ed..e82f86ba7 100644 --- a/src/test/regress/expected/upgrade_distributed_function_before.out +++ b/src/test/regress/expected/upgrade_distributed_function_before.out @@ -2,19 +2,6 @@ CREATE SCHEMA upgrade_distributed_function_before; SET search_path TO upgrade_distributed_function_before, public; SET citus.replication_model TO streaming; SET citus.shard_replication_factor TO 1; --- set sync intervals to less than 15s so wait_until_metadata_sync never times out -ALTER SYSTEM SET citus.metadata_sync_interval TO 3000; -ALTER SYSTEM SET citus.metadata_sync_retry_interval TO 500; -SELECT pg_reload_conf(); - pg_reload_conf ----------------- - t -(1 row) - -CREATE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) - RETURNS void - LANGUAGE C STRICT - AS 'citus'; CREATE TABLE t1 (a int PRIMARY KEY, b int); SELECT create_distributed_table('t1','a'); create_distributed_table diff --git a/src/test/regress/failure_base_schedule b/src/test/regress/failure_base_schedule index 6ecbe588b..4c2f91355 100644 --- a/src/test/regress/failure_base_schedule +++ b/src/test/regress/failure_base_schedule @@ -4,3 +4,4 @@ test: failure_test_helpers # this should only be run by pg_regress_multi, you don't need it test: failure_setup test: multi_test_helpers +test: multi_test_catalog_views diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule index f56cce9b2..8212e1bdd 100644 --- a/src/test/regress/failure_schedule +++ b/src/test/regress/failure_schedule @@ -4,6 +4,7 @@ test: failure_test_helpers # this should only be run by pg_regress_multi, you don't need it test: failure_setup test: multi_test_helpers +test: multi_test_catalog_views test: failure_ddl test: failure_truncate test: failure_create_index_concurrently diff --git a/src/test/regress/minimal_schedule b/src/test/regress/minimal_schedule index eeb399992..26372118a 100644 --- a/src/test/regress/minimal_schedule +++ b/src/test/regress/minimal_schedule @@ -1,2 +1,3 @@ test: multi_cluster_management test: multi_test_helpers +test: multi_test_catalog_views diff --git a/src/test/regress/multi_mx_schedule b/src/test/regress/multi_mx_schedule index 80ccc76f7..726050b64 100644 --- a/src/test/regress/multi_mx_schedule +++ b/src/test/regress/multi_mx_schedule @@ -14,9 +14,10 @@ # Tests around schema changes, these are run first, so there's no preexisting objects. # --- test: multi_extension +test: multi_test_helpers test: multi_mx_node_metadata test: multi_cluster_management -test: multi_test_helpers +test: multi_test_catalog_views # the following test has to be run sequentially test: multi_mx_create_table diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 25a257fe8..0d48602b2 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -25,6 +25,7 @@ test: multi_cluster_management test: alter_role_propagation test: propagate_extension_commands test: multi_test_helpers +test: multi_test_catalog_views test: multi_table_ddl test: multi_name_lengths test: multi_name_resolution @@ -267,6 +268,7 @@ test: multi_replicate_reference_table test: multi_reference_table test: foreign_key_to_reference_table test: replicate_reference_tables_to_coordinator + test: remove_coordinator # ---------- diff --git a/src/test/regress/multi_task_tracker_extra_schedule b/src/test/regress/multi_task_tracker_extra_schedule index 22e45e050..0e0bfd6d4 100644 --- a/src/test/regress/multi_task_tracker_extra_schedule +++ b/src/test/regress/multi_task_tracker_extra_schedule @@ -17,6 +17,7 @@ test: multi_extension test: multi_cluster_management test: multi_table_ddl test: multi_test_helpers +test: multi_test_catalog_views # ---------- # The following distributed tests depend on creating a partitioned table and diff --git a/src/test/regress/mx_base_schedule b/src/test/regress/mx_base_schedule index 6452ea5dc..cb2ba5f68 100644 --- a/src/test/regress/mx_base_schedule +++ b/src/test/regress/mx_base_schedule @@ -3,6 +3,7 @@ # ---------- test: multi_cluster_management test: multi_test_helpers +test: multi_test_catalog_views # the following test has to be run sequentially test: multi_mx_create_table diff --git a/src/test/regress/sql/distributed_functions.sql b/src/test/regress/sql/distributed_functions.sql index 1672b04ac..d8486a3a8 100644 --- a/src/test/regress/sql/distributed_functions.sql +++ b/src/test/regress/sql/distributed_functions.sql @@ -9,16 +9,6 @@ CREATE SCHEMA function_tests2 AUTHORIZATION functionuser; SET search_path TO function_tests; SET citus.shard_count TO 4; --- set sync intervals to less than 15s so wait_until_metadata_sync never times out -ALTER SYSTEM SET citus.metadata_sync_interval TO 3000; -ALTER SYSTEM SET citus.metadata_sync_retry_interval TO 500; -SELECT pg_reload_conf(); - -CREATE OR REPLACE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) - RETURNS void - LANGUAGE C STRICT - AS 'citus'; - -- Create and distribute a simple function CREATE FUNCTION add(integer, integer) RETURNS integer AS 'select $1 + $2;' @@ -361,7 +351,7 @@ SET citus.replication_model TO "statement"; SELECT create_distributed_table('replicated_table_func_test', 'a'); SELECT create_distributed_function('add_with_param_names(int, int)', '$1', colocate_with:='replicated_table_func_test'); -SELECT wait_until_metadata_sync(); +SELECT public.wait_until_metadata_sync(); -- a function can be colocated with a different distribution argument type -- as long as there is a coercion path @@ -429,7 +419,7 @@ SET citus.shard_count TO 55; SELECT create_distributed_function('add_with_param_names(int, int)', 'val1'); -- sync metadata to workers for consistent results when clearing objects -SELECT wait_until_metadata_sync(); +SELECT public.wait_until_metadata_sync(); SET client_min_messages TO error; -- suppress cascading objects dropping diff --git a/src/test/regress/sql/multi_multiuser.sql b/src/test/regress/sql/multi_multiuser.sql index 010b0bda0..76ba3ee50 100644 --- a/src/test/regress/sql/multi_multiuser.sql +++ b/src/test/regress/sql/multi_multiuser.sql @@ -8,15 +8,6 @@ SET citus.next_shard_id TO 1420000; SET citus.shard_replication_factor TO 1; -ALTER SYSTEM SET citus.metadata_sync_interval TO 3000; -ALTER SYSTEM SET citus.metadata_sync_retry_interval TO 500; -SELECT pg_reload_conf(); -CREATE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) - RETURNS void - LANGUAGE C STRICT - AS 'citus'; - - CREATE TABLE test (id integer, val integer); SELECT create_distributed_table('test', 'id'); diff --git a/src/test/regress/sql/multi_mx_add_coordinator.sql b/src/test/regress/sql/multi_mx_add_coordinator.sql index 3ac026ee6..b5854feb6 100644 --- a/src/test/regress/sql/multi_mx_add_coordinator.sql +++ b/src/test/regress/sql/multi_mx_add_coordinator.sql @@ -7,6 +7,11 @@ SET citus.next_placement_id TO 7000000; SET citus.replication_model TO streaming; SET client_min_messages TO WARNING; +CREATE USER reprefuser WITH LOGIN; +SELECT run_command_on_workers('CREATE USER reprefuser WITH LOGIN'); +SET citus.enable_alter_role_propagation TO ON; +ALTER ROLE reprefuser WITH CREATEDB; + SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0); -- test that coordinator pg_dist_node entry is synced to the workers @@ -18,6 +23,18 @@ SELECT verify_metadata('localhost', :worker_1_port), CREATE TABLE ref(a int); SELECT create_reference_table('ref'); +-- alter role from mx worker isn't propagated +\c - - - :worker_1_port +SET citus.enable_alter_role_propagation TO ON; +ALTER ROLE reprefuser WITH CREATEROLE; +select rolcreatedb, rolcreaterole from pg_roles where rolname = 'reprefuser'; +\c - - - :worker_2_port +select rolcreatedb, rolcreaterole from pg_roles where rolname = 'reprefuser'; +\c - - - :master_port +SET search_path TO mx_add_coordinator,public; +SET client_min_messages TO WARNING; +select rolcreatedb, rolcreaterole from pg_roles where rolname = 'reprefuser'; + SET citus.log_local_commands TO ON; SET client_min_messages TO DEBUG; @@ -45,6 +62,13 @@ INSERT INTO ref VALUES (1), (2), (3); UPDATE ref SET a = a + 1; DELETE FROM ref WHERE a > 3; +-- Test we don't allow reference/local joins on mx workers +CREATE TABLE local_table (a int); +INSERT INTO local_table VALUES (2), (4); + +SELECT r.a FROM ref r JOIN local_table lt on r.a = lt.a; + + \c - - - :master_port SET search_path TO mx_add_coordinator,public; SELECT * FROM ref ORDER BY a; diff --git a/src/test/regress/sql/multi_mx_ddl.sql b/src/test/regress/sql/multi_mx_ddl.sql index 1577420f0..956238995 100644 --- a/src/test/regress/sql/multi_mx_ddl.sql +++ b/src/test/regress/sql/multi_mx_ddl.sql @@ -1,5 +1,4 @@ -- Tests related to distributed DDL commands on mx cluster - SELECT * FROM mx_ddl_table ORDER BY key; -- CREATE INDEX @@ -18,7 +17,6 @@ UPDATE mx_ddl_table SET version=0.1 WHERE version IS NULL; -- SET NOT NULL ALTER TABLE mx_ddl_table ALTER COLUMN version SET NOT NULL; - -- See that the changes are applied on coordinator, worker tables and shards SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; SELECT "relname", "Column", "Type", "Definition" FROM index_attrs WHERE diff --git a/src/test/regress/sql/multi_mx_node_metadata.sql b/src/test/regress/sql/multi_mx_node_metadata.sql index 1ff77325c..e31a34038 100644 --- a/src/test/regress/sql/multi_mx_node_metadata.sql +++ b/src/test/regress/sql/multi_mx_node_metadata.sql @@ -12,43 +12,6 @@ SET citus.replication_model TO streaming; SET citus.shard_count TO 8; SET citus.shard_replication_factor TO 1; --- set sync intervals to less than 15s so wait_until_metadata_sync never times out -ALTER SYSTEM SET citus.metadata_sync_interval TO 3000; -ALTER SYSTEM SET citus.metadata_sync_retry_interval TO 500; -SELECT pg_reload_conf(); - -CREATE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) - RETURNS void - LANGUAGE C STRICT - AS 'citus'; - --- Verifies pg_dist_node and pg_dist_palcement in the given worker matches the ones in coordinator -CREATE FUNCTION verify_metadata(hostname TEXT, port INTEGER, master_port INTEGER DEFAULT 57636) - RETURNS BOOLEAN - LANGUAGE sql - AS $$ -SELECT wait_until_metadata_sync(); -WITH dist_node_summary AS ( - SELECT 'SELECT jsonb_agg(ROW(nodeid, groupid, nodename, nodeport, isactive) ORDER BY nodeid) FROM pg_dist_node' as query -), dist_node_check AS ( - SELECT count(distinct result) = 1 AS matches - FROM dist_node_summary CROSS JOIN LATERAL - master_run_on_worker(ARRAY[hostname, 'localhost'], ARRAY[port, master_port], - ARRAY[dist_node_summary.query, dist_node_summary.query], - false) -), dist_placement_summary AS ( - SELECT 'SELECT jsonb_agg(pg_dist_placement ORDER BY shardid) FROM pg_dist_placement)' AS query -), dist_placement_check AS ( - SELECT count(distinct result) = 1 AS matches - FROM dist_placement_summary CROSS JOIN LATERAL - master_run_on_worker(ARRAY[hostname, 'localhost'], ARRAY[port, master_port], - ARRAY[dist_placement_summary.query, dist_placement_summary.query], - false) -) -SELECT dist_node_check.matches AND dist_placement_check.matches -FROM dist_node_check CROSS JOIN dist_placement_check -$$; - -- Simulates a readonly node by setting default_transaction_read_only. CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN) RETURNS TEXT diff --git a/src/test/regress/sql/multi_test_catalog_views.sql b/src/test/regress/sql/multi_test_catalog_views.sql new file mode 100644 index 000000000..f2c5a50cf --- /dev/null +++ b/src/test/regress/sql/multi_test_catalog_views.sql @@ -0,0 +1,100 @@ +-- The following views are intended as alternatives to \d commands, whose +-- output changed in PostgreSQL 10. In particular, they must be used any time +-- a test wishes to print out the structure of a relation, which previously +-- was safely accomplished by a \d invocation. +SELECT run_command_on_master_and_workers( +$desc_views$ +CREATE VIEW table_fkey_cols AS +SELECT rc.constraint_name AS "name", + kcu.column_name AS "column_name", + uc_kcu.column_name AS "refd_column_name", + format('%I.%I', kcu.table_schema, kcu.table_name)::regclass::oid AS relid, + format('%I.%I', uc_kcu.table_schema, uc_kcu.table_name)::regclass::oid AS refd_relid, + rc.constraint_schema AS "schema" +FROM information_schema.referential_constraints rc, + information_schema.key_column_usage kcu, + information_schema.key_column_usage uc_kcu +WHERE rc.constraint_schema = kcu.constraint_schema AND + rc.constraint_name = kcu.constraint_name AND + rc.unique_constraint_schema = uc_kcu.constraint_schema AND + rc.unique_constraint_name = uc_kcu.constraint_name; + +CREATE VIEW table_fkeys AS +SELECT name AS "Constraint", + format('FOREIGN KEY (%s) REFERENCES %s(%s)', + string_agg(DISTINCT quote_ident(column_name), ', '), + string_agg(DISTINCT refd_relid::regclass::text, ', '), + string_agg(DISTINCT quote_ident(refd_column_name), ', ')) AS "Definition", + "relid" +FROM table_fkey_cols +GROUP BY (name, relid); + +CREATE VIEW table_attrs AS +SELECT c.column_name AS "name", + c.data_type AS "type", + CASE + WHEN character_maximum_length IS NOT NULL THEN + format('(%s)', character_maximum_length) + WHEN data_type = 'numeric' AND numeric_precision IS NOT NULL THEN + format('(%s,%s)', numeric_precision, numeric_scale) + ELSE '' + END AS "modifier", + c.column_default AS "default", + (NOT c.is_nullable::boolean) AS "notnull", + format('%I.%I', c.table_schema, c.table_name)::regclass::oid AS "relid" +FROM information_schema.columns AS c +ORDER BY ordinal_position; + +CREATE VIEW table_desc AS +SELECT "name" AS "Column", + "type" || "modifier" AS "Type", + rtrim(( + CASE "notnull" + WHEN true THEN 'not null ' + ELSE '' + END + ) || ( + CASE WHEN "default" IS NULL THEN '' + ELSE 'default ' || "default" + END + )) AS "Modifiers", + "relid" +FROM table_attrs; + +CREATE VIEW table_checks AS +SELECT cc.constraint_name AS "Constraint", + ('CHECK ' || regexp_replace(check_clause, '^\((.*)\)$', '\1')) AS "Definition", + format('%I.%I', ccu.table_schema, ccu.table_name)::regclass::oid AS relid +FROM information_schema.check_constraints cc, + information_schema.constraint_column_usage ccu +WHERE cc.constraint_schema = ccu.constraint_schema AND + cc.constraint_name = ccu.constraint_name +ORDER BY cc.constraint_name ASC; + +CREATE VIEW index_attrs AS +WITH indexoid AS ( + SELECT c.oid, + n.nspname, + c.relname + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE pg_catalog.pg_table_is_visible(c.oid) + ORDER BY 2, 3 +) +SELECT + indexoid.nspname AS "nspname", + indexoid.relname AS "relname", + a.attrelid AS "relid", + a.attname AS "Column", + pg_catalog.format_type(a.atttypid, a.atttypmod) AS "Type", + pg_catalog.pg_get_indexdef(a.attrelid, a.attnum, TRUE) AS "Definition" +FROM pg_catalog.pg_attribute a +LEFT JOIN indexoid ON (a.attrelid = indexoid.oid) +WHERE true + AND a.attnum > 0 + AND NOT a.attisdropped +ORDER BY a.attrelid, a.attnum; + +$desc_views$ +); + diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index 28e464f7a..160bcb7f0 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -8,106 +8,6 @@ BEGIN PERFORM run_command_on_workers(p_sql); END;$$; --- The following views are intended as alternatives to \d commands, whose --- output changed in PostgreSQL 10. In particular, they must be used any time --- a test wishes to print out the structure of a relation, which previously --- was safely accomplished by a \d invocation. -SELECT run_command_on_master_and_workers( -$desc_views$ -CREATE VIEW table_fkey_cols AS -SELECT rc.constraint_name AS "name", - kcu.column_name AS "column_name", - uc_kcu.column_name AS "refd_column_name", - format('%I.%I', kcu.table_schema, kcu.table_name)::regclass::oid AS relid, - format('%I.%I', uc_kcu.table_schema, uc_kcu.table_name)::regclass::oid AS refd_relid, - rc.constraint_schema AS "schema" -FROM information_schema.referential_constraints rc, - information_schema.key_column_usage kcu, - information_schema.key_column_usage uc_kcu -WHERE rc.constraint_schema = kcu.constraint_schema AND - rc.constraint_name = kcu.constraint_name AND - rc.unique_constraint_schema = uc_kcu.constraint_schema AND - rc.unique_constraint_name = uc_kcu.constraint_name; - -CREATE VIEW table_fkeys AS -SELECT name AS "Constraint", - format('FOREIGN KEY (%s) REFERENCES %s(%s)', - string_agg(DISTINCT quote_ident(column_name), ', '), - string_agg(DISTINCT refd_relid::regclass::text, ', '), - string_agg(DISTINCT quote_ident(refd_column_name), ', ')) AS "Definition", - "relid" -FROM table_fkey_cols -GROUP BY (name, relid); - -CREATE VIEW table_attrs AS -SELECT c.column_name AS "name", - c.data_type AS "type", - CASE - WHEN character_maximum_length IS NOT NULL THEN - format('(%s)', character_maximum_length) - WHEN data_type = 'numeric' AND numeric_precision IS NOT NULL THEN - format('(%s,%s)', numeric_precision, numeric_scale) - ELSE '' - END AS "modifier", - c.column_default AS "default", - (NOT c.is_nullable::boolean) AS "notnull", - format('%I.%I', c.table_schema, c.table_name)::regclass::oid AS "relid" -FROM information_schema.columns AS c -ORDER BY ordinal_position; - -CREATE VIEW table_desc AS -SELECT "name" AS "Column", - "type" || "modifier" AS "Type", - rtrim(( - CASE "notnull" - WHEN true THEN 'not null ' - ELSE '' - END - ) || ( - CASE WHEN "default" IS NULL THEN '' - ELSE 'default ' || "default" - END - )) AS "Modifiers", - "relid" -FROM table_attrs; - -CREATE VIEW table_checks AS -SELECT cc.constraint_name AS "Constraint", - ('CHECK ' || regexp_replace(check_clause, '^\((.*)\)$', '\1')) AS "Definition", - format('%I.%I', ccu.table_schema, ccu.table_name)::regclass::oid AS relid -FROM information_schema.check_constraints cc, - information_schema.constraint_column_usage ccu -WHERE cc.constraint_schema = ccu.constraint_schema AND - cc.constraint_name = ccu.constraint_name -ORDER BY cc.constraint_name ASC; - -CREATE VIEW index_attrs AS -WITH indexoid AS ( - SELECT c.oid, - n.nspname, - c.relname - FROM pg_catalog.pg_class c - LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - WHERE pg_catalog.pg_table_is_visible(c.oid) - ORDER BY 2, 3 -) -SELECT - indexoid.nspname AS "nspname", - indexoid.relname AS "relname", - a.attrelid AS "relid", - a.attname AS "Column", - pg_catalog.format_type(a.atttypid, a.atttypmod) AS "Type", - pg_catalog.pg_get_indexdef(a.attrelid, a.attnum, TRUE) AS "Definition" -FROM pg_catalog.pg_attribute a -LEFT JOIN indexoid ON (a.attrelid = indexoid.oid) -WHERE true - AND a.attnum > 0 - AND NOT a.attisdropped -ORDER BY a.attrelid, a.attnum; - -$desc_views$ -); - -- Create a function to make sure that queries returning the same result CREATE FUNCTION raise_failed_execution(query text) RETURNS void AS $$ BEGIN @@ -172,3 +72,40 @@ BEGIN RETURN true; END; $func$; + +CREATE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) + RETURNS void + LANGUAGE C STRICT + AS 'citus'; + +-- set sync intervals to less than 15s so wait_until_metadata_sync never times out +ALTER SYSTEM SET citus.metadata_sync_interval TO 3000; +ALTER SYSTEM SET citus.metadata_sync_retry_interval TO 500; +SELECT pg_reload_conf(); + +-- Verifies pg_dist_node and pg_dist_palcement in the given worker matches the ones in coordinator +CREATE FUNCTION verify_metadata(hostname TEXT, port INTEGER, master_port INTEGER DEFAULT 57636) + RETURNS BOOLEAN + LANGUAGE sql + AS $$ +SELECT wait_until_metadata_sync(); +WITH dist_node_summary AS ( + SELECT 'SELECT jsonb_agg(ROW(nodeid, groupid, nodename, nodeport, isactive) ORDER BY nodeid) FROM pg_dist_node' as query +), dist_node_check AS ( + SELECT count(distinct result) = 1 AS matches + FROM dist_node_summary CROSS JOIN LATERAL + master_run_on_worker(ARRAY[hostname, 'localhost'], ARRAY[port, master_port], + ARRAY[dist_node_summary.query, dist_node_summary.query], + false) +), dist_placement_summary AS ( + SELECT 'SELECT jsonb_agg(pg_dist_placement ORDER BY shardid) FROM pg_dist_placement)' AS query +), dist_placement_check AS ( + SELECT count(distinct result) = 1 AS matches + FROM dist_placement_summary CROSS JOIN LATERAL + master_run_on_worker(ARRAY[hostname, 'localhost'], ARRAY[port, master_port], + ARRAY[dist_placement_summary.query, dist_placement_summary.query], + false) +) +SELECT dist_node_check.matches AND dist_placement_check.matches +FROM dist_node_check CROSS JOIN dist_placement_check +$$; diff --git a/src/test/regress/sql/replicate_reference_tables_to_coordinator.sql b/src/test/regress/sql/replicate_reference_tables_to_coordinator.sql index 5a2af41cf..9804cd9aa 100644 --- a/src/test/regress/sql/replicate_reference_tables_to_coordinator.sql +++ b/src/test/regress/sql/replicate_reference_tables_to_coordinator.sql @@ -48,11 +48,45 @@ INSERT INTO local_table VALUES (2), (4), (7), (20); EXPLAIN SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers; SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers ORDER BY 1; +-- test non equijoin +SELECT lt.a, sq.a, sq.b +FROM local_table lt +JOIN squares sq ON sq.a > lt.a and sq.b > 90 +ORDER BY 1,2,3; + -- error if in transaction block BEGIN; SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers ORDER BY 1; ROLLBACK; +-- error if in a DO block +DO $$ +BEGIN + PERFORM local_table.a, numbers.a FROM local_table NATURAL JOIN numbers; +END; +$$; + +-- test plpgsql function +CREATE FUNCTION test_reference_local_join_plpgsql_func() +RETURNS void AS $$ +BEGIN + INSERT INTO local_table VALUES (21); + INSERT INTO numbers VALUES (4); + PERFORM local_table.a, numbers.a FROM local_table NATURAL JOIN numbers ORDER BY 1; + RAISE EXCEPTION ''; + PERFORM local_table.a, numbers.a FROM local_table NATURAL JOIN numbers ORDER BY 1; +END; +$$ LANGUAGE plpgsql; +SELECT test_reference_local_join_plpgsql_func(); +SELECT sum(a) FROM local_table; +SELECT sum(a) FROM numbers; + +-- error if in procedure's subtransaction +CREATE PROCEDURE test_reference_local_join_proc() AS $$ +SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers ORDER BY 1; +$$ LANGUAGE sql; +CALL test_reference_local_join_proc(); + -- error if in a transaction block even if reference table is not in search path CREATE SCHEMA s1; CREATE TABLE s1.ref(a int); @@ -65,30 +99,41 @@ ROLLBACK; DROP SCHEMA s1 CASCADE; -- shouldn't plan locally if modifications happen in CTEs, ... -WITH ins AS (INSERT INTO numbers VALUES (1) RETURNING *) SELECT * FROM numbers, local_table; -WITH t AS (SELECT *, random() x FROM numbers FOR UPDATE) SELECT * FROM numbers, local_table - WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); +WITH ins AS (INSERT INTO numbers VALUES (1) RETURNING *) +SELECT * FROM numbers, local_table; + +WITH t AS (SELECT *, random() x FROM numbers FOR UPDATE) +SELECT * FROM numbers, local_table +WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); -- but this should be fine -WITH t AS (SELECT *, random() x FROM numbers) SELECT * FROM numbers, local_table - WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); +WITH t AS (SELECT *, random() x FROM numbers) +SELECT * FROM numbers, local_table +WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); -- shouldn't plan locally even if distributed table is in CTE or subquery CREATE TABLE dist(a int); SELECT create_distributed_table('dist', 'a'); -WITH t AS (SELECT *, random() x FROM dist) SELECT * FROM numbers, local_table - WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); +INSERT INTO dist VALUES (20),(30); + +WITH t AS (SELECT *, random() x FROM dist) +SELECT * FROM numbers, local_table +WHERE EXISTS (SELECT * FROM t WHERE t.x = numbers.a); + +-- test CTE being reference/local join for distributed query +WITH t as (SELECT n.a, random() x FROM numbers n NATURAL JOIN local_table l) +SELECT a FROM t NATURAL JOIN dist; -- error if FOR UPDATE/FOR SHARE - SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers FOR SHARE; - SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers FOR UPDATE; +SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers FOR SHARE; +SELECT local_table.a, numbers.a FROM local_table NATURAL JOIN numbers FOR UPDATE; -- clean-up SET client_min_messages TO ERROR; DROP SCHEMA replicate_ref_to_coordinator CASCADE; -- Make sure the shard was dropped - SELECT 'numbers_8000001'::regclass::oid; +SELECT 'numbers_8000001'::regclass::oid; SET search_path TO DEFAULT; RESET client_min_messages; diff --git a/src/test/regress/sql/upgrade_distributed_function_before.sql b/src/test/regress/sql/upgrade_distributed_function_before.sql index d65b91d68..24c2bcc49 100644 --- a/src/test/regress/sql/upgrade_distributed_function_before.sql +++ b/src/test/regress/sql/upgrade_distributed_function_before.sql @@ -3,17 +3,6 @@ SET search_path TO upgrade_distributed_function_before, public; SET citus.replication_model TO streaming; SET citus.shard_replication_factor TO 1; --- set sync intervals to less than 15s so wait_until_metadata_sync never times out -ALTER SYSTEM SET citus.metadata_sync_interval TO 3000; -ALTER SYSTEM SET citus.metadata_sync_retry_interval TO 500; -SELECT pg_reload_conf(); - -CREATE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) - RETURNS void - LANGUAGE C STRICT - AS 'citus'; - - CREATE TABLE t1 (a int PRIMARY KEY, b int); SELECT create_distributed_table('t1','a'); INSERT INTO t1 VALUES (11), (12);