From cf98b9d6d5762865239d24f7f94ec9d82b3097b1 Mon Sep 17 00:00:00 2001 From: SaitTalhaNisanci Date: Thu, 14 May 2020 10:51:24 +0300 Subject: [PATCH] not wait forever for metadata sync in tests (#3760) We shouldn't wait forever for metada sync in tests, otherwise when a test gets stuck, we don't know which line causes the problem. --- .../expected/distributed_functions.out | 4 ++-- .../expected/distributed_procedure.out | 2 +- src/test/regress/expected/multi_multiuser.out | 2 +- .../expected/multi_mx_add_coordinator.out | 4 ++-- .../multi_mx_function_table_reference.out | 2 +- .../expected/multi_mx_node_metadata.out | 24 +++++++++---------- .../expected/multi_test_helpers_superuser.out | 2 +- .../regress/sql/distributed_functions.sql | 4 ++-- .../regress/sql/distributed_procedure.sql | 2 +- src/test/regress/sql/multi_multiuser.sql | 2 +- .../regress/sql/multi_mx_add_coordinator.sql | 4 ++-- .../sql/multi_mx_function_table_reference.sql | 2 +- .../regress/sql/multi_mx_node_metadata.sql | 24 +++++++++---------- .../sql/multi_test_helpers_superuser.sql | 2 +- 14 files changed, 40 insertions(+), 40 deletions(-) diff --git a/src/test/regress/expected/distributed_functions.out b/src/test/regress/expected/distributed_functions.out index 4d2f29c36..71973f2cb 100644 --- a/src/test/regress/expected/distributed_functions.out +++ b/src/test/regress/expected/distributed_functions.out @@ -660,7 +660,7 @@ SELECT create_distributed_function('eq_with_param_names(macaddr, macaddr)', '$1' ERROR: cannot colocate function "eq_with_param_names" and table "replicated_table_func_test" DETAIL: Citus currently only supports colocating function with distributed tables that are created using streaming replication model. HINT: When distributing tables make sure that citus.replication_model = 'streaming' -SELECT public.wait_until_metadata_sync(); +SELECT public.wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -788,7 +788,7 @@ SELECT create_distributed_function('eq_with_param_names(macaddr, macaddr)', 'val ERROR: cannot distribute the function "eq_with_param_names" since there is no table to colocate with HINT: Provide a distributed table via "colocate_with" option to create_distributed_function() -- sync metadata to workers for consistent results when clearing objects -SELECT public.wait_until_metadata_sync(); +SELECT public.wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- diff --git a/src/test/regress/expected/distributed_procedure.out b/src/test/regress/expected/distributed_procedure.out index d819c4294..c13dd07a7 100644 --- a/src/test/regress/expected/distributed_procedure.out +++ b/src/test/regress/expected/distributed_procedure.out @@ -49,7 +49,7 @@ SELECT create_distributed_function('raise_info(text)', '$1', colocate_with := 'c (1 row) -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_multiuser.out b/src/test/regress/expected/multi_multiuser.out index 2159ea2a7..0b84a601e 100644 --- a/src/test/regress/expected/multi_multiuser.out +++ b/src/test/regress/expected/multi_multiuser.out @@ -487,7 +487,7 @@ SELECT run_command_on_workers($$SELECT proowner::regrole FROM pg_proc WHERE pron (localhost,57638,t,usage_access) (2 rows) -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_mx_add_coordinator.out b/src/test/regress/expected/multi_mx_add_coordinator.out index 87426589e..bdf42af57 100644 --- a/src/test/regress/expected/multi_mx_add_coordinator.out +++ b/src/test/regress/expected/multi_mx_add_coordinator.out @@ -24,7 +24,7 @@ SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0); (1 row) -- test that coordinator pg_dist_node entry is synced to the workers -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -172,7 +172,7 @@ SELECT master_remove_node('localhost', :master_port); (1 row) -- test that coordinator pg_dist_node entry was removed from the workers -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_mx_function_table_reference.out b/src/test/regress/expected/multi_mx_function_table_reference.out index a433615cb..606294670 100644 --- a/src/test/regress/expected/multi_mx_function_table_reference.out +++ b/src/test/regress/expected/multi_mx_function_table_reference.out @@ -67,7 +67,7 @@ SELECT 1 FROM master_add_node('localhost', :worker_2_port); 1 (1 row) -SELECT public.wait_until_metadata_sync(); +SELECT public.wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_mx_node_metadata.out b/src/test/regress/expected/multi_mx_node_metadata.out index d5e5f8a9c..e1321e549 100644 --- a/src/test/regress/expected/multi_mx_node_metadata.out +++ b/src/test/regress/expected/multi_mx_node_metadata.out @@ -99,7 +99,7 @@ SELECT nodeid, nodename, nodeport, hasmetadata, metadatasynced FROM pg_dist_node END; -- wait until maintenance daemon does the next metadata sync, and then -- check if metadata is synced again -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -140,7 +140,7 @@ SELECT nodeid, nodename, nodeport, hasmetadata, metadatasynced FROM pg_dist_node END; -- maintenace daemon metadata sync should fail, because node is still unwriteable. -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -159,7 +159,7 @@ SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', :worker_1_port); 1 (1 row) -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -234,7 +234,7 @@ SELECT mark_node_readonly('localhost', :worker_2_port, FALSE); t (1 row) -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -262,7 +262,7 @@ SELECT count(*) FROM dist_table_2; (1 row) END; -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -275,7 +275,7 @@ SELECT mark_node_readonly('localhost', :worker_2_port, FALSE); t (1 row) -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -401,7 +401,7 @@ SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', 12345); PREPARE TRANSACTION 'tx01'; COMMIT PREPARED 'tx01'; -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -423,7 +423,7 @@ SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', :worker_1_port); PREPARE TRANSACTION 'tx01'; COMMIT PREPARED 'tx01'; -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -481,7 +481,7 @@ SELECT master_update_node(:nodeid_2, 'localhost', 1); (1 row) -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -517,7 +517,7 @@ SELECT master_update_node(:nodeid_2, 'localhost', :worker_2_port); (1 row) -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -545,7 +545,7 @@ SELECT master_update_node(:nodeid_1, 'localhost', 1); (1 row) -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- @@ -576,7 +576,7 @@ SELECT master_update_node(:nodeid_1, 'localhost', :worker_1_port); (1 row) -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); wait_until_metadata_sync --------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_test_helpers_superuser.out b/src/test/regress/expected/multi_test_helpers_superuser.out index f51f0e4cc..d80f310b9 100644 --- a/src/test/regress/expected/multi_test_helpers_superuser.out +++ b/src/test/regress/expected/multi_test_helpers_superuser.out @@ -16,7 +16,7 @@ CREATE OR REPLACE FUNCTION verify_metadata(hostname TEXT, port INTEGER, master_p RETURNS BOOLEAN LANGUAGE sql AS $$ -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); WITH dist_node_summary AS ( SELECT 'SELECT jsonb_agg(ROW(nodeid, groupid, nodename, nodeport, isactive) ORDER BY nodeid) FROM pg_dist_node' as query ), dist_node_check AS ( diff --git a/src/test/regress/sql/distributed_functions.sql b/src/test/regress/sql/distributed_functions.sql index f8d7ac08d..57ec04ed7 100644 --- a/src/test/regress/sql/distributed_functions.sql +++ b/src/test/regress/sql/distributed_functions.sql @@ -381,7 +381,7 @@ SET citus.replication_model TO "statement"; SELECT create_distributed_table('replicated_table_func_test', 'a'); SELECT create_distributed_function('eq_with_param_names(macaddr, macaddr)', '$1', colocate_with:='replicated_table_func_test'); -SELECT public.wait_until_metadata_sync(); +SELECT public.wait_until_metadata_sync(30000); -- a function can be colocated with a different distribution argument type -- as long as there is a coercion path @@ -449,7 +449,7 @@ SET citus.shard_count TO 55; SELECT create_distributed_function('eq_with_param_names(macaddr, macaddr)', 'val1'); -- sync metadata to workers for consistent results when clearing objects -SELECT public.wait_until_metadata_sync(); +SELECT public.wait_until_metadata_sync(30000); SET citus.shard_replication_factor TO 1; diff --git a/src/test/regress/sql/distributed_procedure.sql b/src/test/regress/sql/distributed_procedure.sql index 1bed22750..6f0cbbb34 100644 --- a/src/test/regress/sql/distributed_procedure.sql +++ b/src/test/regress/sql/distributed_procedure.sql @@ -34,7 +34,7 @@ SET citus.shard_replication_factor TO 1; SELECT create_distributed_table('colocation_table','id'); SELECT create_distributed_function('raise_info(text)', '$1', colocate_with := 'colocation_table'); -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); SELECT * FROM run_command_on_workers($$CALL procedure_tests.raise_info('hello');$$) ORDER BY 1,2; SELECT public.verify_function_is_same_on_workers('procedure_tests.raise_info(text)'); diff --git a/src/test/regress/sql/multi_multiuser.sql b/src/test/regress/sql/multi_multiuser.sql index 2d3ed8019..10e19f8a4 100644 --- a/src/test/regress/sql/multi_multiuser.sql +++ b/src/test/regress/sql/multi_multiuser.sql @@ -300,7 +300,7 @@ SELECT proowner::regrole FROM pg_proc WHERE proname = 'usage_access_func'; SELECT run_command_on_workers($$SELECT typowner::regrole FROM pg_type WHERE typname = 'usage_access_type'$$); SELECT run_command_on_workers($$SELECT proowner::regrole FROM pg_proc WHERE proname = 'usage_access_func'$$); -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); CREATE TABLE colocation_table(id text); SELECT create_distributed_table('colocation_table','id'); diff --git a/src/test/regress/sql/multi_mx_add_coordinator.sql b/src/test/regress/sql/multi_mx_add_coordinator.sql index ea628bd21..c9f839ede 100644 --- a/src/test/regress/sql/multi_mx_add_coordinator.sql +++ b/src/test/regress/sql/multi_mx_add_coordinator.sql @@ -16,7 +16,7 @@ ALTER ROLE reprefuser WITH CREATEDB; SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0); -- test that coordinator pg_dist_node entry is synced to the workers -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); SELECT verify_metadata('localhost', :worker_1_port), verify_metadata('localhost', :worker_2_port); @@ -82,7 +82,7 @@ SELECT count(*) FROM run_command_on_workers('SELECT recover_prepared_transaction SELECT master_remove_node('localhost', :master_port); -- test that coordinator pg_dist_node entry was removed from the workers -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); SELECT verify_metadata('localhost', :worker_1_port), verify_metadata('localhost', :worker_2_port); diff --git a/src/test/regress/sql/multi_mx_function_table_reference.sql b/src/test/regress/sql/multi_mx_function_table_reference.sql index ac249d879..ab6af2a9f 100644 --- a/src/test/regress/sql/multi_mx_function_table_reference.sql +++ b/src/test/regress/sql/multi_mx_function_table_reference.sql @@ -44,7 +44,7 @@ SELECT create_distributed_function('zoop(int)', '$1'); -- now add the worker back, this triggers function distribution which should not fail. SELECT 1 FROM master_add_node('localhost', :worker_2_port); -SELECT public.wait_until_metadata_sync(); +SELECT public.wait_until_metadata_sync(30000); -- clean up after testing diff --git a/src/test/regress/sql/multi_mx_node_metadata.sql b/src/test/regress/sql/multi_mx_node_metadata.sql index 44ada118e..2f6b22872 100644 --- a/src/test/regress/sql/multi_mx_node_metadata.sql +++ b/src/test/regress/sql/multi_mx_node_metadata.sql @@ -62,7 +62,7 @@ END; -- wait until maintenance daemon does the next metadata sync, and then -- check if metadata is synced again -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node; SELECT verify_metadata('localhost', :worker_1_port); @@ -76,12 +76,12 @@ SELECT nodeid, nodename, nodeport, hasmetadata, metadatasynced FROM pg_dist_node END; -- maintenace daemon metadata sync should fail, because node is still unwriteable. -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node; -- update it back to :worker_1_port, now metadata should be synced SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', :worker_1_port); -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node; -------------------------------------------------------------------------- @@ -110,7 +110,7 @@ SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node ORDER BY nodeid; -- Make the node writeable. SELECT mark_node_readonly('localhost', :worker_2_port, FALSE); -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); -- Mark the node readonly again, so the following master_update_node warns SELECT mark_node_readonly('localhost', :worker_2_port, TRUE); @@ -121,11 +121,11 @@ SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', :worker_1_port); SELECT count(*) FROM dist_table_2; END; -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); -- Make the node writeable. SELECT mark_node_readonly('localhost', :worker_2_port, FALSE); -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', :worker_1_port); SELECT verify_metadata('localhost', :worker_1_port), @@ -173,7 +173,7 @@ SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', 12345); PREPARE TRANSACTION 'tx01'; COMMIT PREPARED 'tx01'; -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node ORDER BY nodeid; BEGIN; @@ -181,7 +181,7 @@ SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', :worker_1_port); PREPARE TRANSACTION 'tx01'; COMMIT PREPARED 'tx01'; -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node ORDER BY nodeid; SELECT verify_metadata('localhost', :worker_1_port), @@ -203,7 +203,7 @@ SELECT verify_metadata('localhost', :worker_1_port); -- Test master_disable_node() when the node that is being disabled is actually down ------------------------------------------------------------------------------------ SELECT master_update_node(:nodeid_2, 'localhost', 1); -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); -- set metadatasynced so we try porpagating metadata changes UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodeid_2); @@ -218,7 +218,7 @@ SELECT 1 FROM master_disable_node('localhost', 1); SELECT verify_metadata('localhost', :worker_1_port); SELECT master_update_node(:nodeid_2, 'localhost', :worker_2_port); -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); SELECT 1 FROM master_activate_node('localhost', :worker_2_port); SELECT verify_metadata('localhost', :worker_1_port); @@ -229,7 +229,7 @@ SELECT verify_metadata('localhost', :worker_1_port); ------------------------------------------------------------------------------------ -- node 1 is down. SELECT master_update_node(:nodeid_1, 'localhost', 1); -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); -- set metadatasynced so we try porpagating metadata changes UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodeid_2); @@ -243,7 +243,7 @@ SELECT 1 FROM master_disable_node('localhost', :worker_2_port); -- bring up node 1 SELECT master_update_node(:nodeid_1, 'localhost', :worker_1_port); -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); SELECT 1 FROM master_activate_node('localhost', :worker_2_port); diff --git a/src/test/regress/sql/multi_test_helpers_superuser.sql b/src/test/regress/sql/multi_test_helpers_superuser.sql index d3ee92ca1..748e9ef89 100644 --- a/src/test/regress/sql/multi_test_helpers_superuser.sql +++ b/src/test/regress/sql/multi_test_helpers_superuser.sql @@ -13,7 +13,7 @@ CREATE OR REPLACE FUNCTION verify_metadata(hostname TEXT, port INTEGER, master_p RETURNS BOOLEAN LANGUAGE sql AS $$ -SELECT wait_until_metadata_sync(); +SELECT wait_until_metadata_sync(30000); WITH dist_node_summary AS ( SELECT 'SELECT jsonb_agg(ROW(nodeid, groupid, nodename, nodeport, isactive) ORDER BY nodeid) FROM pg_dist_node' as query ), dist_node_check AS (