From 56da3cf6aaba439d86b279d0ea98b2c5022f259a Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Wed, 19 Oct 2022 13:03:14 +0300 Subject: [PATCH] Increase node_connection_timeout to prevent flakiness in shard_rebalancer regression tests (#6445) In CI shard_rebalancer sometimes fails with this error: ```diff SET citus.node_connection_timeout to 60; BEGIN; SET LOCAL citus.shard_replication_factor TO 2; SET citus.log_remote_commands TO ON; SET SESSION citus.max_adaptive_executor_pool_size TO 5; SELECT replicate_table_shards('dist_table_test_2', max_shard_copies := 4, shard_transfer_mode:='block_writes'); +WARNING: could not establish connection after 60 ms ``` Source https://app.circleci.com/pipelines/github/citusdata/citus/28128/workflows/38eeacc4-4191-4366-87ed-9a628414965a/jobs/847458?invite=true#step-107-21 This PR avoids this issue by increasing ```citus.node_connection_timeout``` to 35s. --- src/test/regress/expected/shard_rebalancer.out | 10 +++++----- src/test/regress/sql/shard_rebalancer.sql | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/regress/expected/shard_rebalancer.out b/src/test/regress/expected/shard_rebalancer.out index 2efc70676..7ba0e1212 100644 --- a/src/test/regress/expected/shard_rebalancer.out +++ b/src/test/regress/expected/shard_rebalancer.out @@ -194,7 +194,7 @@ SELECT pg_sleep(.1); -- wait to make sure the config has changed before running (1 row) -- replicate reference table should ignore the coordinator -SET citus.node_connection_timeout to 60; +SET citus.node_connection_timeout to '35s'; BEGIN; SET LOCAL citus.shard_replication_factor TO 2; SET citus.log_remote_commands TO ON; @@ -213,7 +213,7 @@ NOTICE: issuing SET LOCAL citus.max_adaptive_executor_pool_size TO '5'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing SET LOCAL citus.next_shard_id TO '433105'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET LOCAL citus.node_connection_timeout TO '60'; +NOTICE: issuing SET LOCAL citus.node_connection_timeout TO '35000'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing SET LOCAL citus.shard_count TO '4'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx @@ -234,7 +234,7 @@ NOTICE: issuing SET LOCAL citus.max_adaptive_executor_pool_size TO '5'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing SET LOCAL citus.next_shard_id TO '433105'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET LOCAL citus.node_connection_timeout TO '60'; +NOTICE: issuing SET LOCAL citus.node_connection_timeout TO '35000'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing SET LOCAL citus.shard_count TO '4'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx @@ -255,7 +255,7 @@ NOTICE: issuing SET LOCAL citus.max_adaptive_executor_pool_size TO '5'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing SET LOCAL citus.next_shard_id TO '433105'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET LOCAL citus.node_connection_timeout TO '60'; +NOTICE: issuing SET LOCAL citus.node_connection_timeout TO '35000'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing SET LOCAL citus.shard_count TO '4'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx @@ -276,7 +276,7 @@ NOTICE: issuing SET LOCAL citus.max_adaptive_executor_pool_size TO '5'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing SET LOCAL citus.next_shard_id TO '433105'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET LOCAL citus.node_connection_timeout TO '60'; +NOTICE: issuing SET LOCAL citus.node_connection_timeout TO '35000'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing SET LOCAL citus.shard_count TO '4'; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx diff --git a/src/test/regress/sql/shard_rebalancer.sql b/src/test/regress/sql/shard_rebalancer.sql index 83f5b4f6d..9af0be49b 100644 --- a/src/test/regress/sql/shard_rebalancer.sql +++ b/src/test/regress/sql/shard_rebalancer.sql @@ -97,7 +97,7 @@ SELECT pg_reload_conf(); SELECT pg_sleep(.1); -- wait to make sure the config has changed before running the GUC -- replicate reference table should ignore the coordinator -SET citus.node_connection_timeout to 60; +SET citus.node_connection_timeout to '35s'; BEGIN; SET LOCAL citus.shard_replication_factor TO 2; SET citus.log_remote_commands TO ON;