external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix flakiness in multi_maintenance_multiple_databases.sql x2

pull/7286/head
ivyazmitinov 2024-02-12 13:48:44 +01:00
parent 5b2c297ca4
commit 10f4777aaf
2 changed files with 137 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
src/test/regress/expected/multi_maintenance_multiple_databases.out
View File

@ -255,32 +255,6 @@ WHERE state = 'idle'
(1 row)
\c - - - :master_port
-- Let maintenance do it's work...
DO
$$
BEGIN
FOR i IN 0 .. 300
LOOP
IF i = 300 THEN RAISE 'Waited too long'; END IF;
PERFORM pg_stat_clear_snapshot();
PERFORM * FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
IF (SELECT count(*) = 0 AS pg_dist_transaction_after_recovery_coordinator_test
FROM pg_database,
dblink(format('dbname=%s host=localhost port=%s user=postgres', datname,
(SELECT setting::int FROM pg_settings WHERE name = 'port')),
$statement$
SELECT groupid, gid
FROM pg_dist_transaction
WHERE gid LIKE 'citus_0_1234_4_0_%'
OR gid LIKE 'citus_0_should_be_forgotten_%'
$statement$) AS t(groupid integer, gid text)
WHERE datname LIKE 'db%') THEN
EXIT;
END IF;
PERFORM pg_sleep_for('1 SECOND'::interval);
END LOOP;
END
$$;
-- Verify maintenance result
SELECT count(*) = 0 AS too_many_clients_test
FROM regexp_split_to_table(pg_read_file('../log/postmaster.log'), E'\n') AS t(log_line)
@ -290,15 +264,43 @@ WHERE log_line LIKE '%sorry, too many clients already%';
t
(1 row)
DO
$$
DECLARE
pg_dist_transaction_after_recovery_coordinator_test boolean;
cached_connections_after_recovery_coordinator_test boolean;
BEGIN
FOR i IN 0 .. 300
LOOP
IF i = 300 THEN RAISE 'Waited too long'; END IF;
SELECT count(*) = 0
FROM pg_database,
dblink(format('dbname=%s host=localhost port=%s user=postgres', datname,
(SELECT setting::int FROM pg_settings WHERE name = 'port')),
$statement$
SELECT groupid, gid
FROM pg_dist_transaction
WHERE gid LIKE 'citus_0_1234_4_0_%'
OR gid LIKE 'citus_0_should_be_forgotten_%'
$statement$) AS t(groupid integer, gid text)
WHERE datname LIKE 'db%'
INTO pg_dist_transaction_after_recovery_coordinator_test;
SELECT count(*) = 0 AS cached_connections_after_recovery_coordinator_test
FROM pg_stat_activity
WHERE state = 'idle'
AND now() - backend_start > '5 seconds'::interval;
cached_connections_after_recovery_coordinator_test
---------------------------------------------------------------------
t
(1 row)
AND now() - backend_start > '5 seconds'::interval
INTO cached_connections_after_recovery_coordinator_test;
IF (pg_dist_transaction_after_recovery_coordinator_test
AND cached_connections_after_recovery_coordinator_test) THEN
EXIT;
END IF;
PERFORM pg_sleep_for('1 SECOND'::interval);
END LOOP;
END
$$;
\c - - - :worker_1_port
SELECT count(*) = 0 AS too_many_clients_test
FROM regexp_split_to_table(pg_read_file('../log/postmaster.log'), E'\n') AS t(log_line)
@ -317,15 +319,22 @@ WHERE gid LIKE 'citus_0_1234_4_0_%'
t
(1 row)
SELECT count(*) = 0 AS cached_connections_after_recovery_worker_1_test
DO
$$
BEGIN
FOR i IN 0 .. 300
LOOP
IF i = 300 THEN RAISE 'Waited too long'; END IF;
IF (SELECT count(*) = 0 AS cached_connections_after_recovery_worker_1_test
FROM pg_stat_activity
WHERE state = 'idle'
AND now() - backend_start > '5 seconds'::interval;
cached_connections_after_recovery_worker_1_test
---------------------------------------------------------------------
t
(1 row)
AND now() - backend_start > '5 seconds'::interval) THEN
EXIT;
END IF;
PERFORM pg_sleep_for('1 SECOND'::interval);
END LOOP;
END
$$;
\c - - - :worker_2_port
SELECT count(*) = 0 AS too_many_clients_test
FROM regexp_split_to_table(pg_read_file('../log/postmaster.log'), E'\n') AS t(log_line)
@ -344,15 +353,22 @@ WHERE gid LIKE 'citus_0_1234_4_0_%'
t
(1 row)
SELECT count(*) = 0 AS cached_connections_after_recovery_worker_2_test
DO
$$
BEGIN
FOR i IN 0 .. 300
LOOP
IF i = 300 THEN RAISE 'Waited too long'; END IF;
IF (SELECT count(*) = 0 AS cached_connections_after_recovery_worker_2_test
FROM pg_stat_activity
WHERE state = 'idle'
AND now() - backend_start > '5 seconds'::interval;
cached_connections_after_recovery_worker_2_test
---------------------------------------------------------------------
t
(1 row)
AND now() - backend_start > '5 seconds'::interval) THEN
EXIT;
END IF;
PERFORM pg_sleep_for('1 SECOND'::interval);
END LOOP;
END
$$;
-- Cleanup
\c - - - :master_port
SELECT $definition$

74
src/test/regress/sql/multi_maintenance_multiple_databases.sql
View File

@ -220,17 +220,22 @@ WHERE state = 'idle'
\c - - - :master_port
-- Let maintenance do it's work...
-- Verify maintenance result
SELECT count(*) = 0 AS too_many_clients_test
FROM regexp_split_to_table(pg_read_file('../log/postmaster.log'), E'\n') AS t(log_line)
WHERE log_line LIKE '%sorry, too many clients already%';
DO
$$
DECLARE
pg_dist_transaction_after_recovery_coordinator_test boolean;
cached_connections_after_recovery_coordinator_test boolean;
BEGIN
FOR i IN 0 .. 300
LOOP
IF i = 300 THEN RAISE 'Waited too long'; END IF;
PERFORM pg_stat_clear_snapshot();
PERFORM * FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
IF (SELECT count(*) = 0 AS pg_dist_transaction_after_recovery_coordinator_test
SELECT count(*) = 0
FROM pg_database,
dblink(format('dbname=%s host=localhost port=%s user=postgres', datname,
(SELECT setting::int FROM pg_settings WHERE name = 'port')),
@ -240,24 +245,24 @@ $$
WHERE gid LIKE 'citus_0_1234_4_0_%'
OR gid LIKE 'citus_0_should_be_forgotten_%'
$statement$) AS t(groupid integer, gid text)
WHERE datname LIKE 'db%') THEN
EXIT;
END IF;
PERFORM pg_sleep_for('1 SECOND'::interval);
END LOOP;
END
$$;
-- Verify maintenance result
SELECT count(*) = 0 AS too_many_clients_test
FROM regexp_split_to_table(pg_read_file('../log/postmaster.log'), E'\n') AS t(log_line)
WHERE log_line LIKE '%sorry, too many clients already%';
WHERE datname LIKE 'db%'
INTO pg_dist_transaction_after_recovery_coordinator_test;
SELECT count(*) = 0 AS cached_connections_after_recovery_coordinator_test
FROM pg_stat_activity
WHERE state = 'idle'
AND now() - backend_start > '5 seconds'::interval;
AND now() - backend_start > '5 seconds'::interval
INTO cached_connections_after_recovery_coordinator_test;
IF (pg_dist_transaction_after_recovery_coordinator_test
AND cached_connections_after_recovery_coordinator_test) THEN
EXIT;
END IF;
PERFORM pg_sleep_for('1 SECOND'::interval);
END LOOP;
END
$$;
\c - - - :worker_1_port
@ -270,10 +275,22 @@ FROM pg_prepared_xacts
WHERE gid LIKE 'citus_0_1234_4_0_%'
OR gid LIKE 'citus_0_should_be_forgotten_%';
SELECT count(*) = 0 AS cached_connections_after_recovery_worker_1_test
DO
$$
BEGIN
FOR i IN 0 .. 300
LOOP
IF i = 300 THEN RAISE 'Waited too long'; END IF;
IF (SELECT count(*) = 0 AS cached_connections_after_recovery_worker_1_test
FROM pg_stat_activity
WHERE state = 'idle'
AND now() - backend_start > '5 seconds'::interval;
AND now() - backend_start > '5 seconds'::interval) THEN
EXIT;
END IF;
PERFORM pg_sleep_for('1 SECOND'::interval);
END LOOP;
END
$$;
\c - - - :worker_2_port
@ -286,11 +303,22 @@ FROM pg_prepared_xacts
WHERE gid LIKE 'citus_0_1234_4_0_%'
OR gid LIKE 'citus_0_should_be_forgotten_%';
SELECT count(*) = 0 AS cached_connections_after_recovery_worker_2_test
DO
$$
BEGIN
FOR i IN 0 .. 300
LOOP
IF i = 300 THEN RAISE 'Waited too long'; END IF;
IF (SELECT count(*) = 0 AS cached_connections_after_recovery_worker_2_test
FROM pg_stat_activity
WHERE state = 'idle'
AND now() - backend_start > '5 seconds'::interval;
AND now() - backend_start > '5 seconds'::interval) THEN
EXIT;
END IF;
PERFORM pg_sleep_for('1 SECOND'::interval);
END LOOP;
END
$$;
-- Cleanup