external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

simplify vacuum test and fix the flakiness (#3704) 

look at sent commands to simplify complex logic in vacuum test

also normalize connection id as that can differ when we don't have to
choose a specific connection.
pull/3490/merge
SaitTalhaNisanci 2020-04-03 21:39:54 +03:00 committed by GitHub
parent a0f95c5b70
commit 3d3605be80
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 99 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/test/regress/bin/normalize.sed
View File

@ -68,6 +68,9 @@ s/(NOTICE: [a-z]+ cascades to table ".*)_[0-9]{5,}"/\1_xxxxx"/g
/^LINE [0-9]+:.*$/d /^LINE [0-9]+:.*$/d
/^ *\^$/d /^ *\^$/d
# connection id
s/connectionId: [0-9]+/connectionId: xxxxxxx/g
# Remove trailing whitespace # Remove trailing whitespace
s/ *$//g s/ *$//g

6
src/test/regress/expected/multi_router_planner.out
View File

@ -1756,7 +1756,7 @@ SELECT * FROM articles_range where author_id = 1;
DEBUG: Creating router plan DEBUG: Creating router plan
DEBUG: Plan is router executable DEBUG: Plan is router executable
NOTICE: issuing SELECT id, author_id, title, word_count FROM public.articles_range_840012 articles_range WHERE (author_id OPERATOR(pg_catalog.=) 1) NOTICE: issuing SELECT id, author_id, title, word_count FROM public.articles_range_840012 articles_range WHERE (author_id OPERATOR(pg_catalog.=) 1)
DETAIL: on server postgres@localhost:xxxxx connectionId: 2 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
id | author_id | title | word_count id | author_id | title | word_count
--------------------------------------------------------------------- ---------------------------------------------------------------------
(0 rows) (0 rows)
@ -1765,7 +1765,7 @@ SELECT * FROM articles_range where author_id = 1 or author_id = 5;
DEBUG: Creating router plan DEBUG: Creating router plan
DEBUG: Plan is router executable DEBUG: Plan is router executable
NOTICE: issuing SELECT id, author_id, title, word_count FROM public.articles_range_840012 articles_range WHERE ((author_id OPERATOR(pg_catalog.=) 1) OR (author_id OPERATOR(pg_catalog.=) 5)) NOTICE: issuing SELECT id, author_id, title, word_count FROM public.articles_range_840012 articles_range WHERE ((author_id OPERATOR(pg_catalog.=) 1) OR (author_id OPERATOR(pg_catalog.=) 5))
DETAIL: on server postgres@localhost:xxxxx connectionId: 2 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
id | author_id | title | word_count id | author_id | title | word_count
--------------------------------------------------------------------- ---------------------------------------------------------------------
(0 rows) (0 rows)
@ -1785,7 +1785,7 @@ SELECT * FROM articles_range ar join authors_range au on (ar.author_id = au.id)
DEBUG: Creating router plan DEBUG: Creating router plan
DEBUG: Plan is router executable DEBUG: Plan is router executable
NOTICE: issuing SELECT ar.id, ar.author_id, ar.title, ar.word_count, au.name, au.id FROM (public.articles_range_840012 ar JOIN public.authors_range_840008 au ON ((ar.author_id OPERATOR(pg_catalog.=) au.id))) WHERE (ar.author_id OPERATOR(pg_catalog.=) 1) NOTICE: issuing SELECT ar.id, ar.author_id, ar.title, ar.word_count, au.name, au.id FROM (public.articles_range_840012 ar JOIN public.authors_range_840008 au ON ((ar.author_id OPERATOR(pg_catalog.=) au.id))) WHERE (ar.author_id OPERATOR(pg_catalog.=) 1)
DETAIL: on server postgres@localhost:xxxxx connectionId: 2 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
id | author_id | title | word_count | name | id id | author_id | title | word_count | name | id
--------------------------------------------------------------------- ---------------------------------------------------------------------
(0 rows) (0 rows)

278
src/test/regress/expected/multi_utilities.out
View File

@ -197,164 +197,64 @@ SELECT master_create_worker_shards('second_dustbunnies', 1, 2);
(1 row) (1 row)
-- following approach adapted from PostgreSQL's stats.sql file
-- save relevant stat counter values in refreshable view
\c - - - :worker_1_port
CREATE MATERIALIZED VIEW prevcounts AS
SELECT analyze_count, vacuum_count FROM pg_stat_user_tables
WHERE relname='dustbunnies_990002';
-- create function that sleeps until those counters increment
create function wait_for_stats() returns void as $$
declare
start_time timestamptz := clock_timestamp();
analyze_updated bool;
vacuum_updated bool;
begin
-- we don't want to wait forever; loop will exit after 10 seconds
for i in 1 .. 100 loop
-- check to see if analyze has been updated
SELECT (st.analyze_count >= pc.analyze_count + 1) INTO analyze_updated
FROM pg_stat_user_tables AS st, pg_class AS cl, prevcounts AS pc
WHERE st.relname='dustbunnies_990002' AND cl.relname='dustbunnies_990002';
-- check to see if vacuum has been updated
SELECT (st.vacuum_count >= pc.vacuum_count + 1) INTO vacuum_updated
FROM pg_stat_user_tables AS st, pg_class AS cl, prevcounts AS pc
WHERE st.relname='dustbunnies_990002' AND cl.relname='dustbunnies_990002';
exit when analyze_updated or vacuum_updated;
-- wait a little
perform pg_sleep(0.1);
-- reset stats snapshot so we can test again
perform pg_stat_clear_snapshot();
-- fail if we reach the end of this loop
if i = 100 then
raise 'Waited too long for analyze/vacuum';
end if;
end loop;
-- report time waited in postmaster log (where it won't change test output)
raise log 'wait_for_stats delayed % seconds',
extract(epoch from clock_timestamp() - start_time);
end
$$ language plpgsql;
\c - - - :worker_2_port
CREATE MATERIALIZED VIEW prevcounts AS
SELECT analyze_count, vacuum_count FROM pg_stat_user_tables
WHERE relname='dustbunnies_990002';
-- create function that sleeps until those counters increment
create function wait_for_stats() returns void as $$
declare
start_time timestamptz := clock_timestamp();
analyze_updated bool;
vacuum_updated bool;
begin
-- we don't want to wait forever; loop will exit after 10 seconds
for i in 1 .. 100 loop
-- check to see if analyze has been updated
SELECT (st.analyze_count >= pc.analyze_count + 1) INTO analyze_updated
FROM pg_stat_user_tables AS st, pg_class AS cl, prevcounts AS pc
WHERE st.relname='dustbunnies_990002' AND cl.relname='dustbunnies_990002';
-- check to see if vacuum has been updated
SELECT (st.vacuum_count >= pc.vacuum_count + 1) INTO vacuum_updated
FROM pg_stat_user_tables AS st, pg_class AS cl, prevcounts AS pc
WHERE st.relname='dustbunnies_990002' AND cl.relname='dustbunnies_990002';
exit when analyze_updated or vacuum_updated;
-- wait a little
perform pg_sleep(0.1);
-- reset stats snapshot so we can test again
perform pg_stat_clear_snapshot();
-- fail if we reach the end of this loop
if i = 100 then
raise 'Waited too long for analyze/vacuum';
end if;
end loop;
-- report time waited in postmaster log (where it won't change test output)
raise log 'wait_for_stats delayed % seconds',
extract(epoch from clock_timestamp() - start_time);
end
$$ language plpgsql;
-- run VACUUM and ANALYZE against the table on the master -- run VACUUM and ANALYZE against the table on the master
\c - - - :master_port \c - - - :master_port
SET citus.log_remote_commands TO ON;
VACUUM dustbunnies; VACUUM dustbunnies;
NOTICE: issuing VACUUM public.dustbunnies_990002
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing VACUUM public.dustbunnies_990002
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
ANALYZE dustbunnies; ANALYZE dustbunnies;
-- verify that the VACUUM and ANALYZE ran NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
\c - - - :worker_1_port DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
SELECT wait_for_stats(); NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
wait_for_stats DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
--------------------------------------------------------------------- NOTICE: issuing ANALYZE public.dustbunnies_990002
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
(1 row) NOTICE: issuing ANALYZE public.dustbunnies_990002
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
REFRESH MATERIALIZED VIEW prevcounts; NOTICE: issuing COMMIT
SELECT pg_stat_get_vacuum_count('dustbunnies_990002'::regclass); DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
pg_stat_get_vacuum_count NOTICE: issuing COMMIT
--------------------------------------------------------------------- DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
1
(1 row)
SELECT pg_stat_get_analyze_count('dustbunnies_990002'::regclass);
pg_stat_get_analyze_count
---------------------------------------------------------------------
1
(1 row)
-- get file node to verify VACUUM FULL
SELECT relfilenode AS oldnode FROM pg_class WHERE oid='dustbunnies_990002'::regclass
\gset
-- send a VACUUM FULL and a VACUUM ANALYZE -- send a VACUUM FULL and a VACUUM ANALYZE
\c - - - :master_port
VACUUM (FULL) dustbunnies; VACUUM (FULL) dustbunnies;
NOTICE: issuing VACUUM (FULL) public.dustbunnies_990002
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing VACUUM (FULL) public.dustbunnies_990002
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
VACUUM ANALYZE dustbunnies; VACUUM ANALYZE dustbunnies;
-- verify that relfilenode changed NOTICE: issuing VACUUM (ANALYZE) public.dustbunnies_990002
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing VACUUM (ANALYZE) public.dustbunnies_990002
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
\c - - - :worker_1_port \c - - - :worker_1_port
SELECT relfilenode != :oldnode AS table_rewritten FROM pg_class
WHERE oid='dustbunnies_990002'::regclass;
table_rewritten
---------------------------------------------------------------------
t
(1 row)
-- verify the VACUUM ANALYZE incremented both vacuum and analyze counts
SELECT wait_for_stats();
wait_for_stats
---------------------------------------------------------------------
(1 row)
SELECT pg_stat_get_vacuum_count('dustbunnies_990002'::regclass);
pg_stat_get_vacuum_count
---------------------------------------------------------------------
2
(1 row)
SELECT pg_stat_get_analyze_count('dustbunnies_990002'::regclass);
pg_stat_get_analyze_count
---------------------------------------------------------------------
2
(1 row)
-- disable auto-VACUUM for next test -- disable auto-VACUUM for next test
ALTER TABLE dustbunnies_990002 SET (autovacuum_enabled = false); ALTER TABLE dustbunnies_990002 SET (autovacuum_enabled = false);
SELECT relfrozenxid AS frozenxid FROM pg_class WHERE oid='dustbunnies_990002'::regclass SELECT relfrozenxid AS frozenxid FROM pg_class WHERE oid='dustbunnies_990002'::regclass
\gset \gset
-- send a VACUUM FREEZE after adding a new row -- send a VACUUM FREEZE after adding a new row
\c - - - :master_port \c - - - :master_port
SET citus.log_remote_commands TO ON;
INSERT INTO dustbunnies VALUES (5, 'peter'); INSERT INTO dustbunnies VALUES (5, 'peter');
NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing INSERT INTO public.dustbunnies_990002 (id, name) VALUES (5, 'peter'::text)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing INSERT INTO public.dustbunnies_990002 (id, name) VALUES (5, 'peter'::text)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing COMMIT
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing COMMIT
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
VACUUM (FREEZE) dustbunnies; VACUUM (FREEZE) dustbunnies;
NOTICE: issuing VACUUM (FREEZE) public.dustbunnies_990002
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing VACUUM (FREEZE) public.dustbunnies_990002
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
-- verify that relfrozenxid increased -- verify that relfrozenxid increased
\c - - - :worker_1_port \c - - - :worker_1_port
SELECT relfrozenxid::text::integer > :frozenxid AS frozen_performed FROM pg_class SELECT relfrozenxid::text::integer > :frozenxid AS frozen_performed FROM pg_class
@ -376,8 +276,33 @@ WHERE tablename = 'dustbunnies_990002' ORDER BY attname;
-- add NULL values, then perform column-specific ANALYZE -- add NULL values, then perform column-specific ANALYZE
\c - - - :master_port \c - - - :master_port
SET citus.log_remote_commands TO ON;
INSERT INTO dustbunnies VALUES (6, NULL, NULL); INSERT INTO dustbunnies VALUES (6, NULL, NULL);
NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing INSERT INTO public.dustbunnies_990002 (id, name, age) VALUES (6, NULL::text, NULL::integer)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing INSERT INTO public.dustbunnies_990002 (id, name, age) VALUES (6, NULL::text, NULL::integer)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing COMMIT
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing COMMIT
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
ANALYZE dustbunnies (name); ANALYZE dustbunnies (name);
NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing ANALYZE public.dustbunnies_990002 (name)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing ANALYZE public.dustbunnies_990002 (name)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing COMMIT
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing COMMIT
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
-- verify that name's NULL ratio is updated but age's is not -- verify that name's NULL ratio is updated but age's is not
\c - - - :worker_1_port \c - - - :worker_1_port
SELECT attname, null_frac FROM pg_stats SELECT attname, null_frac FROM pg_stats
@ -389,43 +314,22 @@ WHERE tablename = 'dustbunnies_990002' ORDER BY attname;
name | 0.166667 name | 0.166667
(3 rows) (3 rows)
REFRESH MATERIALIZED VIEW prevcounts;
\c - - - :worker_2_port
REFRESH MATERIALIZED VIEW prevcounts;
\c - - - :master_port \c - - - :master_port
SET citus.log_remote_commands TO ON;
-- verify warning for unqualified VACUUM -- verify warning for unqualified VACUUM
VACUUM; VACUUM;
WARNING: not propagating VACUUM command to worker nodes WARNING: not propagating VACUUM command to worker nodes
HINT: Provide a specific table in order to VACUUM distributed tables. HINT: Provide a specific table in order to VACUUM distributed tables.
-- check for multiple table vacuum -- check for multiple table vacuum
VACUUM dustbunnies, second_dustbunnies; VACUUM dustbunnies, second_dustbunnies;
\c - - - :worker_1_port NOTICE: issuing VACUUM public.dustbunnies_990002
REFRESH MATERIALIZED VIEW prevcounts; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
\c - - - :worker_2_port NOTICE: issuing VACUUM public.dustbunnies_990002
REFRESH MATERIALIZED VIEW prevcounts; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
\c - - - :master_port NOTICE: issuing VACUUM public.second_dustbunnies_990003
-- check the current number of vacuum and analyze run on dustbunnies DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
SELECT run_command_on_workers($$SELECT wait_for_stats()$$); NOTICE: issuing VACUUM public.second_dustbunnies_990003
run_command_on_workers DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
---------------------------------------------------------------------
(localhost,57637,t,"")
(localhost,57638,t,"")
(2 rows)
SELECT run_command_on_workers($$SELECT pg_stat_get_vacuum_count(tablename::regclass) from pg_tables where tablename LIKE 'dustbunnies_%' limit 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,57637,t,4)
(localhost,57638,t,4)
(2 rows)
SELECT run_command_on_workers($$SELECT pg_stat_get_analyze_count(tablename::regclass) from pg_tables where tablename LIKE 'dustbunnies_%' limit 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,57637,t,3)
(localhost,57638,t,3)
(2 rows)
-- and warning when using targeted VACUUM without DDL propagation -- and warning when using targeted VACUUM without DDL propagation
SET citus.enable_ddl_propagation to false; SET citus.enable_ddl_propagation to false;
VACUUM dustbunnies; VACUUM dustbunnies;
@ -435,28 +339,6 @@ ANALYZE dustbunnies;
WARNING: not propagating ANALYZE command to worker nodes WARNING: not propagating ANALYZE command to worker nodes
HINT: Set citus.enable_ddl_propagation to true in order to send targeted ANALYZE commands to worker nodes. HINT: Set citus.enable_ddl_propagation to true in order to send targeted ANALYZE commands to worker nodes.
SET citus.enable_ddl_propagation to DEFAULT; SET citus.enable_ddl_propagation to DEFAULT;
-- should not propagate the vacuum and analyze
SELECT run_command_on_workers($$SELECT wait_for_stats()$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,57637,t,"")
(localhost,57638,t,"")
(2 rows)
SELECT run_command_on_workers($$SELECT pg_stat_get_vacuum_count(tablename::regclass) from pg_tables where tablename LIKE 'dustbunnies_%' limit 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,57637,t,4)
(localhost,57638,t,4)
(2 rows)
SELECT run_command_on_workers($$SELECT pg_stat_get_analyze_count(tablename::regclass) from pg_tables where tablename LIKE 'dustbunnies_%' limit 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,57637,t,3)
(localhost,57638,t,3)
(2 rows)
-- test worker_hash -- test worker_hash
SELECT worker_hash(123); SELECT worker_hash(123);
worker_hash worker_hash
@ -484,6 +366,14 @@ SELECT citus_truncate_trigger();
ERROR: must be called as trigger ERROR: must be called as trigger
-- confirm that citus_create_restore_point works -- confirm that citus_create_restore_point works
SELECT 1 FROM citus_create_restore_point('regression-test'); SELECT 1 FROM citus_create_restore_point('regression-test');
NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing SELECT pg_catalog.pg_create_restore_point($1::text)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing SELECT pg_catalog.pg_create_restore_point($1::text)
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
?column? ?column?
--------------------------------------------------------------------- ---------------------------------------------------------------------
1 1

144
src/test/regress/sql/multi_utilities.sql
View File

@ -130,131 +130,20 @@ CREATE TABLE second_dustbunnies(id integer, name text, age integer);
SELECT master_create_distributed_table('second_dustbunnies', 'id', 'hash'); SELECT master_create_distributed_table('second_dustbunnies', 'id', 'hash');
SELECT master_create_worker_shards('second_dustbunnies', 1, 2); SELECT master_create_worker_shards('second_dustbunnies', 1, 2);
-- following approach adapted from PostgreSQL's stats.sql file
-- save relevant stat counter values in refreshable view
\c - - - :worker_1_port
CREATE MATERIALIZED VIEW prevcounts AS
SELECT analyze_count, vacuum_count FROM pg_stat_user_tables
WHERE relname='dustbunnies_990002';
-- create function that sleeps until those counters increment
create function wait_for_stats() returns void as $$
declare
start_time timestamptz := clock_timestamp();
analyze_updated bool;
vacuum_updated bool;
begin
-- we don't want to wait forever; loop will exit after 10 seconds
for i in 1 .. 100 loop
-- check to see if analyze has been updated
SELECT (st.analyze_count >= pc.analyze_count + 1) INTO analyze_updated
FROM pg_stat_user_tables AS st, pg_class AS cl, prevcounts AS pc
WHERE st.relname='dustbunnies_990002' AND cl.relname='dustbunnies_990002';
-- check to see if vacuum has been updated
SELECT (st.vacuum_count >= pc.vacuum_count + 1) INTO vacuum_updated
FROM pg_stat_user_tables AS st, pg_class AS cl, prevcounts AS pc
WHERE st.relname='dustbunnies_990002' AND cl.relname='dustbunnies_990002';
exit when analyze_updated or vacuum_updated;
-- wait a little
perform pg_sleep(0.1);
-- reset stats snapshot so we can test again
perform pg_stat_clear_snapshot();
-- fail if we reach the end of this loop
if i = 100 then
raise 'Waited too long for analyze/vacuum';
end if;
end loop;
-- report time waited in postmaster log (where it won't change test output)
raise log 'wait_for_stats delayed % seconds',
extract(epoch from clock_timestamp() - start_time);
end
$$ language plpgsql;
\c - - - :worker_2_port
CREATE MATERIALIZED VIEW prevcounts AS
SELECT analyze_count, vacuum_count FROM pg_stat_user_tables
WHERE relname='dustbunnies_990002';
-- create function that sleeps until those counters increment
create function wait_for_stats() returns void as $$
declare
start_time timestamptz := clock_timestamp();
analyze_updated bool;
vacuum_updated bool;
begin
-- we don't want to wait forever; loop will exit after 10 seconds
for i in 1 .. 100 loop
-- check to see if analyze has been updated
SELECT (st.analyze_count >= pc.analyze_count + 1) INTO analyze_updated
FROM pg_stat_user_tables AS st, pg_class AS cl, prevcounts AS pc
WHERE st.relname='dustbunnies_990002' AND cl.relname='dustbunnies_990002';
-- check to see if vacuum has been updated
SELECT (st.vacuum_count >= pc.vacuum_count + 1) INTO vacuum_updated
FROM pg_stat_user_tables AS st, pg_class AS cl, prevcounts AS pc
WHERE st.relname='dustbunnies_990002' AND cl.relname='dustbunnies_990002';
exit when analyze_updated or vacuum_updated;
-- wait a little
perform pg_sleep(0.1);
-- reset stats snapshot so we can test again
perform pg_stat_clear_snapshot();
-- fail if we reach the end of this loop
if i = 100 then
raise 'Waited too long for analyze/vacuum';
end if;
end loop;
-- report time waited in postmaster log (where it won't change test output)
raise log 'wait_for_stats delayed % seconds',
extract(epoch from clock_timestamp() - start_time);
end
$$ language plpgsql;
-- run VACUUM and ANALYZE against the table on the master -- run VACUUM and ANALYZE against the table on the master
\c - - - :master_port \c - - - :master_port
SET citus.log_remote_commands TO ON;
VACUUM dustbunnies; VACUUM dustbunnies;
ANALYZE dustbunnies; ANALYZE dustbunnies;
-- verify that the VACUUM and ANALYZE ran
\c - - - :worker_1_port
SELECT wait_for_stats();
REFRESH MATERIALIZED VIEW prevcounts;
SELECT pg_stat_get_vacuum_count('dustbunnies_990002'::regclass);
SELECT pg_stat_get_analyze_count('dustbunnies_990002'::regclass);
-- get file node to verify VACUUM FULL
SELECT relfilenode AS oldnode FROM pg_class WHERE oid='dustbunnies_990002'::regclass
\gset
-- send a VACUUM FULL and a VACUUM ANALYZE -- send a VACUUM FULL and a VACUUM ANALYZE
\c - - - :master_port
VACUUM (FULL) dustbunnies; VACUUM (FULL) dustbunnies;
VACUUM ANALYZE dustbunnies; VACUUM ANALYZE dustbunnies;
-- verify that relfilenode changed
\c - - - :worker_1_port \c - - - :worker_1_port
SELECT relfilenode != :oldnode AS table_rewritten FROM pg_class
WHERE oid='dustbunnies_990002'::regclass;
-- verify the VACUUM ANALYZE incremented both vacuum and analyze counts
SELECT wait_for_stats();
SELECT pg_stat_get_vacuum_count('dustbunnies_990002'::regclass);
SELECT pg_stat_get_analyze_count('dustbunnies_990002'::regclass);
-- disable auto-VACUUM for next test -- disable auto-VACUUM for next test
ALTER TABLE dustbunnies_990002 SET (autovacuum_enabled = false); ALTER TABLE dustbunnies_990002 SET (autovacuum_enabled = false);
SELECT relfrozenxid AS frozenxid FROM pg_class WHERE oid='dustbunnies_990002'::regclass SELECT relfrozenxid AS frozenxid FROM pg_class WHERE oid='dustbunnies_990002'::regclass
@ -262,6 +151,8 @@ SELECT relfrozenxid AS frozenxid FROM pg_class WHERE oid='dustbunnies_990002'::r
-- send a VACUUM FREEZE after adding a new row -- send a VACUUM FREEZE after adding a new row
\c - - - :master_port \c - - - :master_port
SET citus.log_remote_commands TO ON;
INSERT INTO dustbunnies VALUES (5, 'peter'); INSERT INTO dustbunnies VALUES (5, 'peter');
VACUUM (FREEZE) dustbunnies; VACUUM (FREEZE) dustbunnies;
@ -276,6 +167,8 @@ WHERE tablename = 'dustbunnies_990002' ORDER BY attname;
-- add NULL values, then perform column-specific ANALYZE -- add NULL values, then perform column-specific ANALYZE
\c - - - :master_port \c - - - :master_port
SET citus.log_remote_commands TO ON;
INSERT INTO dustbunnies VALUES (6, NULL, NULL); INSERT INTO dustbunnies VALUES (6, NULL, NULL);
ANALYZE dustbunnies (name); ANALYZE dustbunnies (name);
@ -284,40 +177,21 @@ ANALYZE dustbunnies (name);
SELECT attname, null_frac FROM pg_stats SELECT attname, null_frac FROM pg_stats
WHERE tablename = 'dustbunnies_990002' ORDER BY attname; WHERE tablename = 'dustbunnies_990002' ORDER BY attname;
REFRESH MATERIALIZED VIEW prevcounts;
\c - - - :worker_2_port
REFRESH MATERIALIZED VIEW prevcounts;
\c - - - :master_port \c - - - :master_port
SET citus.log_remote_commands TO ON;
-- verify warning for unqualified VACUUM -- verify warning for unqualified VACUUM
VACUUM; VACUUM;
-- check for multiple table vacuum -- check for multiple table vacuum
VACUUM dustbunnies, second_dustbunnies; VACUUM dustbunnies, second_dustbunnies;
\c - - - :worker_1_port
REFRESH MATERIALIZED VIEW prevcounts;
\c - - - :worker_2_port
REFRESH MATERIALIZED VIEW prevcounts;
\c - - - :master_port
-- check the current number of vacuum and analyze run on dustbunnies
SELECT run_command_on_workers($$SELECT wait_for_stats()$$);
SELECT run_command_on_workers($$SELECT pg_stat_get_vacuum_count(tablename::regclass) from pg_tables where tablename LIKE 'dustbunnies_%' limit 1$$);
SELECT run_command_on_workers($$SELECT pg_stat_get_analyze_count(tablename::regclass) from pg_tables where tablename LIKE 'dustbunnies_%' limit 1$$);
-- and warning when using targeted VACUUM without DDL propagation -- and warning when using targeted VACUUM without DDL propagation
SET citus.enable_ddl_propagation to false; SET citus.enable_ddl_propagation to false;
VACUUM dustbunnies; VACUUM dustbunnies;
ANALYZE dustbunnies; ANALYZE dustbunnies;
SET citus.enable_ddl_propagation to DEFAULT; SET citus.enable_ddl_propagation to DEFAULT;
-- should not propagate the vacuum and analyze
SELECT run_command_on_workers($$SELECT wait_for_stats()$$);
SELECT run_command_on_workers($$SELECT pg_stat_get_vacuum_count(tablename::regclass) from pg_tables where tablename LIKE 'dustbunnies_%' limit 1$$);
SELECT run_command_on_workers($$SELECT pg_stat_get_analyze_count(tablename::regclass) from pg_tables where tablename LIKE 'dustbunnies_%' limit 1$$);
-- test worker_hash -- test worker_hash
SELECT worker_hash(123); SELECT worker_hash(123);
SELECT worker_hash('1997-08-08'::date); SELECT worker_hash('1997-08-08'::date);