diff --git a/src/test/regress/expected/ensure_no_intermediate_data_leak.out b/src/test/regress/expected/ensure_no_intermediate_data_leak.out index 4b9fb7a3a..2ed340294 100644 --- a/src/test/regress/expected/ensure_no_intermediate_data_leak.out +++ b/src/test/regress/expected/ensure_no_intermediate_data_leak.out @@ -1,20 +1,17 @@ ------ --- THIS TEST SHOULD IDEALLY BE EXECUTED AT THE END OF --- THE REGRESSION TEST SUITE TO MAKE SURE THAT WE --- CLEAR ALL INTERMEDIATE RESULTS ON BOTH THE COORDINATOR +-- THIS TEST SHOULD IDEALLY BE EXECUTED AT THE END OF +-- THE REGRESSION TEST SUITE TO MAKE SURE THAT WE +-- CLEAR ALL INTERMEDIATE RESULTS ON BOTH THE COORDINATOR -- AND ON THE WORKERS. HOWEVER, WE HAVE SOME ISSUES AROUND --- WINDOWS SUPPORT, FAILURES IN TASK-TRACKER EXECUTOR --- SO WE DISABLE THIS TEST ON WINDOWS +-- WINDOWS SUPPORT SO WE DISABLE THIS TEST ON WINDOWS ------ SELECT pg_ls_dir('base/pgsql_job_cache') WHERE citus_version() NOT ILIKE '%windows%'; pg_ls_dir ----------- (0 rows) -SELECT run_command_on_workers($$SELECT pg_ls_dir('base/pgsql_job_cache') WHERE citus_version() NOT ILIKE '%windows%'$$); - run_command_on_workers ------------------------- - (localhost,57637,t,"") - (localhost,57638,t,"") -(2 rows) +SELECT * FROM run_command_on_workers($$SELECT pg_ls_dir('base/pgsql_job_cache') r WHERE citus_version() NOT ILIKE '%windows%'$$) WHERE result <> ''; + nodename | nodeport | success | result +----------+----------+---------+-------- +(0 rows) diff --git a/src/test/regress/expected/multi_multiuser.out b/src/test/regress/expected/multi_multiuser.out index 28d7f114b..1276fa493 100644 --- a/src/test/regress/expected/multi_multiuser.out +++ b/src/test/regress/expected/multi_multiuser.out @@ -421,7 +421,7 @@ INSERT INTO full_access_user_schema.t1 VALUES (1),(2),(3); -- not allowed to create a table SELECT create_distributed_table('full_access_user_schema.t1', 'id'); ERROR: permission denied for schema full_access_user_schema -CONTEXT: while executing command on localhost:57638 +CONTEXT: while executing command on localhost:57637 RESET ROLE; SET ROLE usage_access; CREATE TYPE usage_access_type AS ENUM ('a', 'b'); @@ -672,7 +672,7 @@ ERROR: could not receive file "base/pgsql_job_cache/job_0042/task_000001/p_0000 -- different user should not be able to fetch partition file SET ROLE usage_access; SELECT worker_fetch_partition_file(42, 1, 1, 1, 'localhost', :worker_1_port); -WARNING: could not open file "base/pgsql_job_cache/job_0042/task_000001/p_00001.44518": No such file or directory +WARNING: could not open file "base/pgsql_job_cache/job_0042/task_000001/p_00001.18110": No such file or directory CONTEXT: while executing command on localhost:57637 ERROR: could not receive file "base/pgsql_job_cache/job_0042/task_000001/p_00001" from localhost:57637 -- only the user whom created the files should be able to fetch @@ -711,7 +711,7 @@ RESET ROLE; -- test that the super user is unable to read the contents of the intermediate file, -- although it does create the table SELECT worker_merge_files_into_table(42, 1, ARRAY['a'], ARRAY['integer']); -WARNING: Task file "task_000001.43115" does not have expected suffix ".10" +WARNING: Task file "task_000001.18048" does not have expected suffix ".10" worker_merge_files_into_table ------------------------------- @@ -753,7 +753,7 @@ SELECT worker_merge_files_and_run_query(42, 1, 'CREATE TABLE task_000001_merge(merge_column_0 int)', 'CREATE TABLE task_000001 (a) AS SELECT sum(merge_column_0) FROM task_000001_merge' ); -WARNING: Task file "task_000001.43115" does not have expected suffix ".10" +WARNING: Task file "task_000001.18048" does not have expected suffix ".10" worker_merge_files_and_run_query ---------------------------------- @@ -811,6 +811,13 @@ SELECT count(*) FROM pg_merge_job_0042.task_000001; DROP TABLE pg_merge_job_0042.task_000001, pg_merge_job_0042.task_000001_merge; -- drop table so we can reuse the same files for more tests RESET ROLE; \c - - - :master_port +SELECT run_command_on_workers($$SELECT task_tracker_cleanup_job(42);$$); + run_command_on_workers +------------------------ + (localhost,57637,t,"") + (localhost,57638,t,"") +(2 rows) + DROP SCHEMA full_access_user_schema CASCADE; NOTICE: drop cascades to 4 other objects DETAIL: drop cascades to table full_access_user_schema.t1 diff --git a/src/test/regress/expected/task_tracker_cleanup_job.out b/src/test/regress/expected/task_tracker_cleanup_job.out index 664084a77..4b5ade859 100644 --- a/src/test/regress/expected/task_tracker_cleanup_job.out +++ b/src/test/regress/expected/task_tracker_cleanup_job.out @@ -101,3 +101,10 @@ SELECT isdir FROM pg_stat_file('base/pgsql_job_cache/job_401010/task_801107'); ERROR: could not stat file "base/pgsql_job_cache/job_401010/task_801107": No such file or directory SELECT isdir FROM pg_stat_file('base/pgsql_job_cache/job_401010'); ERROR: could not stat file "base/pgsql_job_cache/job_401010": No such file or directory +-- Also clean up worker_cleanup_job_schema_cache job +SELECT task_tracker_cleanup_job(2); + task_tracker_cleanup_job +-------------------------- + +(1 row) + diff --git a/src/test/regress/expected/worker_remove_files.out b/src/test/regress/expected/worker_remove_files.out new file mode 100644 index 000000000..0d2a80734 --- /dev/null +++ b/src/test/regress/expected/worker_remove_files.out @@ -0,0 +1,8 @@ +-- Clear job directory used by previous tests +\set JobId 201010 +SELECT task_tracker_cleanup_job(:JobId); + task_tracker_cleanup_job +-------------------------- + +(1 row) + diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule index 8212e1bdd..c410c678e 100644 --- a/src/test/regress/failure_schedule +++ b/src/test/regress/failure_schedule @@ -30,3 +30,6 @@ test: failure_savepoints test: failure_multi_row_insert test: failure_mx_metadata_sync test: failure_connection_establishment + +# test that no tests leaked intermediate results. This should always be last +test: ensure_no_intermediate_data_leak diff --git a/src/test/regress/multi_follower_schedule b/src/test/regress/multi_follower_schedule index 9c0b63099..3e6efee6d 100644 --- a/src/test/regress/multi_follower_schedule +++ b/src/test/regress/multi_follower_schedule @@ -3,3 +3,6 @@ test: multi_follower_select_statements test: multi_follower_dml test: multi_follower_configure_followers test: multi_follower_task_tracker + +# test that no tests leaked intermediate results. This should always be last +test: ensure_no_intermediate_data_leak diff --git a/src/test/regress/multi_mx_schedule b/src/test/regress/multi_mx_schedule index 726050b64..38d000eb4 100644 --- a/src/test/regress/multi_mx_schedule +++ b/src/test/regress/multi_mx_schedule @@ -42,3 +42,6 @@ test: multi_mx_transaction_recovery test: multi_mx_modifying_xacts test: multi_mx_explain test: multi_mx_reference_table + +# test that no tests leaked intermediate results. This should always be last +test: ensure_no_intermediate_data_leak diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index fe8045e77..c65cc7d7a 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -313,3 +313,8 @@ test: distributed_procedure # deparsing logic tests # --------- test: multi_deparse_function multi_deparse_procedure + +# --------- +# test that no tests leaked intermediate results. This should always be last +# --------- +test: ensure_no_intermediate_data_leak diff --git a/src/test/regress/multi_task_tracker_extra_schedule b/src/test/regress/multi_task_tracker_extra_schedule index 0e0bfd6d4..6cefdd098 100644 --- a/src/test/regress/multi_task_tracker_extra_schedule +++ b/src/test/regress/multi_task_tracker_extra_schedule @@ -110,3 +110,7 @@ test: multi_drop_extension # ---------- test: multi_schema_support +# ---------- +# test that no tests leaked intermediate results. This should always be last +# ---------- +test: ensure_no_intermediate_data_leak diff --git a/src/test/regress/sql/ensure_no_intermediate_data_leak.sql b/src/test/regress/sql/ensure_no_intermediate_data_leak.sql index 614208f43..b265a46de 100644 --- a/src/test/regress/sql/ensure_no_intermediate_data_leak.sql +++ b/src/test/regress/sql/ensure_no_intermediate_data_leak.sql @@ -1,12 +1,11 @@ ------ -- THIS TEST SHOULD IDEALLY BE EXECUTED AT THE END OF --- THE REGRESSION TEST SUITE TO MAKE SURE THAT WE +-- THE REGRESSION TEST SUITE TO MAKE SURE THAT WE -- CLEAR ALL INTERMEDIATE RESULTS ON BOTH THE COORDINATOR -- AND ON THE WORKERS. HOWEVER, WE HAVE SOME ISSUES AROUND --- WINDOWS SUPPORT, FAILURES IN TASK-TRACKER EXECUTOR --- SO WE DISABLE THIS TEST ON WINDOWS +-- WINDOWS SUPPORT SO WE DISABLE THIS TEST ON WINDOWS ------ SELECT pg_ls_dir('base/pgsql_job_cache') WHERE citus_version() NOT ILIKE '%windows%'; -SELECT run_command_on_workers($$SELECT pg_ls_dir('base/pgsql_job_cache') WHERE citus_version() NOT ILIKE '%windows%'$$); +SELECT * FROM run_command_on_workers($$SELECT pg_ls_dir('base/pgsql_job_cache') r WHERE citus_version() NOT ILIKE '%windows%'$$) WHERE result <> ''; diff --git a/src/test/regress/sql/multi_multiuser.sql b/src/test/regress/sql/multi_multiuser.sql index 76ba3ee50..651e489e5 100644 --- a/src/test/regress/sql/multi_multiuser.sql +++ b/src/test/regress/sql/multi_multiuser.sql @@ -488,6 +488,7 @@ RESET ROLE; \c - - - :master_port +SELECT run_command_on_workers($$SELECT task_tracker_cleanup_job(42);$$); DROP SCHEMA full_access_user_schema CASCADE; DROP TABLE diff --git a/src/test/regress/sql/task_tracker_cleanup_job.sql b/src/test/regress/sql/task_tracker_cleanup_job.sql index 9a8b9af82..ffabf1eed 100644 --- a/src/test/regress/sql/task_tracker_cleanup_job.sql +++ b/src/test/regress/sql/task_tracker_cleanup_job.sql @@ -46,3 +46,6 @@ SELECT task_tracker_task_status(:JobId, :RunningTaskId); SELECT isdir FROM pg_stat_file('base/pgsql_job_cache/job_401010/task_801107'); SELECT isdir FROM pg_stat_file('base/pgsql_job_cache/job_401010'); + +-- Also clean up worker_cleanup_job_schema_cache job +SELECT task_tracker_cleanup_job(2); diff --git a/src/test/regress/sql/worker_remove_files.sql b/src/test/regress/sql/worker_remove_files.sql new file mode 100644 index 000000000..9b3ce85ba --- /dev/null +++ b/src/test/regress/sql/worker_remove_files.sql @@ -0,0 +1,5 @@ +-- Clear job directory used by previous tests + +\set JobId 201010 + +SELECT task_tracker_cleanup_job(:JobId); diff --git a/src/test/regress/worker_schedule b/src/test/regress/worker_schedule index e3b3d7a51..031bdff90 100644 --- a/src/test/regress/worker_schedule +++ b/src/test/regress/worker_schedule @@ -19,6 +19,7 @@ test: worker_hash_partition worker_hash_partition_complex test: worker_merge_range_files worker_merge_hash_files test: worker_binary_data_partition worker_null_data_partition test: worker_check_invalid_arguments +test: worker_remove_files # ---------- # All task tracker tests use the following tables @@ -26,3 +27,8 @@ test: worker_check_invalid_arguments test: task_tracker_create_table test: task_tracker_assign_task task_tracker_partition_task test: task_tracker_cleanup_job + +# --------- +# test that no tests leaked intermediate results. This should always be last +# --------- +test: ensure_no_intermediate_data_leak