End regression tests with ensure_no_intermediate_data_leak

Also update tests to clean up jobs when they're directly testing job udfs

src/test/regress/expected/ensure_no_intermediate_data_leak.out

@@ -1,20 +1,17 @@
 ------
--- THIS TEST SHOULD IDEALLY BE EXECUTED AT THE END OF 
--- THE REGRESSION TEST SUITE  TO MAKE SURE THAT WE 
--- CLEAR ALL INTERMEDIATE RESULTS ON BOTH THE COORDINATOR 
+-- THIS TEST SHOULD IDEALLY BE EXECUTED AT THE END OF
+-- THE REGRESSION TEST SUITE TO MAKE SURE THAT WE
+-- CLEAR ALL INTERMEDIATE RESULTS ON BOTH THE COORDINATOR
 -- AND ON THE WORKERS. HOWEVER, WE HAVE SOME ISSUES AROUND
--- WINDOWS SUPPORT, FAILURES IN TASK-TRACKER EXECUTOR
--- SO WE DISABLE THIS TEST ON WINDOWS
+-- WINDOWS SUPPORT SO WE DISABLE THIS TEST ON WINDOWS
 ------
 SELECT pg_ls_dir('base/pgsql_job_cache') WHERE citus_version() NOT ILIKE '%windows%';
  pg_ls_dir 
 -----------
 (0 rows)
 
-SELECT run_command_on_workers($$SELECT pg_ls_dir('base/pgsql_job_cache') WHERE citus_version() NOT ILIKE '%windows%'$$);
- run_command_on_workers 
-------------------------
- (localhost,57637,t,"")
- (localhost,57638,t,"")
-(2 rows)
+SELECT * FROM run_command_on_workers($$SELECT pg_ls_dir('base/pgsql_job_cache') r WHERE citus_version() NOT ILIKE '%windows%'$$) WHERE result <> '';
+ nodename | nodeport | success | result 
+----------+----------+---------+--------
+(0 rows)
 

src/test/regress/expected/multi_multiuser.out

@@ -421,7 +421,7 @@ INSERT INTO full_access_user_schema.t1 VALUES (1),(2),(3);
 -- not allowed to create a table
 SELECT create_distributed_table('full_access_user_schema.t1', 'id');
 ERROR:  permission denied for schema full_access_user_schema
-CONTEXT:  while executing command on localhost:57638
+CONTEXT:  while executing command on localhost:57637
 RESET ROLE;
 SET ROLE usage_access;
 CREATE TYPE usage_access_type AS ENUM ('a', 'b');
@@ -672,7 +672,7 @@ ERROR:  could not receive file "base/pgsql_job_cache/job_0042/task_000001/p_0000
 -- different user should not be able to fetch partition file
 SET ROLE usage_access;
 SELECT worker_fetch_partition_file(42, 1, 1, 1, 'localhost', :worker_1_port);
-WARNING:  could not open file "base/pgsql_job_cache/job_0042/task_000001/p_00001.44518": No such file or directory
+WARNING:  could not open file "base/pgsql_job_cache/job_0042/task_000001/p_00001.18110": No such file or directory
 CONTEXT:  while executing command on localhost:57637
 ERROR:  could not receive file "base/pgsql_job_cache/job_0042/task_000001/p_00001" from localhost:57637
 -- only the user whom created the files should be able to fetch
@@ -711,7 +711,7 @@ RESET ROLE;
 -- test that the super user is unable to read the contents of the intermediate file,
 -- although it does create the table
 SELECT worker_merge_files_into_table(42, 1, ARRAY['a'], ARRAY['integer']);
-WARNING:  Task file "task_000001.43115" does not have expected suffix ".10"
+WARNING:  Task file "task_000001.18048" does not have expected suffix ".10"
  worker_merge_files_into_table 
 -------------------------------
  
@@ -753,7 +753,7 @@ SELECT worker_merge_files_and_run_query(42, 1,
     'CREATE TABLE task_000001_merge(merge_column_0 int)',
     'CREATE TABLE task_000001 (a) AS SELECT sum(merge_column_0) FROM task_000001_merge'
 );
-WARNING:  Task file "task_000001.43115" does not have expected suffix ".10"
+WARNING:  Task file "task_000001.18048" does not have expected suffix ".10"
  worker_merge_files_and_run_query 
 ----------------------------------
  
@@ -811,6 +811,13 @@ SELECT count(*) FROM pg_merge_job_0042.task_000001;
 DROP TABLE pg_merge_job_0042.task_000001, pg_merge_job_0042.task_000001_merge; -- drop table so we can reuse the same files for more tests
 RESET ROLE;
 \c - - - :master_port
+SELECT run_command_on_workers($$SELECT task_tracker_cleanup_job(42);$$);
+ run_command_on_workers 
+------------------------
+ (localhost,57637,t,"")
+ (localhost,57638,t,"")
+(2 rows)
+
 DROP SCHEMA full_access_user_schema CASCADE;
 NOTICE:  drop cascades to 4 other objects
 DETAIL:  drop cascades to table full_access_user_schema.t1

src/test/regress/expected/task_tracker_cleanup_job.out

@@ -101,3 +101,10 @@ SELECT isdir FROM pg_stat_file('base/pgsql_job_cache/job_401010/task_801107');
 ERROR:  could not stat file "base/pgsql_job_cache/job_401010/task_801107": No such file or directory
 SELECT isdir FROM pg_stat_file('base/pgsql_job_cache/job_401010');
 ERROR:  could not stat file "base/pgsql_job_cache/job_401010": No such file or directory
+-- Also clean up worker_cleanup_job_schema_cache job
+SELECT task_tracker_cleanup_job(2);
+ task_tracker_cleanup_job 
+--------------------------
+ 
+(1 row)
+

src/test/regress/expected/worker_remove_files.out

@@ -0,0 +1,8 @@
+-- Clear job directory used by previous tests
+\set JobId 201010
+SELECT task_tracker_cleanup_job(:JobId);
+ task_tracker_cleanup_job 
+--------------------------
+ 
+(1 row)
+

src/test/regress/failure_schedule

@@ -30,3 +30,6 @@ test: failure_savepoints
 test: failure_multi_row_insert
 test: failure_mx_metadata_sync
 test: failure_connection_establishment
+
+# test that no tests leaked intermediate results. This should always be last
+test: ensure_no_intermediate_data_leak

src/test/regress/multi_follower_schedule

@@ -3,3 +3,6 @@ test: multi_follower_select_statements
 test: multi_follower_dml
 test: multi_follower_configure_followers
 test: multi_follower_task_tracker
+
+# test that no tests leaked intermediate results. This should always be last
+test: ensure_no_intermediate_data_leak

src/test/regress/multi_mx_schedule

@@ -42,3 +42,6 @@ test: multi_mx_transaction_recovery
 test: multi_mx_modifying_xacts
 test: multi_mx_explain
 test: multi_mx_reference_table
+
+# test that no tests leaked intermediate results. This should always be last
+test: ensure_no_intermediate_data_leak

src/test/regress/multi_schedule

@@ -313,3 +313,8 @@ test: distributed_procedure
 # deparsing logic tests
 # ---------
 test: multi_deparse_function multi_deparse_procedure
+
+# ---------
+# test that no tests leaked intermediate results. This should always be last
+# ---------
+test: ensure_no_intermediate_data_leak

src/test/regress/multi_task_tracker_extra_schedule

@@ -110,3 +110,7 @@ test: multi_drop_extension
 # ----------
 test: multi_schema_support
 
+# ----------
+# test that no tests leaked intermediate results. This should always be last
+# ----------
+test: ensure_no_intermediate_data_leak

src/test/regress/sql/ensure_no_intermediate_data_leak.sql

@@ -1,12 +1,11 @@
 
 ------
 -- THIS TEST SHOULD IDEALLY BE EXECUTED AT THE END OF
--- THE REGRESSION TEST SUITE  TO MAKE SURE THAT WE
+-- THE REGRESSION TEST SUITE TO MAKE SURE THAT WE
 -- CLEAR ALL INTERMEDIATE RESULTS ON BOTH THE COORDINATOR
 -- AND ON THE WORKERS. HOWEVER, WE HAVE SOME ISSUES AROUND
--- WINDOWS SUPPORT, FAILURES IN TASK-TRACKER EXECUTOR
--- SO WE DISABLE THIS TEST ON WINDOWS
+-- WINDOWS SUPPORT SO WE DISABLE THIS TEST ON WINDOWS
 ------
 
 SELECT pg_ls_dir('base/pgsql_job_cache') WHERE citus_version() NOT ILIKE '%windows%';
-SELECT run_command_on_workers($$SELECT pg_ls_dir('base/pgsql_job_cache') WHERE citus_version() NOT ILIKE '%windows%'$$);
+SELECT * FROM run_command_on_workers($$SELECT pg_ls_dir('base/pgsql_job_cache') r WHERE citus_version() NOT ILIKE '%windows%'$$) WHERE result <> '';

src/test/regress/sql/multi_multiuser.sql

@@ -488,6 +488,7 @@ RESET ROLE;
 
 \c - - - :master_port
 
+SELECT run_command_on_workers($$SELECT task_tracker_cleanup_job(42);$$);
 
 DROP SCHEMA full_access_user_schema CASCADE;
 DROP TABLE

src/test/regress/sql/task_tracker_cleanup_job.sql

@@ -46,3 +46,6 @@ SELECT task_tracker_task_status(:JobId, :RunningTaskId);
 
 SELECT isdir FROM pg_stat_file('base/pgsql_job_cache/job_401010/task_801107');
 SELECT isdir FROM pg_stat_file('base/pgsql_job_cache/job_401010');
+
+-- Also clean up worker_cleanup_job_schema_cache job
+SELECT task_tracker_cleanup_job(2);

src/test/regress/sql/worker_remove_files.sql

@@ -0,0 +1,5 @@
+-- Clear job directory used by previous tests
+
+\set JobId 201010
+
+SELECT task_tracker_cleanup_job(:JobId);

src/test/regress/worker_schedule

@@ -19,6 +19,7 @@ test: worker_hash_partition worker_hash_partition_complex
 test: worker_merge_range_files worker_merge_hash_files
 test: worker_binary_data_partition worker_null_data_partition
 test: worker_check_invalid_arguments
+test: worker_remove_files
 
 # ----------
 # All task tracker tests use the following tables
@@ -26,3 +27,8 @@ test: worker_check_invalid_arguments
 test: task_tracker_create_table
 test: task_tracker_assign_task task_tracker_partition_task
 test: task_tracker_cleanup_job
+
+# ---------
+# test that no tests leaked intermediate results. This should always be last
+# ---------
+test: ensure_no_intermediate_data_leak