external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3423 from citusdata/remove-directory-even-if-new-files-added 

CitusRemoveDirectory: loop when directory is not empty
pull/3447/head
Philip Dubé 2020-01-30 20:21:47 +00:00 committed by GitHub
parent 6b43fab325 84a500ffc6
commit d7204c9696
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 81 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/backend/distributed/executor/intermediate_results.c
View File

@ -667,10 +667,7 @@ RemoveIntermediateResultsDirectory(void)
{
if (CreatedResultsDirectory)
{
StringInfo resultsDirectory = makeStringInfo();
appendStringInfoString(resultsDirectory, IntermediateResultsDirectory());
CitusRemoveDirectory(resultsDirectory);
CitusRemoveDirectory(IntermediateResultsDirectory());
CreatedResultsDirectory = false;
}

2
src/backend/distributed/executor/multi_server_executor.c
View File

@ -187,7 +187,7 @@ void
RemoveJobDirectory(uint64 jobId)
{
StringInfo jobDirectoryName = MasterJobDirectoryName(jobId);
CitusRemoveDirectory(jobDirectoryName);
CitusRemoveDirectory(jobDirectoryName->data);
ResourceOwnerForgetJobDirectory(CurrentResourceOwner, jobId);
}

2
src/backend/distributed/test/file_utils.c
View File

@ -23,7 +23,7 @@ citus_rm_job_directory(PG_FUNCTION_ARGS)
appendStringInfo(jobCacheDirectory, "base/%s/%s%0*" INT64_MODIFIER "u",
PG_JOB_CACHE_DIR, JOB_DIRECTORY_PREFIX,
MIN_JOB_DIRNAME_WIDTH, jobId);
CitusRemoveDirectory(jobCacheDirectory);
CitusRemoveDirectory(jobCacheDirectory->data);
FreeStringInfo(jobCacheDirectory);
PG_RETURN_VOID();

2
src/backend/distributed/worker/task_tracker.c
View File

@ -344,7 +344,7 @@ TrackerCleanupJobDirectories(void)
StringInfo jobCacheDirectory = makeStringInfo();
appendStringInfo(jobCacheDirectory, "base/%s", PG_JOB_CACHE_DIR);
CitusRemoveDirectory(jobCacheDirectory);
CitusRemoveDirectory(jobCacheDirectory->data);
CitusCreateDirectory(jobCacheDirectory);
FreeStringInfo(jobCacheDirectory);

2
src/backend/distributed/worker/task_tracker_protocol.c
View File

@ -231,7 +231,7 @@ task_tracker_cleanup_job(PG_FUNCTION_ARGS)
* writing to a table within the schema.
*/
StringInfo jobDirectoryName = JobDirectoryName(jobId);
CitusRemoveDirectory(jobDirectoryName);
CitusRemoveDirectory(jobDirectoryName->data);
RemoveJobSchema(jobSchemaName);
UnlockJobResource(jobId, AccessExclusiveLock);

2
src/backend/distributed/worker/worker_merge_protocol.c
View File

@ -97,7 +97,7 @@ worker_repartition_cleanup(PG_FUNCTION_ARGS)
Oid schemaId = get_namespace_oid(jobSchemaName->data, false);
EnsureSchemaOwner(schemaId);
CitusRemoveDirectory(jobDirectoryName);
CitusRemoveDirectory(jobDirectoryName->data);
RemoveJobSchema(jobSchemaName);
PG_RETURN_VOID();
}

136
src/backend/distributed/worker/worker_partition_protocol.c
View File

@ -85,7 +85,7 @@ static uint32 RangePartitionId(Datum partitionValue, Oid partitionCollation,
static uint32 HashPartitionId(Datum partitionValue, Oid partitionCollation,
const void *context);
static StringInfo UserPartitionFilename(StringInfo directoryName, uint32 partitionId);
static bool FileIsLink(char *filename, struct stat filestat);
static bool FileIsLink(const char *filename, struct stat filestat);
/* exports for SQL callable functions */
@ -158,7 +158,7 @@ worker_range_partition_table(PG_FUNCTION_ARGS)
/* close partition files and atomically rename (commit) them */
ClosePartitionFiles(partitionFileArray, fileCount);
CitusRemoveDirectory(taskDirectory);
CitusRemoveDirectory(taskDirectory->data);
RenameDirectory(taskAttemptDirectory, taskDirectory);
PG_RETURN_VOID();
@ -232,7 +232,7 @@ worker_hash_partition_table(PG_FUNCTION_ARGS)
/* close partition files and atomically rename (commit) them */
ClosePartitionFiles(partitionFileArray, fileCount);
CitusRemoveDirectory(taskDirectory);
CitusRemoveDirectory(taskDirectory->data);
RenameDirectory(taskAttemptDirectory, taskDirectory);
PG_RETURN_VOID();
@ -698,7 +698,7 @@ FileIsLink(char *filename, struct stat filestat)
#else
static bool
FileIsLink(char *filename, struct stat filestat)
FileIsLink(const char *filename, struct stat filestat)
{
return S_ISLNK(filestat.st_mode);
}
@ -714,79 +714,91 @@ FileIsLink(char *filename, struct stat filestat)
* system library's remove_all() method.
*/
void
CitusRemoveDirectory(StringInfo filename)
CitusRemoveDirectory(const char *filename)
{
struct stat fileStat;
int removed = 0;
int fileStated = stat(filename->data, &fileStat);
if (fileStated < 0)
/* files may be added during execution, loop when that occurs */
while (true)
{
if (errno == ENOENT)
{
return; /* if file does not exist, return */
}
else
{
ereport(ERROR, (errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m", filename->data)));
}
}
struct stat fileStat;
int removed = 0;
/*
* If this is a directory, iterate over all its contents and for each
* content, recurse into this function. Also, make sure that we do not
* recurse into symbolic links.
*/
if (S_ISDIR(fileStat.st_mode) && !FileIsLink(filename->data, fileStat))
{
const char *directoryName = filename->data;
DIR *directory = AllocateDir(directoryName);
if (directory == NULL)
int fileStated = stat(filename, &fileStat);
if (fileStated < 0)
{
ereport(ERROR, (errcode_for_file_access(),
errmsg("could not open directory \"%s\": %m",
directoryName)));
}
struct dirent *directoryEntry = ReadDir(directory, directoryName);
for (; directoryEntry != NULL; directoryEntry = ReadDir(directory, directoryName))
{
const char *baseFilename = directoryEntry->d_name;
/* if system file, skip it */
if (strncmp(baseFilename, ".", MAXPGPATH) == 0 ||
strncmp(baseFilename, "..", MAXPGPATH) == 0)
if (errno == ENOENT)
{
continue;
return; /* if file does not exist, return */
}
else
{
ereport(ERROR, (errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m", filename)));
}
}
/*
* If this is a directory, iterate over all its contents and for each
* content, recurse into this function. Also, make sure that we do not
* recurse into symbolic links.
*/
if (S_ISDIR(fileStat.st_mode) && !FileIsLink(filename, fileStat))
{
const char *directoryName = filename;
DIR *directory = AllocateDir(directoryName);
if (directory == NULL)
{
ereport(ERROR, (errcode_for_file_access(),
errmsg("could not open directory \"%s\": %m",
directoryName)));
}
StringInfo fullFilename = makeStringInfo();
appendStringInfo(fullFilename, "%s/%s", directoryName, baseFilename);
struct dirent *directoryEntry = ReadDir(directory, directoryName);
for (; directoryEntry != NULL; directoryEntry = ReadDir(directory,
directoryName))
{
const char *baseFilename = directoryEntry->d_name;
CitusRemoveDirectory(fullFilename);
/* if system file, skip it */
if (strncmp(baseFilename, ".", MAXPGPATH) == 0 ||
strncmp(baseFilename, "..", MAXPGPATH) == 0)
{
continue;
}
resetStringInfo(fullFilename);
appendStringInfo(fullFilename, "%s/%s", directoryName, baseFilename);
CitusRemoveDirectory(fullFilename->data);
}
FreeStringInfo(fullFilename);
FreeDir(directory);
}
FreeDir(directory);
}
/* we now have an empty directory or a regular file, remove it */
if (S_ISDIR(fileStat.st_mode))
{
removed = rmdir(filename);
/* we now have an empty directory or a regular file, remove it */
if (S_ISDIR(fileStat.st_mode))
{
removed = rmdir(filename->data);
}
else
{
removed = unlink(filename->data);
}
if (errno == ENOTEMPTY || errno == EEXIST)
{
continue;
}
}
else
{
removed = unlink(filename);
}
if (removed != 0 && errno != ENOENT)
{
ereport(ERROR, (errcode_for_file_access(),
errmsg("could not remove file \"%s\": %m", filename->data)));
if (removed != 0 && errno != ENOENT)
{
ereport(ERROR, (errcode_for_file_access(),
errmsg("could not remove file \"%s\": %m", filename)));
}
return;
}
}

2
src/include/distributed/worker_protocol.h
View File

@ -117,7 +117,7 @@ extern bool CacheDirectoryElement(const char *filename);
extern bool JobDirectoryElement(const char *filename);
extern bool DirectoryExists(StringInfo directoryName);
extern void CitusCreateDirectory(StringInfo directoryName);
extern void CitusRemoveDirectory(StringInfo filename);
extern void CitusRemoveDirectory(const char *filename);
extern StringInfo InitTaskDirectory(uint64 jobId, uint32 taskId);
extern void RemoveJobSchema(StringInfo schemaName);
extern Datum * DeconstructArrayObject(ArrayType *arrayObject);