From 3db49b43c20c16f074b28e46d7b7c871ca965ed9 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Tue, 9 Aug 2022 22:38:42 +0200 Subject: [PATCH] add lock for background task monitor to make sure there is only ever one running per database at the same time --- .../distributed/utils/background_jobs.c | 21 +++++++++++++++++-- src/include/distributed/resource_lock.h | 3 ++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/utils/background_jobs.c b/src/backend/distributed/utils/background_jobs.c index 49dbfb280..c319577ca 100644 --- a/src/backend/distributed/utils/background_jobs.c +++ b/src/backend/distributed/utils/background_jobs.c @@ -28,6 +28,7 @@ #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/shard_cleaner.h" +#include "distributed/resource_lock.h" bool BackgroundTaskMonitorDebugDelay = false; @@ -120,13 +121,30 @@ CitusBackgroundTaskMonitorMain(Datum arg) /* connect to database, after that we can actually access catalogs */ BackgroundWorkerInitializeConnectionByOid(databaseOid, extensionOwner, 0); - /* TODO get lock to make sure there is only one worker running per databasse */ + /* + * There should be exactly one background task monitor running, running multiple would + * cause conflicts on processing the tasks in the catalog table as well as violate + * parallelism guarantees. To make sure there is at most, exactly one backend running + * we take a session lock on the CITUS_BACKGROUND_TASK_MONITOR operation. + */ + LOCKTAG tag = { 0 }; + SET_LOCKTAG_CITUS_OPERATION(tag, CITUS_BACKGROUND_TASK_MONITOR); + const bool sessionLock = true; + const bool dontWait = true; + LockAcquireResult locked = + LockAcquire(&tag, AccessExclusiveLock, sessionLock, dontWait); + if (locked == LOCKACQUIRE_NOT_AVAIL) + { + ereport(ERROR, (errmsg("background task monitor already running for database"))); + exit(0); + } /* make worker recognizable in pg_stat_activity */ pgstat_report_appname("citus background task monitor"); ereport(LOG, (errmsg("citus background task monitor"))); + /* TODO this is here for debugging purposses, remove before merge. */ if (BackgroundTaskMonitorDebugDelay) { pg_usleep(30 * 1000 * 1000); @@ -147,7 +165,6 @@ CitusBackgroundTaskMonitorMain(Datum arg) StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); - /* TODO have an actual function to check if the worker is still running */ ResetRunningBackgroundTasks(); PopActiveSnapshot(); diff --git a/src/include/distributed/resource_lock.h b/src/include/distributed/resource_lock.h index c808e9157..dc1fb4e96 100644 --- a/src/include/distributed/resource_lock.h +++ b/src/include/distributed/resource_lock.h @@ -48,7 +48,8 @@ typedef enum AdvisoryLocktagClass /* CitusOperations has constants for citus operations */ typedef enum CitusOperations { - CITUS_TRANSACTION_RECOVERY = 0 + CITUS_TRANSACTION_RECOVERY = 0, + CITUS_BACKGROUND_TASK_MONITOR = 1 } CitusOperations; /* reuse advisory lock, but with different, unused field 4 (4)*/