diff --git a/src/Titano.Data/Jobs/JobEngine.cs b/src/Titano.Data/Jobs/JobEngine.cs
index 815ccc12..d6296ebc 100644
--- a/src/Titano.Data/Jobs/JobEngine.cs
+++ b/src/Titano.Data/Jobs/JobEngine.cs
@@ -36,10 +36,14 @@ public sealed class JobEngine : IAsyncDisposable
/// Queued jobs of selective kinds examined per claim; jobs declined beyond it wait until those ahead leave the queue.
private const int SelectiveWindow = 100;
+ /// Claims lost to other workers before a worker waits for the next poll.
+ private const int MaxClaimRaces = 16;
+
private readonly TitanoDatabase _db;
private readonly Dictionary _handlers;
private IReadOnlyCollection _kinds = [];
private Dictionary _selective = [];
+ private IReadOnlyCollection _others = [];
private readonly ConcurrentDictionary _running = new();
private readonly Channel _wake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
private readonly CancellationTokenSource _shutdown = new();
@@ -71,6 +75,7 @@ public sealed class JobEngine : IAsyncDisposable
// Only kinds this engine can run: an empty filter would claim (and fail) every job, render-node work included.
_kinds = [.. (Options.Kinds ?? _handlers.Keys).Where(_handlers.ContainsKey).Distinct()];
_selective = _kinds.Select(k => _handlers[k]).OfType().ToDictionary(h => h.Kind);
+ _others = [.. _kinds.Where(k => !_selective.ContainsKey(k))];
if (_kinds.Count == 0)
{
Log.Info("jobs", "No job handlers; this engine claims nothing");
@@ -176,6 +181,7 @@ public sealed class JobEngine : IAsyncDisposable
if (await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } job) JobChanged?.Invoke(job);
}
+ /// Claims and runs jobs until shutdown. Nothing a job or a handler throws ends the worker.
private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown)
{
while (!shutdown.IsCancellationRequested)
@@ -185,11 +191,11 @@ public sealed class JobEngine : IAsyncDisposable
{
job = await ClaimAsync(workerId, shutdown).ConfigureAwait(false);
}
- catch (OperationCanceledException)
+ catch (OperationCanceledException) when (shutdown.IsCancellationRequested)
{
return;
}
- catch (DatabaseException ex)
+ catch (Exception ex)
{
Log.Warning("jobs", "Claiming a job failed", ex);
job = null;
@@ -210,35 +216,64 @@ public sealed class JobEngine : IAsyncDisposable
continue;
}
- await RunAsync(job, shutdown).ConfigureAwait(false);
+ try
+ {
+ await RunAsync(job, shutdown).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (!shutdown.IsCancellationRequested)
+ {
+ // The job's final state could not be stored; it stays running and is requeued at the next start.
+ Log.Error("jobs", $"{job.Kind} job {job.Id} could not be finished", ex);
+ }
}
}
///
- /// The next job in claim order (priority, then age). Without selective handlers the database picks it atomically;
- /// with them, the head of the other kinds competes with the first job a selective handler accepts among the first
- /// of its kinds, and the winner is claimed if it is still queued.
+ /// The next job in claim order (priority, then age). Without selective handlers the database picks it atomically.
+ /// With them, the first job a selective handler accepts among the first of its kinds
+ /// competes with the head of the other kinds: the other kinds are still claimed atomically, the selective job by id.
+ /// A lost race means another worker took a job, so looking again at once makes progress.
///
private async Task ClaimAsync(string workerId, CancellationToken cancellationToken)
{
if (_selective.Count == 0) return await _db.Jobs.ClaimNextAsync(workerId, _kinds, cancellationToken).ConfigureAwait(false);
- var others = _kinds.Where(k => !_selective.ContainsKey(k)).ToList();
- for (var attempt = 0; attempt < 3; attempt++)
+ for (var attempt = 0; attempt < MaxClaimRaces; attempt++)
{
- var best = others.Count > 0 ? (await _db.Jobs.ListQueuedAsync(others, 1, cancellationToken).ConfigureAwait(false)).FirstOrDefault() : null;
- foreach (var job in await _db.Jobs.ListQueuedAsync(_selective.Keys, SelectiveWindow, cancellationToken).ConfigureAwait(false))
+ Job? candidate = null;
+ foreach (var queued in await _db.Jobs.ListQueuedAsync(_selective.Keys, SelectiveWindow, cancellationToken).ConfigureAwait(false))
{
- if (!_selective[job.Kind].Accepts(job)) continue;
- if (best is null || ClaimsBefore(job, best)) best = job;
+ if (!Accepts(queued)) continue;
+ candidate = queued;
break;
}
- if (best is null) return null;
- // Another claimer may have been faster: look again.
- if (await _db.Jobs.TryClaimAsync(best.Id, workerId, cancellationToken).ConfigureAwait(false) is { } claimed) return claimed;
+ if (candidate is null) return _others.Count > 0 ? await _db.Jobs.ClaimNextAsync(workerId, _others, cancellationToken).ConfigureAwait(false) : null;
+ if (_others.Count > 0 && (await _db.Jobs.ListQueuedAsync(_others, 1, cancellationToken).ConfigureAwait(false)).FirstOrDefault() is { } head
+ && ClaimsBefore(head, candidate))
+ {
+ if (await _db.Jobs.ClaimNextAsync(workerId, _others, cancellationToken).ConfigureAwait(false) is { } next) return next;
+ }
+ else if (await _db.Jobs.TryClaimAsync(candidate.Id, workerId, cancellationToken).ConfigureAwait(false) is { } claimed)
+ {
+ return claimed;
+ }
}
return null;
}
+ /// A job its handler cannot judge is taken, so it fails here with its reason instead of blocking the queue.
+ private bool Accepts(Job job)
+ {
+ try
+ {
+ return _selective[job.Kind].Accepts(job);
+ }
+ catch (Exception ex)
+ {
+ Log.Warning("jobs", $"{job.Kind} job {job.Id} could not be examined; it is claimed so that it fails with its reason", ex);
+ return true;
+ }
+ }
+
private static bool ClaimsBefore(Job a, Job b) =>
a.Priority != b.Priority ? a.Priority > b.Priority : a.Created != b.Created ? a.Created < b.Created : a.Id.CompareTo(b.Id) < 0;
diff --git a/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs
index df22bc0b..283648b6 100644
--- a/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs
+++ b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs
@@ -39,6 +39,17 @@ public sealed class JobEngineTests(TestContext context)
public Task RunAsync(JobContext c) => Task.CompletedTask;
}
+ /// Cannot judge jobs without parameters (as a render handler with a malformed preset) and fails them when run.
+ private sealed class FragileHandler(JobKind kind) : ISelectiveJobHandler
+ {
+ public JobKind Kind => kind;
+
+ public bool Accepts(Job job) => job.ParametersJson?.Length > 0 ? true : throw new ArgumentNullException(nameof(job));
+
+ public Task RunAsync(JobContext c) =>
+ c.Job.ParametersJson is null ? throw new InvalidDataException("no parameters") : Task.CompletedTask;
+ }
+
private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler
{
public JobKind Kind => kind;
@@ -197,6 +208,24 @@ public sealed class JobEngineTests(TestContext context)
Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId));
}
+ [Test]
+ public async Task ASelectiveHandlerThatThrowsDoesNotStopTheWorkers()
+ {
+ await using var db = await OpenAsync();
+ await using var engine = new JobEngine(db, [new FragileHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
+ new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
+ var broken = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
+ await engine.StartAsync();
+
+ var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed, TimeSpan.FromSeconds(10));
+ Assert.Equal("no parameters", failed.Error);
+ // The workers are still there for every kind.
+ var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
+ var render = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: "{}");
+ await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
+ await WaitForStateAsync(db, render.Id, JobState.Done, TimeSpan.FromSeconds(10));
+ }
+
[Test]
public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns()
{