fix(jobs): a job a handler cannot judge or finish never ends a worker

Accepts that throws now claims the job so it fails with its reason; claim
and finish errors are logged and the worker carries on. Other kinds keep
their atomic claim next to selective ones, and a lost race retries at once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 22:59:00 +00:00
1 parent 1cd94af0f0
commit 04a40ea22a
2 files changed
+79 -15

No files matched your search

+50 -15
View File
@@ -36,10 +36,14 @@ public sealed class JobEngine : IAsyncDisposable
/// <summary>Queued jobs of selective kinds examined per claim; jobs declined beyond it wait until those ahead leave the queue.</summary>
private const int SelectiveWindow = 100;
/// <summary>Claims lost to other workers before a worker waits for the next poll.</summary>
private const int MaxClaimRaces = 16;
private readonly TitanoDatabase _db;
private readonly Dictionary<JobKind, IJobHandler> _handlers;
private IReadOnlyCollection<JobKind> _kinds = [];
private Dictionary<JobKind, ISelectiveJobHandler> _selective = [];
private IReadOnlyCollection<JobKind> _others = [];
private readonly ConcurrentDictionary<Guid, CancellationTokenSource> _running = new();
private readonly Channel<bool> _wake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
private readonly CancellationTokenSource _shutdown = new();
@@ -71,6 +75,7 @@ public sealed class JobEngine : IAsyncDisposable
// Only kinds this engine can run: an empty filter would claim (and fail) every job, render-node work included.
_kinds = [.. (Options.Kinds ?? _handlers.Keys).Where(_handlers.ContainsKey).Distinct()];
_selective = _kinds.Select(k => _handlers[k]).OfType<ISelectiveJobHandler>().ToDictionary(h => h.Kind);
_others = [.. _kinds.Where(k => !_selective.ContainsKey(k))];
if (_kinds.Count == 0)
{
Log.Info("jobs", "No job handlers; this engine claims nothing");
@@ -176,6 +181,7 @@ public sealed class JobEngine : IAsyncDisposable
if (await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } job) JobChanged?.Invoke(job);
}
/// <summary>Claims and runs jobs until shutdown. Nothing a job or a handler throws ends the worker.</summary>
private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown)
{
while (!shutdown.IsCancellationRequested)
@@ -185,11 +191,11 @@ public sealed class JobEngine : IAsyncDisposable
{
job = await ClaimAsync(workerId, shutdown).ConfigureAwait(false);
}
catch (OperationCanceledException)
catch (OperationCanceledException) when (shutdown.IsCancellationRequested)
{
return;
}
catch (DatabaseException ex)
catch (Exception ex)
{
Log.Warning("jobs", "Claiming a job failed", ex);
job = null;
@@ -210,35 +216,64 @@ public sealed class JobEngine : IAsyncDisposable
continue;
}
await RunAsync(job, shutdown).ConfigureAwait(false);
try
{
await RunAsync(job, shutdown).ConfigureAwait(false);
}
catch (Exception ex) when (!shutdown.IsCancellationRequested)
{
// The job's final state could not be stored; it stays running and is requeued at the next start.
Log.Error("jobs", $"{job.Kind} job {job.Id} could not be finished", ex);
}
}
}
/// <summary>
/// The next job in claim order (priority, then age). Without selective handlers the database picks it atomically;
/// with them, the head of the other kinds competes with the first job a selective handler accepts among the first
/// <see cref="SelectiveWindow"/> of its kinds, and the winner is claimed if it is still queued.
/// The next job in claim order (priority, then age). Without selective handlers the database picks it atomically.
/// With them, the first job a selective handler accepts among the first <see cref="SelectiveWindow"/> of its kinds
/// competes with the head of the other kinds: the other kinds are still claimed atomically, the selective job by id.
/// A lost race means another worker took a job, so looking again at once makes progress.
/// </summary>
private async Task<Job?> ClaimAsync(string workerId, CancellationToken cancellationToken)
{
if (_selective.Count == 0) return await _db.Jobs.ClaimNextAsync(workerId, _kinds, cancellationToken).ConfigureAwait(false);
var others = _kinds.Where(k => !_selective.ContainsKey(k)).ToList();
for (var attempt = 0; attempt < 3; attempt++)
for (var attempt = 0; attempt < MaxClaimRaces; attempt++)
{
var best = others.Count > 0 ? (await _db.Jobs.ListQueuedAsync(others, 1, cancellationToken).ConfigureAwait(false)).FirstOrDefault() : null;
foreach (var job in await _db.Jobs.ListQueuedAsync(_selective.Keys, SelectiveWindow, cancellationToken).ConfigureAwait(false))
Job? candidate = null;
foreach (var queued in await _db.Jobs.ListQueuedAsync(_selective.Keys, SelectiveWindow, cancellationToken).ConfigureAwait(false))
{
if (!_selective[job.Kind].Accepts(job)) continue;
if (best is null || ClaimsBefore(job, best)) best = job;
if (!Accepts(queued)) continue;
candidate = queued;
break;
}
if (best is null) return null;
// Another claimer may have been faster: look again.
if (await _db.Jobs.TryClaimAsync(best.Id, workerId, cancellationToken).ConfigureAwait(false) is { } claimed) return claimed;
if (candidate is null) return _others.Count > 0 ? await _db.Jobs.ClaimNextAsync(workerId, _others, cancellationToken).ConfigureAwait(false) : null;
if (_others.Count > 0 && (await _db.Jobs.ListQueuedAsync(_others, 1, cancellationToken).ConfigureAwait(false)).FirstOrDefault() is { } head
&& ClaimsBefore(head, candidate))
{
if (await _db.Jobs.ClaimNextAsync(workerId, _others, cancellationToken).ConfigureAwait(false) is { } next) return next;
}
else if (await _db.Jobs.TryClaimAsync(candidate.Id, workerId, cancellationToken).ConfigureAwait(false) is { } claimed)
{
return claimed;
}
}
return null;
}
/// <summary>A job its handler cannot judge is taken, so it fails here with its reason instead of blocking the queue.</summary>
private bool Accepts(Job job)
{
try
{
return _selective[job.Kind].Accepts(job);
}
catch (Exception ex)
{
Log.Warning("jobs", $"{job.Kind} job {job.Id} could not be examined; it is claimed so that it fails with its reason", ex);
return true;
}
}
private static bool ClaimsBefore(Job a, Job b) =>
a.Priority != b.Priority ? a.Priority > b.Priority : a.Created != b.Created ? a.Created < b.Created : a.Id.CompareTo(b.Id) < 0;
@@ -39,6 +39,17 @@ public sealed class JobEngineTests(TestContext context)
public Task RunAsync(JobContext c) => Task.CompletedTask;
}
/// <summary>Cannot judge jobs without parameters (as a render handler with a malformed preset) and fails them when run.</summary>
private sealed class FragileHandler(JobKind kind) : ISelectiveJobHandler
{
public JobKind Kind => kind;
public bool Accepts(Job job) => job.ParametersJson?.Length > 0 ? true : throw new ArgumentNullException(nameof(job));
public Task RunAsync(JobContext c) =>
c.Job.ParametersJson is null ? throw new InvalidDataException("no parameters") : Task.CompletedTask;
}
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
{
public JobKind Kind => kind;
@@ -197,6 +208,24 @@ public sealed class JobEngineTests(TestContext context)
Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId));
}
[Test]
public async Task ASelectiveHandlerThatThrowsDoesNotStopTheWorkers()
{
await using var db = await OpenAsync();
await using var engine = new JobEngine(db, [new FragileHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
var broken = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
await engine.StartAsync();
var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed, TimeSpan.FromSeconds(10));
Assert.Equal("no parameters", failed.Error);
// The workers are still there for every kind.
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
var render = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: "{}");
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
await WaitForStateAsync(db, render.Id, JobState.Done, TimeSpan.FromSeconds(10));
}
[Test]
public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns()
{