fix(jobs): a job a handler cannot judge or finish never ends a worker
Accepts that throws now claims the job so it fails with its reason; claim and finish errors are logged and the worker carries on. Other kinds keep their atomic claim next to selective ones, and a lost race retries at once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
2 files changed
+79
-15
No files matched your search
@@ -36,10 +36,14 @@ public sealed class JobEngine : IAsyncDisposable
|
||||
/// <summary>Queued jobs of selective kinds examined per claim; jobs declined beyond it wait until those ahead leave the queue.</summary>
|
||||
private const int SelectiveWindow = 100;
|
||||
|
||||
/// <summary>Claims lost to other workers before a worker waits for the next poll.</summary>
|
||||
private const int MaxClaimRaces = 16;
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly Dictionary<JobKind, IJobHandler> _handlers;
|
||||
private IReadOnlyCollection<JobKind> _kinds = [];
|
||||
private Dictionary<JobKind, ISelectiveJobHandler> _selective = [];
|
||||
private IReadOnlyCollection<JobKind> _others = [];
|
||||
private readonly ConcurrentDictionary<Guid, CancellationTokenSource> _running = new();
|
||||
private readonly Channel<bool> _wake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
|
||||
private readonly CancellationTokenSource _shutdown = new();
|
||||
@@ -71,6 +75,7 @@ public sealed class JobEngine : IAsyncDisposable
|
||||
// Only kinds this engine can run: an empty filter would claim (and fail) every job, render-node work included.
|
||||
_kinds = [.. (Options.Kinds ?? _handlers.Keys).Where(_handlers.ContainsKey).Distinct()];
|
||||
_selective = _kinds.Select(k => _handlers[k]).OfType<ISelectiveJobHandler>().ToDictionary(h => h.Kind);
|
||||
_others = [.. _kinds.Where(k => !_selective.ContainsKey(k))];
|
||||
if (_kinds.Count == 0)
|
||||
{
|
||||
Log.Info("jobs", "No job handlers; this engine claims nothing");
|
||||
@@ -176,6 +181,7 @@ public sealed class JobEngine : IAsyncDisposable
|
||||
if (await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } job) JobChanged?.Invoke(job);
|
||||
}
|
||||
|
||||
/// <summary>Claims and runs jobs until shutdown. Nothing a job or a handler throws ends the worker.</summary>
|
||||
private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown)
|
||||
{
|
||||
while (!shutdown.IsCancellationRequested)
|
||||
@@ -185,11 +191,11 @@ public sealed class JobEngine : IAsyncDisposable
|
||||
{
|
||||
job = await ClaimAsync(workerId, shutdown).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
catch (OperationCanceledException) when (shutdown.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (DatabaseException ex)
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning("jobs", "Claiming a job failed", ex);
|
||||
job = null;
|
||||
@@ -210,35 +216,64 @@ public sealed class JobEngine : IAsyncDisposable
|
||||
continue;
|
||||
}
|
||||
|
||||
await RunAsync(job, shutdown).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await RunAsync(job, shutdown).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (!shutdown.IsCancellationRequested)
|
||||
{
|
||||
// The job's final state could not be stored; it stays running and is requeued at the next start.
|
||||
Log.Error("jobs", $"{job.Kind} job {job.Id} could not be finished", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The next job in claim order (priority, then age). Without selective handlers the database picks it atomically;
|
||||
/// with them, the head of the other kinds competes with the first job a selective handler accepts among the first
|
||||
/// <see cref="SelectiveWindow"/> of its kinds, and the winner is claimed if it is still queued.
|
||||
/// The next job in claim order (priority, then age). Without selective handlers the database picks it atomically.
|
||||
/// With them, the first job a selective handler accepts among the first <see cref="SelectiveWindow"/> of its kinds
|
||||
/// competes with the head of the other kinds: the other kinds are still claimed atomically, the selective job by id.
|
||||
/// A lost race means another worker took a job, so looking again at once makes progress.
|
||||
/// </summary>
|
||||
private async Task<Job?> ClaimAsync(string workerId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_selective.Count == 0) return await _db.Jobs.ClaimNextAsync(workerId, _kinds, cancellationToken).ConfigureAwait(false);
|
||||
var others = _kinds.Where(k => !_selective.ContainsKey(k)).ToList();
|
||||
for (var attempt = 0; attempt < 3; attempt++)
|
||||
for (var attempt = 0; attempt < MaxClaimRaces; attempt++)
|
||||
{
|
||||
var best = others.Count > 0 ? (await _db.Jobs.ListQueuedAsync(others, 1, cancellationToken).ConfigureAwait(false)).FirstOrDefault() : null;
|
||||
foreach (var job in await _db.Jobs.ListQueuedAsync(_selective.Keys, SelectiveWindow, cancellationToken).ConfigureAwait(false))
|
||||
Job? candidate = null;
|
||||
foreach (var queued in await _db.Jobs.ListQueuedAsync(_selective.Keys, SelectiveWindow, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!_selective[job.Kind].Accepts(job)) continue;
|
||||
if (best is null || ClaimsBefore(job, best)) best = job;
|
||||
if (!Accepts(queued)) continue;
|
||||
candidate = queued;
|
||||
break;
|
||||
}
|
||||
if (best is null) return null;
|
||||
// Another claimer may have been faster: look again.
|
||||
if (await _db.Jobs.TryClaimAsync(best.Id, workerId, cancellationToken).ConfigureAwait(false) is { } claimed) return claimed;
|
||||
if (candidate is null) return _others.Count > 0 ? await _db.Jobs.ClaimNextAsync(workerId, _others, cancellationToken).ConfigureAwait(false) : null;
|
||||
if (_others.Count > 0 && (await _db.Jobs.ListQueuedAsync(_others, 1, cancellationToken).ConfigureAwait(false)).FirstOrDefault() is { } head
|
||||
&& ClaimsBefore(head, candidate))
|
||||
{
|
||||
if (await _db.Jobs.ClaimNextAsync(workerId, _others, cancellationToken).ConfigureAwait(false) is { } next) return next;
|
||||
}
|
||||
else if (await _db.Jobs.TryClaimAsync(candidate.Id, workerId, cancellationToken).ConfigureAwait(false) is { } claimed)
|
||||
{
|
||||
return claimed;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>A job its handler cannot judge is taken, so it fails here with its reason instead of blocking the queue.</summary>
|
||||
private bool Accepts(Job job)
|
||||
{
|
||||
try
|
||||
{
|
||||
return _selective[job.Kind].Accepts(job);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning("jobs", $"{job.Kind} job {job.Id} could not be examined; it is claimed so that it fails with its reason", ex);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool ClaimsBefore(Job a, Job b) =>
|
||||
a.Priority != b.Priority ? a.Priority > b.Priority : a.Created != b.Created ? a.Created < b.Created : a.Id.CompareTo(b.Id) < 0;
|
||||
|
||||
|
||||
@@ -39,6 +39,17 @@ public sealed class JobEngineTests(TestContext context)
|
||||
public Task RunAsync(JobContext c) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Cannot judge jobs without parameters (as a render handler with a malformed preset) and fails them when run.</summary>
|
||||
private sealed class FragileHandler(JobKind kind) : ISelectiveJobHandler
|
||||
{
|
||||
public JobKind Kind => kind;
|
||||
|
||||
public bool Accepts(Job job) => job.ParametersJson?.Length > 0 ? true : throw new ArgumentNullException(nameof(job));
|
||||
|
||||
public Task RunAsync(JobContext c) =>
|
||||
c.Job.ParametersJson is null ? throw new InvalidDataException("no parameters") : Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => kind;
|
||||
@@ -197,6 +208,24 @@ public sealed class JobEngineTests(TestContext context)
|
||||
Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ASelectiveHandlerThatThrowsDoesNotStopTheWorkers()
|
||||
{
|
||||
await using var db = await OpenAsync();
|
||||
await using var engine = new JobEngine(db, [new FragileHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
|
||||
new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
||||
var broken = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
|
||||
await engine.StartAsync();
|
||||
|
||||
var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed, TimeSpan.FromSeconds(10));
|
||||
Assert.Equal("no parameters", failed.Error);
|
||||
// The workers are still there for every kind.
|
||||
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
|
||||
var render = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: "{}");
|
||||
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
||||
await WaitForStateAsync(db, render.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user