fix(jobs): claim only handled kinds and requeue numbered workers after a crash

An engine with no handlers (or a filter it could not serve) claimed every
job, so a host without a render handler would fail render-node work.
With several workers, jobs carry node ids such as pc#1, but crash
recovery only matched the bare node id and left them running forever.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 12:49:15 +00:00
1 parent b4f61e29b9
commit 6ac27413a7
3 files changed
+52 -5

No files matched your search

+9 -2
View File
@@ -34,6 +34,7 @@ public sealed class JobEngine : IAsyncDisposable
{
private readonly TitanoDatabase _db;
private readonly Dictionary<JobKind, IJobHandler> _handlers;
private IReadOnlyCollection<JobKind> _kinds = [];
private readonly ConcurrentDictionary<Guid, CancellationTokenSource> _running = new();
private readonly Channel<bool> _wake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
private readonly CancellationTokenSource _shutdown = new();
@@ -62,6 +63,13 @@ public sealed class JobEngine : IAsyncDisposable
var requeued = await _db.Jobs.RequeueRunningAsync(Options.WorkerId, cancellationToken).ConfigureAwait(false);
if (requeued > 0) Log.Info("jobs", $"Requeued {requeued} interrupted job(s)");
await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false);
// Only kinds this engine can run: an empty filter would claim (and fail) every job, render-node work included.
_kinds = [.. (Options.Kinds ?? _handlers.Keys).Where(_handlers.ContainsKey).Distinct()];
if (_kinds.Count == 0)
{
Log.Info("jobs", "No job handlers; this engine claims nothing");
return;
}
for (var i = 0; i < Options.Workers; i++)
{
var id = Options.Workers == 1 ? Options.WorkerId : $"{Options.WorkerId}#{i}";
@@ -161,13 +169,12 @@ public sealed class JobEngine : IAsyncDisposable
private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown)
{
var kinds = Options.Kinds ?? [.. _handlers.Keys];
while (!shutdown.IsCancellationRequested)
{
Job? job;
try
{
job = await _db.Jobs.ClaimNextAsync(workerId, kinds, shutdown).ConfigureAwait(false);
job = await _db.Jobs.ClaimNextAsync(workerId, _kinds, shutdown).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
@@ -108,13 +108,14 @@ public sealed class JobRepository
/// <summary>
/// Crash recovery: puts running jobs back in the queue, keeping their checkpoints. With <paramref name="nodeId"/>
/// only that worker's jobs are requeued (other server instances keep theirs).
/// only that node's jobs are requeued, including those of its numbered workers (<c>node#0</c>, <c>node#1</c>, …);
/// other server instances keep theirs.
/// </summary>
public Task<int> RequeueRunningAsync(string? nodeId = null, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync(
$"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE state = {(int)JobState.Running}" +
(nodeId is null ? string.Empty : " AND node_id = $1"),
nodeId is null ? [] : [nodeId], cancellationToken);
(nodeId is null ? string.Empty : " AND (node_id = $1 OR substr(node_id, 1, length($2)) = $2)"),
nodeId is null ? [] : [nodeId, nodeId + "#"], cancellationToken);
private static Job Map(DbRow r) => new()
{
@@ -120,6 +120,45 @@ public sealed class JobEngineTests(TestContext context)
Assert.Equal("7", handler.SeenCheckpoints.Single());
}
[Test]
public async Task JobsOfNumberedWorkersResumeAfterRestart()
{
await using var db = await OpenAsync();
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
await db.Jobs.EnqueueAsync(job);
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc#1"));
var other = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
await db.Jobs.EnqueueAsync(other);
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc2#0"));
// Restarting node "pc" (two workers: pc#0, pc#1) requeues pc#1's job, never another node's.
var handler = new CountingHandler(JobKind.Thumbs, 3);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "pc" });
await engine.StartAsync();
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(other.Id))!.State);
}
[Test]
public async Task OnlyKindsWithAHandlerAreClaimed()
{
await using var db = await OpenAsync();
await using (var idle = new JobEngine(db, [], new JobEngineOptions { Workers = 1, WorkerId = "idle" }))
{
await idle.StartAsync();
var render = await idle.EnqueueAsync(JobKind.Render, []);
await Task.Delay(300);
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(render.Id))!.State);
}
var handler = new CountingHandler(JobKind.Thumbs, 1);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Kinds = [JobKind.Thumbs, JobKind.Render] });
await engine.StartAsync();
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
var pending = (await db.Jobs.ListAsync(JobState.Queued)).Items;
Assert.Count(1, pending);
Assert.Equal(JobKind.Render, pending[0].Kind);
}
[Test]
public async Task HandlerFailureMarksJobFailedWithMessage()
{