From 6ac27413a782b83fc751a6e8cc6060ac87e07d04 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 12:49:15 +0000 Subject: [PATCH] fix(jobs): claim only handled kinds and requeue numbered workers after a crash An engine with no handlers (or a filter it could not serve) claimed every job, so a host without a render handler would fail render-node work. With several workers, jobs carry node ids such as pc#1, but crash recovery only matched the bare node id and left them running forever. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Jobs/JobEngine.cs | 11 +++++- src/Titano.Data/Repositories/JobRepository.cs | 7 ++-- .../Titano.Data.Tests/Jobs/JobEngineTests.cs | 39 +++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/Titano.Data/Jobs/JobEngine.cs b/src/Titano.Data/Jobs/JobEngine.cs index b2b279f4..e6229cba 100644 --- a/src/Titano.Data/Jobs/JobEngine.cs +++ b/src/Titano.Data/Jobs/JobEngine.cs @@ -34,6 +34,7 @@ public sealed class JobEngine : IAsyncDisposable { private readonly TitanoDatabase _db; private readonly Dictionary _handlers; + private IReadOnlyCollection _kinds = []; private readonly ConcurrentDictionary _running = new(); private readonly Channel _wake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite }); private readonly CancellationTokenSource _shutdown = new(); @@ -62,6 +63,13 @@ public sealed class JobEngine : IAsyncDisposable var requeued = await _db.Jobs.RequeueRunningAsync(Options.WorkerId, cancellationToken).ConfigureAwait(false); if (requeued > 0) Log.Info("jobs", $"Requeued {requeued} interrupted job(s)"); await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false); + // Only kinds this engine can run: an empty filter would claim (and fail) every job, render-node work included. + _kinds = [.. (Options.Kinds ?? _handlers.Keys).Where(_handlers.ContainsKey).Distinct()]; + if (_kinds.Count == 0) + { + Log.Info("jobs", "No job handlers; this engine claims nothing"); + return; + } for (var i = 0; i < Options.Workers; i++) { var id = Options.Workers == 1 ? Options.WorkerId : $"{Options.WorkerId}#{i}"; @@ -161,13 +169,12 @@ public sealed class JobEngine : IAsyncDisposable private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown) { - var kinds = Options.Kinds ?? [.. _handlers.Keys]; while (!shutdown.IsCancellationRequested) { Job? job; try { - job = await _db.Jobs.ClaimNextAsync(workerId, kinds, shutdown).ConfigureAwait(false); + job = await _db.Jobs.ClaimNextAsync(workerId, _kinds, shutdown).ConfigureAwait(false); } catch (OperationCanceledException) { diff --git a/src/Titano.Data/Repositories/JobRepository.cs b/src/Titano.Data/Repositories/JobRepository.cs index b3bd9857..6736de5c 100644 --- a/src/Titano.Data/Repositories/JobRepository.cs +++ b/src/Titano.Data/Repositories/JobRepository.cs @@ -108,13 +108,14 @@ public sealed class JobRepository /// /// Crash recovery: puts running jobs back in the queue, keeping their checkpoints. With - /// only that worker's jobs are requeued (other server instances keep theirs). + /// only that node's jobs are requeued, including those of its numbered workers (node#0, node#1, …); + /// other server instances keep theirs. /// public Task RequeueRunningAsync(string? nodeId = null, CancellationToken cancellationToken = default) => _db.ExecuteAsync( $"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE state = {(int)JobState.Running}" + - (nodeId is null ? string.Empty : " AND node_id = $1"), - nodeId is null ? [] : [nodeId], cancellationToken); + (nodeId is null ? string.Empty : " AND (node_id = $1 OR substr(node_id, 1, length($2)) = $2)"), + nodeId is null ? [] : [nodeId, nodeId + "#"], cancellationToken); private static Job Map(DbRow r) => new() { diff --git a/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs index eec610a2..4820dc60 100644 --- a/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs +++ b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs @@ -120,6 +120,45 @@ public sealed class JobEngineTests(TestContext context) Assert.Equal("7", handler.SeenCheckpoints.Single()); } + [Test] + public async Task JobsOfNumberedWorkersResumeAfterRestart() + { + await using var db = await OpenAsync(); + var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow }; + await db.Jobs.EnqueueAsync(job); + Assert.NotNull(await db.Jobs.ClaimNextAsync("pc#1")); + var other = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow }; + await db.Jobs.EnqueueAsync(other); + Assert.NotNull(await db.Jobs.ClaimNextAsync("pc2#0")); + // Restarting node "pc" (two workers: pc#0, pc#1) requeues pc#1's job, never another node's. + var handler = new CountingHandler(JobKind.Thumbs, 3); + await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "pc" }); + await engine.StartAsync(); + await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10)); + Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(other.Id))!.State); + } + + [Test] + public async Task OnlyKindsWithAHandlerAreClaimed() + { + await using var db = await OpenAsync(); + await using (var idle = new JobEngine(db, [], new JobEngineOptions { Workers = 1, WorkerId = "idle" })) + { + await idle.StartAsync(); + var render = await idle.EnqueueAsync(JobKind.Render, []); + await Task.Delay(300); + Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(render.Id))!.State); + } + var handler = new CountingHandler(JobKind.Thumbs, 1); + await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Kinds = [JobKind.Thumbs, JobKind.Render] }); + await engine.StartAsync(); + var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []); + await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10)); + var pending = (await db.Jobs.ListAsync(JobState.Queued)).Items; + Assert.Count(1, pending); + Assert.Equal(JobKind.Render, pending[0].Kind); + } + [Test] public async Task HandlerFailureMarksJobFailedWithMessage() {