From 04a40ea22a6554c3436ec5724b74d86a70e4f157 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 22:59:00 +0000 Subject: [PATCH] fix(jobs): a job a handler cannot judge or finish never ends a worker Accepts that throws now claims the job so it fails with its reason; claim and finish errors are logged and the worker carries on. Other kinds keep their atomic claim next to selective ones, and a lost race retries at once. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Jobs/JobEngine.cs | 65 ++++++++++++++----- .../Titano.Data.Tests/Jobs/JobEngineTests.cs | 29 +++++++++ 2 files changed, 79 insertions(+), 15 deletions(-) diff --git a/src/Titano.Data/Jobs/JobEngine.cs b/src/Titano.Data/Jobs/JobEngine.cs index 815ccc12..d6296ebc 100644 --- a/src/Titano.Data/Jobs/JobEngine.cs +++ b/src/Titano.Data/Jobs/JobEngine.cs @@ -36,10 +36,14 @@ public sealed class JobEngine : IAsyncDisposable /// Queued jobs of selective kinds examined per claim; jobs declined beyond it wait until those ahead leave the queue. private const int SelectiveWindow = 100; + /// Claims lost to other workers before a worker waits for the next poll. + private const int MaxClaimRaces = 16; + private readonly TitanoDatabase _db; private readonly Dictionary _handlers; private IReadOnlyCollection _kinds = []; private Dictionary _selective = []; + private IReadOnlyCollection _others = []; private readonly ConcurrentDictionary _running = new(); private readonly Channel _wake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite }); private readonly CancellationTokenSource _shutdown = new(); @@ -71,6 +75,7 @@ public sealed class JobEngine : IAsyncDisposable // Only kinds this engine can run: an empty filter would claim (and fail) every job, render-node work included. _kinds = [.. (Options.Kinds ?? _handlers.Keys).Where(_handlers.ContainsKey).Distinct()]; _selective = _kinds.Select(k => _handlers[k]).OfType().ToDictionary(h => h.Kind); + _others = [.. _kinds.Where(k => !_selective.ContainsKey(k))]; if (_kinds.Count == 0) { Log.Info("jobs", "No job handlers; this engine claims nothing"); @@ -176,6 +181,7 @@ public sealed class JobEngine : IAsyncDisposable if (await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } job) JobChanged?.Invoke(job); } + /// Claims and runs jobs until shutdown. Nothing a job or a handler throws ends the worker. private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown) { while (!shutdown.IsCancellationRequested) @@ -185,11 +191,11 @@ public sealed class JobEngine : IAsyncDisposable { job = await ClaimAsync(workerId, shutdown).ConfigureAwait(false); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (shutdown.IsCancellationRequested) { return; } - catch (DatabaseException ex) + catch (Exception ex) { Log.Warning("jobs", "Claiming a job failed", ex); job = null; @@ -210,35 +216,64 @@ public sealed class JobEngine : IAsyncDisposable continue; } - await RunAsync(job, shutdown).ConfigureAwait(false); + try + { + await RunAsync(job, shutdown).ConfigureAwait(false); + } + catch (Exception ex) when (!shutdown.IsCancellationRequested) + { + // The job's final state could not be stored; it stays running and is requeued at the next start. + Log.Error("jobs", $"{job.Kind} job {job.Id} could not be finished", ex); + } } } /// - /// The next job in claim order (priority, then age). Without selective handlers the database picks it atomically; - /// with them, the head of the other kinds competes with the first job a selective handler accepts among the first - /// of its kinds, and the winner is claimed if it is still queued. + /// The next job in claim order (priority, then age). Without selective handlers the database picks it atomically. + /// With them, the first job a selective handler accepts among the first of its kinds + /// competes with the head of the other kinds: the other kinds are still claimed atomically, the selective job by id. + /// A lost race means another worker took a job, so looking again at once makes progress. /// private async Task ClaimAsync(string workerId, CancellationToken cancellationToken) { if (_selective.Count == 0) return await _db.Jobs.ClaimNextAsync(workerId, _kinds, cancellationToken).ConfigureAwait(false); - var others = _kinds.Where(k => !_selective.ContainsKey(k)).ToList(); - for (var attempt = 0; attempt < 3; attempt++) + for (var attempt = 0; attempt < MaxClaimRaces; attempt++) { - var best = others.Count > 0 ? (await _db.Jobs.ListQueuedAsync(others, 1, cancellationToken).ConfigureAwait(false)).FirstOrDefault() : null; - foreach (var job in await _db.Jobs.ListQueuedAsync(_selective.Keys, SelectiveWindow, cancellationToken).ConfigureAwait(false)) + Job? candidate = null; + foreach (var queued in await _db.Jobs.ListQueuedAsync(_selective.Keys, SelectiveWindow, cancellationToken).ConfigureAwait(false)) { - if (!_selective[job.Kind].Accepts(job)) continue; - if (best is null || ClaimsBefore(job, best)) best = job; + if (!Accepts(queued)) continue; + candidate = queued; break; } - if (best is null) return null; - // Another claimer may have been faster: look again. - if (await _db.Jobs.TryClaimAsync(best.Id, workerId, cancellationToken).ConfigureAwait(false) is { } claimed) return claimed; + if (candidate is null) return _others.Count > 0 ? await _db.Jobs.ClaimNextAsync(workerId, _others, cancellationToken).ConfigureAwait(false) : null; + if (_others.Count > 0 && (await _db.Jobs.ListQueuedAsync(_others, 1, cancellationToken).ConfigureAwait(false)).FirstOrDefault() is { } head + && ClaimsBefore(head, candidate)) + { + if (await _db.Jobs.ClaimNextAsync(workerId, _others, cancellationToken).ConfigureAwait(false) is { } next) return next; + } + else if (await _db.Jobs.TryClaimAsync(candidate.Id, workerId, cancellationToken).ConfigureAwait(false) is { } claimed) + { + return claimed; + } } return null; } + /// A job its handler cannot judge is taken, so it fails here with its reason instead of blocking the queue. + private bool Accepts(Job job) + { + try + { + return _selective[job.Kind].Accepts(job); + } + catch (Exception ex) + { + Log.Warning("jobs", $"{job.Kind} job {job.Id} could not be examined; it is claimed so that it fails with its reason", ex); + return true; + } + } + private static bool ClaimsBefore(Job a, Job b) => a.Priority != b.Priority ? a.Priority > b.Priority : a.Created != b.Created ? a.Created < b.Created : a.Id.CompareTo(b.Id) < 0; diff --git a/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs index df22bc0b..283648b6 100644 --- a/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs +++ b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs @@ -39,6 +39,17 @@ public sealed class JobEngineTests(TestContext context) public Task RunAsync(JobContext c) => Task.CompletedTask; } + /// Cannot judge jobs without parameters (as a render handler with a malformed preset) and fails them when run. + private sealed class FragileHandler(JobKind kind) : ISelectiveJobHandler + { + public JobKind Kind => kind; + + public bool Accepts(Job job) => job.ParametersJson?.Length > 0 ? true : throw new ArgumentNullException(nameof(job)); + + public Task RunAsync(JobContext c) => + c.Job.ParametersJson is null ? throw new InvalidDataException("no parameters") : Task.CompletedTask; + } + private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler { public JobKind Kind => kind; @@ -197,6 +208,24 @@ public sealed class JobEngineTests(TestContext context) Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId)); } + [Test] + public async Task ASelectiveHandlerThatThrowsDoesNotStopTheWorkers() + { + await using var db = await OpenAsync(); + await using var engine = new JobEngine(db, [new FragileHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)], + new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) }); + var broken = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User); + await engine.StartAsync(); + + var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed, TimeSpan.FromSeconds(10)); + Assert.Equal("no parameters", failed.Error); + // The workers are still there for every kind. + var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []); + var render = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: "{}"); + await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10)); + await WaitForStateAsync(db, render.Id, JobState.Done, TimeSpan.FromSeconds(10)); + } + [Test] public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns() {