fix(jobs): claim only handled kinds and requeue numbered workers after a crash
An engine with no handlers (or a filter it could not serve) claimed every job, so a host without a render handler would fail render-node work. With several workers, jobs carry node ids such as pc#1, but crash recovery only matched the bare node id and left them running forever. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
3 files changed
+52
-5
No files matched your search
@@ -34,6 +34,7 @@ public sealed class JobEngine : IAsyncDisposable
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly Dictionary<JobKind, IJobHandler> _handlers;
|
||||
private IReadOnlyCollection<JobKind> _kinds = [];
|
||||
private readonly ConcurrentDictionary<Guid, CancellationTokenSource> _running = new();
|
||||
private readonly Channel<bool> _wake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
|
||||
private readonly CancellationTokenSource _shutdown = new();
|
||||
@@ -62,6 +63,13 @@ public sealed class JobEngine : IAsyncDisposable
|
||||
var requeued = await _db.Jobs.RequeueRunningAsync(Options.WorkerId, cancellationToken).ConfigureAwait(false);
|
||||
if (requeued > 0) Log.Info("jobs", $"Requeued {requeued} interrupted job(s)");
|
||||
await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false);
|
||||
// Only kinds this engine can run: an empty filter would claim (and fail) every job, render-node work included.
|
||||
_kinds = [.. (Options.Kinds ?? _handlers.Keys).Where(_handlers.ContainsKey).Distinct()];
|
||||
if (_kinds.Count == 0)
|
||||
{
|
||||
Log.Info("jobs", "No job handlers; this engine claims nothing");
|
||||
return;
|
||||
}
|
||||
for (var i = 0; i < Options.Workers; i++)
|
||||
{
|
||||
var id = Options.Workers == 1 ? Options.WorkerId : $"{Options.WorkerId}#{i}";
|
||||
@@ -161,13 +169,12 @@ public sealed class JobEngine : IAsyncDisposable
|
||||
|
||||
private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown)
|
||||
{
|
||||
var kinds = Options.Kinds ?? [.. _handlers.Keys];
|
||||
while (!shutdown.IsCancellationRequested)
|
||||
{
|
||||
Job? job;
|
||||
try
|
||||
{
|
||||
job = await _db.Jobs.ClaimNextAsync(workerId, kinds, shutdown).ConfigureAwait(false);
|
||||
job = await _db.Jobs.ClaimNextAsync(workerId, _kinds, shutdown).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
|
||||
@@ -108,13 +108,14 @@ public sealed class JobRepository
|
||||
|
||||
/// <summary>
|
||||
/// Crash recovery: puts running jobs back in the queue, keeping their checkpoints. With <paramref name="nodeId"/>
|
||||
/// only that worker's jobs are requeued (other server instances keep theirs).
|
||||
/// only that node's jobs are requeued, including those of its numbered workers (<c>node#0</c>, <c>node#1</c>, …);
|
||||
/// other server instances keep theirs.
|
||||
/// </summary>
|
||||
public Task<int> RequeueRunningAsync(string? nodeId = null, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync(
|
||||
$"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE state = {(int)JobState.Running}" +
|
||||
(nodeId is null ? string.Empty : " AND node_id = $1"),
|
||||
nodeId is null ? [] : [nodeId], cancellationToken);
|
||||
(nodeId is null ? string.Empty : " AND (node_id = $1 OR substr(node_id, 1, length($2)) = $2)"),
|
||||
nodeId is null ? [] : [nodeId, nodeId + "#"], cancellationToken);
|
||||
|
||||
private static Job Map(DbRow r) => new()
|
||||
{
|
||||
|
||||
@@ -120,6 +120,45 @@ public sealed class JobEngineTests(TestContext context)
|
||||
Assert.Equal("7", handler.SeenCheckpoints.Single());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task JobsOfNumberedWorkersResumeAfterRestart()
|
||||
{
|
||||
await using var db = await OpenAsync();
|
||||
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
||||
await db.Jobs.EnqueueAsync(job);
|
||||
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc#1"));
|
||||
var other = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
||||
await db.Jobs.EnqueueAsync(other);
|
||||
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc2#0"));
|
||||
// Restarting node "pc" (two workers: pc#0, pc#1) requeues pc#1's job, never another node's.
|
||||
var handler = new CountingHandler(JobKind.Thumbs, 3);
|
||||
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "pc" });
|
||||
await engine.StartAsync();
|
||||
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
||||
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(other.Id))!.State);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OnlyKindsWithAHandlerAreClaimed()
|
||||
{
|
||||
await using var db = await OpenAsync();
|
||||
await using (var idle = new JobEngine(db, [], new JobEngineOptions { Workers = 1, WorkerId = "idle" }))
|
||||
{
|
||||
await idle.StartAsync();
|
||||
var render = await idle.EnqueueAsync(JobKind.Render, []);
|
||||
await Task.Delay(300);
|
||||
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(render.Id))!.State);
|
||||
}
|
||||
var handler = new CountingHandler(JobKind.Thumbs, 1);
|
||||
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Kinds = [JobKind.Thumbs, JobKind.Render] });
|
||||
await engine.StartAsync();
|
||||
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
|
||||
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
||||
var pending = (await db.Jobs.ListAsync(JobState.Queued)).Items;
|
||||
Assert.Count(1, pending);
|
||||
Assert.Equal(JobKind.Render, pending[0].Kind);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task HandlerFailureMarksJobFailedWithMessage()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user