An engine with no handlers (or a filter it could not serve) claimed every job, so a host without a render handler would fail render-node work. With several workers, jobs carry node ids such as pc#1, but crash recovery only matched the bare node id and left them running forever. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
175 lines
8.3 KiB
C#
175 lines
8.3 KiB
C#
using Titano.Core.Model;
|
|
using Titano.Data.Jobs;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Data.Tests.Jobs;
|
|
|
|
[Category("jobs")]
|
|
public sealed class JobEngineTests(TestContext context)
|
|
{
|
|
private sealed class CountingHandler(JobKind kind, int steps, Func<int, CancellationToken, Task>? onStep = null) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
public List<string?> SeenCheckpoints { get; } = [];
|
|
public int Completed;
|
|
|
|
public async Task RunAsync(JobContext c)
|
|
{
|
|
lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint);
|
|
var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0;
|
|
for (var i = start; i < steps; i++)
|
|
{
|
|
c.Cancellation.ThrowIfCancellationRequested();
|
|
if (onStep is not null) await onStep(i, c.Cancellation);
|
|
await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps);
|
|
}
|
|
Interlocked.Increment(ref Completed);
|
|
}
|
|
}
|
|
|
|
private async Task<TitanoDatabase> OpenAsync() =>
|
|
await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
|
|
|
|
private static async Task<Job> WaitForStateAsync(TitanoDatabase db, Guid id, JobState state, TimeSpan timeout)
|
|
{
|
|
var until = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < until)
|
|
{
|
|
var job = await db.Jobs.GetAsync(id);
|
|
if (job?.State == state) return job;
|
|
await Task.Delay(20);
|
|
}
|
|
Assert.Fail($"Job {id} did not reach {state}");
|
|
return null!;
|
|
}
|
|
|
|
[Test]
|
|
public async Task RunsQueuedJobsToCompletion()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var handler = new CountingHandler(JobKind.Verify, 5);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
|
var progress = 0;
|
|
engine.Progress += _ => Interlocked.Increment(ref progress);
|
|
await engine.StartAsync();
|
|
var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A");
|
|
var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B");
|
|
var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(2, handler.Completed);
|
|
Assert.Equal(5L, doneA.ProgressDone);
|
|
Assert.Greater(progress, 0);
|
|
}
|
|
|
|
[Test]
|
|
public async Task CancelStopsARunningJob()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var started = new TaskCompletionSource();
|
|
var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) =>
|
|
{
|
|
if (i == 0) started.TrySetResult();
|
|
await Task.Delay(10, ct);
|
|
});
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await started.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
await engine.CancelAsync(job.Id);
|
|
await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5));
|
|
Assert.Equal(0, handler.Completed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task PauseAndResumeContinueFromCheckpoint()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var gate = new SemaphoreSlim(0);
|
|
var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) =>
|
|
{
|
|
if (i == 10) await gate.WaitAsync(ct);
|
|
});
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Proxies, []);
|
|
var until = DateTime.UtcNow.AddSeconds(10);
|
|
while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20);
|
|
await engine.PauseAsync(job.Id);
|
|
var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5));
|
|
Assert.Equal("10", paused.Checkpoint);
|
|
gate.Release(10);
|
|
await engine.ResumeAsync(job.Id);
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("10", handler.SeenCheckpoints[^1]);
|
|
}
|
|
|
|
[Test]
|
|
public async Task CrashedJobsResumeAfterRestart()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(job);
|
|
var claimed = await db.Jobs.ClaimNextAsync("t");
|
|
Assert.NotNull(claimed);
|
|
await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null);
|
|
// Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it.
|
|
var handler = new CountingHandler(JobKind.Thumbs, 12);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("7", handler.SeenCheckpoints.Single());
|
|
}
|
|
|
|
[Test]
|
|
public async Task JobsOfNumberedWorkersResumeAfterRestart()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(job);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc#1"));
|
|
var other = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(other);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc2#0"));
|
|
// Restarting node "pc" (two workers: pc#0, pc#1) requeues pc#1's job, never another node's.
|
|
var handler = new CountingHandler(JobKind.Thumbs, 3);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "pc" });
|
|
await engine.StartAsync();
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(other.Id))!.State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OnlyKindsWithAHandlerAreClaimed()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
await using (var idle = new JobEngine(db, [], new JobEngineOptions { Workers = 1, WorkerId = "idle" }))
|
|
{
|
|
await idle.StartAsync();
|
|
var render = await idle.EnqueueAsync(JobKind.Render, []);
|
|
await Task.Delay(300);
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(render.Id))!.State);
|
|
}
|
|
var handler = new CountingHandler(JobKind.Thumbs, 1);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Kinds = [JobKind.Thumbs, JobKind.Render] });
|
|
await engine.StartAsync();
|
|
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
|
|
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
var pending = (await db.Jobs.ListAsync(JobState.Queued)).Items;
|
|
Assert.Count(1, pending);
|
|
Assert.Equal(JobKind.Render, pending[0].Kind);
|
|
}
|
|
|
|
[Test]
|
|
public async Task HandlerFailureMarksJobFailedWithMessage()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Render, []);
|
|
var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("encoder lost", failed.Error);
|
|
Assert.Contains("encoder lost", failed.LogTail);
|
|
}
|
|
}
|