Files
Chronos/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs
T
Claude 4007293833 feat(server): render MP4 jobs no render node can take as MJPEG, and say so
BRIEF §14.7: the server falls back to managed formats when no node is
available and says so in the job log. An MP4 job waits for render nodes at
least the node timeout, then as long as an online node can encode its codec
at its size; after that the server's engine takes it and writes each output
it cannot encode as MJPEG in AVI at the same size and frame rate, with a job
log line. A job the server started resumes there. Selective acceptance is
asynchronous for that; when node availability cannot be read the job keeps
waiting.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-03 09:39:17 +00:00

268 lines
13 KiB
C#

using Titano.Core.Model;
using Titano.Data.Jobs;
using Titano.Testing;
namespace Titano.Data.Tests.Jobs;
[Category("jobs")]
public sealed class JobEngineTests(TestContext context)
{
private sealed class CountingHandler(JobKind kind, int steps, Func<int, CancellationToken, Task>? onStep = null) : IJobHandler
{
public JobKind Kind => kind;
public List<string?> SeenCheckpoints { get; } = [];
public int Completed;
public async Task RunAsync(JobContext c)
{
lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint);
var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0;
for (var i = start; i < steps; i++)
{
c.Cancellation.ThrowIfCancellationRequested();
if (onStep is not null) await onStep(i, c.Cancellation);
await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps);
}
Interlocked.Increment(ref Completed);
}
}
/// <summary>Takes only jobs whose parameters say they are its own.</summary>
private sealed class SelectiveHandler(JobKind kind) : ISelectiveJobHandler
{
public const string Mine = """{"mine":true}""";
public JobKind Kind => kind;
public ValueTask<bool> AcceptsAsync(Job job, CancellationToken cancellationToken) => ValueTask.FromResult(job.ParametersJson == Mine);
public Task RunAsync(JobContext c) => Task.CompletedTask;
}
/// <summary>Cannot judge jobs without parameters (as a render handler with a malformed preset) and fails them when run.</summary>
private sealed class FragileHandler(JobKind kind) : ISelectiveJobHandler
{
public JobKind Kind => kind;
public ValueTask<bool> AcceptsAsync(Job job, CancellationToken cancellationToken) =>
job.ParametersJson?.Length > 0 ? ValueTask.FromResult(true) : throw new ArgumentNullException(nameof(job));
public Task RunAsync(JobContext c) =>
c.Job.ParametersJson is null ? throw new InvalidDataException("no parameters") : Task.CompletedTask;
}
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
{
public JobKind Kind => kind;
public Task RunAsync(JobContext context) => run(context);
}
private async Task<TitanoDatabase> OpenAsync() =>
await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
private static async Task<Job> WaitForStateAsync(TitanoDatabase db, Guid id, JobState state, TimeSpan timeout)
{
var until = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < until)
{
var job = await db.Jobs.GetAsync(id);
if (job?.State == state) return job;
await Task.Delay(20);
}
Assert.Fail($"Job {id} did not reach {state}");
return null!;
}
[Test]
public async Task RunsQueuedJobsToCompletion()
{
await using var db = await OpenAsync();
var handler = new CountingHandler(JobKind.Verify, 5);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
var progress = 0;
engine.Progress += _ => Interlocked.Increment(ref progress);
await engine.StartAsync();
var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A");
var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B");
var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10));
await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal(2, handler.Completed);
Assert.Equal(5L, doneA.ProgressDone);
Assert.Greater(progress, 0);
}
[Test]
public async Task CancelStopsARunningJob()
{
await using var db = await OpenAsync();
var started = new TaskCompletionSource();
var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) =>
{
if (i == 0) started.TrySetResult();
await Task.Delay(10, ct);
});
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
await engine.StartAsync();
var job = await engine.EnqueueAsync(JobKind.Scan, []);
await started.Task.WaitAsync(TimeSpan.FromSeconds(10));
await engine.CancelAsync(job.Id);
await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5));
Assert.Equal(0, handler.Completed);
}
[Test]
public async Task PauseAndResumeContinueFromCheckpoint()
{
await using var db = await OpenAsync();
var gate = new SemaphoreSlim(0);
var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) =>
{
if (i == 10) await gate.WaitAsync(ct);
});
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
await engine.StartAsync();
var job = await engine.EnqueueAsync(JobKind.Proxies, []);
var until = DateTime.UtcNow.AddSeconds(10);
while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20);
await engine.PauseAsync(job.Id);
var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5));
Assert.Equal("10", paused.Checkpoint);
gate.Release(10);
await engine.ResumeAsync(job.Id);
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal("10", handler.SeenCheckpoints[^1]);
}
[Test]
public async Task CrashedJobsResumeAfterRestart()
{
await using var db = await OpenAsync();
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
await db.Jobs.EnqueueAsync(job);
var claimed = await db.Jobs.ClaimNextAsync("t");
Assert.NotNull(claimed);
await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null);
// Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it.
var handler = new CountingHandler(JobKind.Thumbs, 12);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
await engine.StartAsync();
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal("7", handler.SeenCheckpoints.Single());
}
[Test]
public async Task JobsOfNumberedWorkersResumeAfterRestart()
{
await using var db = await OpenAsync();
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
await db.Jobs.EnqueueAsync(job);
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc#1"));
var other = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
await db.Jobs.EnqueueAsync(other);
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc2#0"));
// Restarting node "pc" (two workers: pc#0, pc#1) requeues pc#1's job, never another node's.
var handler = new CountingHandler(JobKind.Thumbs, 3);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "pc" });
await engine.StartAsync();
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(other.Id))!.State);
}
[Test]
public async Task OnlyKindsWithAHandlerAreClaimed()
{
await using var db = await OpenAsync();
await using (var idle = new JobEngine(db, [], new JobEngineOptions { Workers = 1, WorkerId = "idle" }))
{
await idle.StartAsync();
var render = await idle.EnqueueAsync(JobKind.Render, []);
await Task.Delay(300);
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(render.Id))!.State);
}
var handler = new CountingHandler(JobKind.Thumbs, 1);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Kinds = [JobKind.Thumbs, JobKind.Render] });
await engine.StartAsync();
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
var pending = (await db.Jobs.ListAsync(JobState.Queued)).Items;
Assert.Count(1, pending);
Assert.Equal(JobKind.Render, pending[0].Kind);
}
[Test]
public async Task SelectiveHandlersLeaveDeclinedJobsQueuedWithoutHoldingUpOthers()
{
await using var db = await OpenAsync();
await using var engine = new JobEngine(db, [new SelectiveHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
var declined = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: """{"mine":false}""", priority: JobPriority.User);
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.Background);
var accepted = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: SelectiveHandler.Mine, priority: JobPriority.User);
await engine.StartAsync();
var render = await WaitForStateAsync(db, accepted.Id, JobState.Done, TimeSpan.FromSeconds(10));
var other = await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.True(render.Started <= other.Started, "claim order: priority first, across selective and other kinds");
await Task.Delay(200);
var waiting = (await db.Jobs.GetAsync(declined.Id))!;
Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId));
}
[Test]
public async Task ASelectiveHandlerThatThrowsDoesNotStopTheWorkers()
{
await using var db = await OpenAsync();
await using var engine = new JobEngine(db, [new FragileHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
var broken = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
await engine.StartAsync();
var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed, TimeSpan.FromSeconds(10));
Assert.Equal("no parameters", failed.Error);
// The workers are still there for every kind.
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
var render = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: "{}");
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
await WaitForStateAsync(db, render.Id, JobState.Done, TimeSpan.FromSeconds(10));
}
[Test]
public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns()
{
await using var db = await OpenAsync();
var target = Guid.CreateVersion7();
Job? thumbs = null, light = null;
var scan = new CallbackHandler(JobKind.Scan, async c =>
{
thumbs = await c.EnqueueFollowUpAsync(JobKind.Thumbs, [target], "Roof", null, JobPriority.Background);
light = await c.EnqueueFollowUpAsync(JobKind.LightAnalyse, [target], "Roof", null, JobPriority.Background);
});
await using var engine = new JobEngine(db, [scan, new CountingHandler(JobKind.Thumbs, 1)],
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync();
var user = Guid.CreateVersion7();
var job = await engine.EnqueueAsync(JobKind.Scan, [], createdBy: user);
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Null(light);
var followUp = await WaitForStateAsync(db, Assert.NotNull(thumbs).Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal((user, JobPriority.Background, "Roof"), (followUp.CreatedBy!.Value, followUp.Priority, followUp.TargetLabel!));
Assert.SequenceEqual([target], followUp.TargetIds);
Assert.Empty((await db.Jobs.ListAsync(null, JobKind.LightAnalyse)).Items);
}
[Test]
public async Task HandlerFailureMarksJobFailedWithMessage()
{
await using var db = await OpenAsync();
var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
await engine.StartAsync();
var job = await engine.EnqueueAsync(JobKind.Render, []);
var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10));
Assert.Equal("encoder lost", failed.Error);
Assert.Contains("encoder lost", failed.LogTail);
}
}