using Chronos.Core.Model; using Chronos.Data.Jobs; using Chronos.Testing; namespace Chronos.Data.Tests.Jobs; [Category("jobs")] public sealed class JobEngineTests(TestContext context) { private sealed class CountingHandler(JobKind kind, int steps, Func? onStep = null) : IJobHandler { public JobKind Kind => kind; public List SeenCheckpoints { get; } = []; public int Completed; public async Task RunAsync(JobContext c) { lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint); var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0; for (var i = start; i < steps; i++) { c.Cancellation.ThrowIfCancellationRequested(); if (onStep is not null) await onStep(i, c.Cancellation); await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps); } Interlocked.Increment(ref Completed); } } /// Takes only jobs whose parameters say they are its own. private sealed class SelectiveHandler(JobKind kind) : ISelectiveJobHandler { public const string Mine = """{"mine":true}"""; public JobKind Kind => kind; public ValueTask AcceptsAsync(Job job, CancellationToken cancellationToken) => ValueTask.FromResult(job.ParametersJson == Mine); public Task RunAsync(JobContext c) => Task.CompletedTask; } /// Cannot judge jobs without parameters (as a render handler with a malformed preset) and fails them when run. private sealed class FragileHandler(JobKind kind) : ISelectiveJobHandler { public JobKind Kind => kind; public ValueTask AcceptsAsync(Job job, CancellationToken cancellationToken) => job.ParametersJson?.Length > 0 ? ValueTask.FromResult(true) : throw new ArgumentNullException(nameof(job)); public Task RunAsync(JobContext c) => c.Job.ParametersJson is null ? throw new InvalidDataException("no parameters") : Task.CompletedTask; } private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler { public JobKind Kind => kind; public Task RunAsync(JobContext context) => run(context); } private async Task OpenAsync() => await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation); private static async Task WaitForStateAsync(ChronosDatabase db, Guid id, JobState state, TimeSpan timeout) { var until = DateTime.UtcNow + timeout; while (DateTime.UtcNow < until) { var job = await db.Jobs.GetAsync(id); if (job?.State == state) return job; await Task.Delay(20); } Assert.Fail($"Job {id} did not reach {state}"); return null!; } [Test] public async Task RunsQueuedJobsToCompletion() { await using var db = await OpenAsync(); var handler = new CountingHandler(JobKind.Verify, 5); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) }); var progress = 0; engine.Progress += _ => Interlocked.Increment(ref progress); await engine.StartAsync(); var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A"); var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B"); var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10)); await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.Equal(2, handler.Completed); Assert.Equal(5L, doneA.ProgressDone); Assert.Greater(progress, 0); } [Test] public async Task CancelStopsARunningJob() { await using var db = await OpenAsync(); var started = new TaskCompletionSource(); var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) => { if (i == 0) started.TrySetResult(); await Task.Delay(10, ct); }); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); await engine.StartAsync(); var job = await engine.EnqueueAsync(JobKind.Scan, []); await started.Task.WaitAsync(TimeSpan.FromSeconds(10)); await engine.CancelAsync(job.Id); await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5)); Assert.Equal(0, handler.Completed); } [Test] public async Task PauseAndResumeContinueFromCheckpoint() { await using var db = await OpenAsync(); var gate = new SemaphoreSlim(0); var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) => { if (i == 10) await gate.WaitAsync(ct); }); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) }); await engine.StartAsync(); var job = await engine.EnqueueAsync(JobKind.Proxies, []); var until = DateTime.UtcNow.AddSeconds(10); while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20); await engine.PauseAsync(job.Id); var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5)); Assert.Equal("10", paused.Checkpoint); gate.Release(10); await engine.ResumeAsync(job.Id); await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.Equal("10", handler.SeenCheckpoints[^1]); } [Test] public async Task CrashedJobsResumeAfterRestart() { await using var db = await OpenAsync(); var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow }; await db.Jobs.EnqueueAsync(job); var claimed = await db.Jobs.ClaimNextAsync("t"); Assert.NotNull(claimed); await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null); // Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it. var handler = new CountingHandler(JobKind.Thumbs, 12); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); await engine.StartAsync(); await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.Equal("7", handler.SeenCheckpoints.Single()); } [Test] public async Task JobsOfNumberedWorkersResumeAfterRestart() { await using var db = await OpenAsync(); var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow }; await db.Jobs.EnqueueAsync(job); Assert.NotNull(await db.Jobs.ClaimNextAsync("pc#1")); var other = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow }; await db.Jobs.EnqueueAsync(other); Assert.NotNull(await db.Jobs.ClaimNextAsync("pc2#0")); // Restarting node "pc" (two workers: pc#0, pc#1) requeues pc#1's job, never another node's. var handler = new CountingHandler(JobKind.Thumbs, 3); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "pc" }); await engine.StartAsync(); await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(other.Id))!.State); } [Test] public async Task OnlyKindsWithAHandlerAreClaimed() { await using var db = await OpenAsync(); await using (var idle = new JobEngine(db, [], new JobEngineOptions { Workers = 1, WorkerId = "idle" })) { await idle.StartAsync(); var render = await idle.EnqueueAsync(JobKind.Render, []); await Task.Delay(300); Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(render.Id))!.State); } var handler = new CountingHandler(JobKind.Thumbs, 1); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Kinds = [JobKind.Thumbs, JobKind.Render] }); await engine.StartAsync(); var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []); await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10)); var pending = (await db.Jobs.ListAsync(JobState.Queued)).Items; Assert.Count(1, pending); Assert.Equal(JobKind.Render, pending[0].Kind); } [Test] public async Task SelectiveHandlersLeaveDeclinedJobsQueuedWithoutHoldingUpOthers() { await using var db = await OpenAsync(); await using var engine = new JobEngine(db, [new SelectiveHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) }); var declined = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: """{"mine":false}""", priority: JobPriority.User); var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.Background); var accepted = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: SelectiveHandler.Mine, priority: JobPriority.User); await engine.StartAsync(); var render = await WaitForStateAsync(db, accepted.Id, JobState.Done, TimeSpan.FromSeconds(10)); var other = await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.True(render.Started <= other.Started, "claim order: priority first, across selective and other kinds"); await Task.Delay(200); var waiting = (await db.Jobs.GetAsync(declined.Id))!; Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId)); } [Test] public async Task ASelectiveHandlerThatThrowsDoesNotStopTheWorkers() { await using var db = await OpenAsync(); await using var engine = new JobEngine(db, [new FragileHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) }); var broken = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User); await engine.StartAsync(); var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed, TimeSpan.FromSeconds(10)); Assert.Equal("no parameters", failed.Error); // The workers are still there for every kind. var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []); var render = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: "{}"); await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10)); await WaitForStateAsync(db, render.Id, JobState.Done, TimeSpan.FromSeconds(10)); } [Test] public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns() { await using var db = await OpenAsync(); var target = Guid.CreateVersion7(); Job? thumbs = null, light = null; var scan = new CallbackHandler(JobKind.Scan, async c => { thumbs = await c.EnqueueFollowUpAsync(JobKind.Thumbs, [target], "Roof", null, JobPriority.Background); light = await c.EnqueueFollowUpAsync(JobKind.LightAnalyse, [target], "Roof", null, JobPriority.Background); }); await using var engine = new JobEngine(db, [scan, new CountingHandler(JobKind.Thumbs, 1)], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) }); await engine.StartAsync(); var user = Guid.CreateVersion7(); var job = await engine.EnqueueAsync(JobKind.Scan, [], createdBy: user); await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.Null(light); var followUp = await WaitForStateAsync(db, Assert.NotNull(thumbs).Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.Equal((user, JobPriority.Background, "Roof"), (followUp.CreatedBy!.Value, followUp.Priority, followUp.TargetLabel!)); Assert.SequenceEqual([target], followUp.TargetIds); Assert.Empty((await db.Jobs.ListAsync(null, JobKind.LightAnalyse)).Items); } /// Waits until holds, failing with after ten seconds. private static async Task UntilAsync(Func condition, string what) { var until = DateTime.UtcNow.AddSeconds(10); while (!condition()) { Assert.True(DateTime.UtcNow < until, what); await Task.Delay(20); } } /// A handler whose jobs run until released, counting those running and started, by kind. private sealed class GatedHandler(JobKind kind, SemaphoreSlim release) : IJobHandler { public int Running; public int Started; public JobKind Kind => kind; public async Task RunAsync(JobContext c) { Interlocked.Increment(ref Running); Interlocked.Increment(ref Started); try { await release.WaitAsync(c.Cancellation); } finally { Interlocked.Decrement(ref Running); } } } [Test] public async Task JobsRunOneAtATimeAndPausingStartsTheNext() { // Owner review 1 (R2): one job at a time by default; pausing the running job lets the next one start. await using var db = await OpenAsync(); var release = new SemaphoreSlim(0); var handler = new GatedHandler(JobKind.Thumbs, release); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50) }); Assert.Equal(1, engine.Options.Workers); await engine.StartAsync(); var jobs = new List(); for (var i = 0; i < 3; i++) jobs.Add(await engine.EnqueueAsync(JobKind.Thumbs, [])); await UntilAsync(() => Volatile.Read(ref handler.Running) == 1, "the first job runs"); await Task.Delay(300); Assert.Equal((1, 1), (Volatile.Read(ref handler.Running), Volatile.Read(ref handler.Started))); Assert.Count(1, engine.RunningJobIds); var first = engine.RunningJobIds.First(); Assert.Equal("pc", (await db.Jobs.GetAsync(first))!.NodeId); await engine.PauseAsync(first); await UntilAsync(() => Volatile.Read(ref handler.Started) == 2 && Volatile.Read(ref handler.Running) == 1, "the next job starts"); Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(first))!.State); release.Release(10); foreach (var job in jobs.Where(j => j.Id != first)) await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(first))!.State); } [Test] public async Task WithoutThePriorityQueueARunningJobIsNeverInterrupted() { await using var db = await OpenAsync(); var release = new SemaphoreSlim(0); var scan = new GatedHandler(JobKind.Scan, release); var render = new GatedHandler(JobKind.Render, release); await using var engine = new JobEngine(db, [scan, render], new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50) }); await engine.StartAsync(); var scanning = await engine.EnqueueAsync(JobKind.Scan, []); await UntilAsync(() => Volatile.Read(ref scan.Running) == 1, "the scan runs"); var rendering = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User); await Task.Delay(400); Assert.Equal((1, 0), (Volatile.Read(ref scan.Running), Volatile.Read(ref render.Started))); Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(rendering.Id))!.State); release.Release(10); await WaitForStateAsync(db, scanning.Id, JobState.Done, TimeSpan.FromSeconds(10)); await WaitForStateAsync(db, rendering.Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.Equal(1, Volatile.Read(ref scan.Started)); } [Test] public async Task AHigherJobPausesTheRunningOneWhichResumesFromItsCheckpoint() { // Owner review 1 (R2): with render at the highest level, starting a render while a scan runs pauses the scan // without asking, renders, and resumes the scan by itself where it was. await using var db = await OpenAsync(); var checkpointed = new TaskCompletionSource(); var go = new SemaphoreSlim(0); var scan = new CountingHandler(JobKind.Scan, 4, async (i, ct) => { if (i < 2) return; checkpointed.TrySetResult(); await go.WaitAsync(ct); }); var renderRelease = new SemaphoreSlim(0); var render = new GatedHandler(JobKind.Render, renderRelease); var queue = new JobPriorityQueue { Enabled = true }; Assert.Equal((JobKindPriority.High, JobKindPriority.Normal), (queue.LevelOf(JobKind.Render), queue.LevelOf(JobKind.Scan))); await using var engine = new JobEngine(db, [scan, render], new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50), PriorityQueue = () => queue }); await engine.StartAsync(); var scanning = await engine.EnqueueAsync(JobKind.Scan, []); await checkpointed.Task.WaitAsync(TimeSpan.FromSeconds(10)); var rendering = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User); await UntilAsync(() => Volatile.Read(ref render.Running) == 1, "the render starts at once"); var paused = await db.Jobs.GetAsync(scanning.Id); Assert.Equal((JobState.Queued, "2", (string?)null), (paused!.State, paused.Checkpoint, paused.NodeId)); Assert.Contains(Chronos.Core.Localization.Loc.L("Jobs.Kind.Render"), paused.LogTail); // The scan waits while the render runs, then resumes from its checkpoint without being asked. go.Release(10); await Task.Delay(300); Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(scanning.Id))!.State); renderRelease.Release(); await WaitForStateAsync(db, rendering.Id, JobState.Done, TimeSpan.FromSeconds(10)); await WaitForStateAsync(db, scanning.Id, JobState.Done, TimeSpan.FromSeconds(10)); lock (scan.SeenCheckpoints) Assert.SequenceEqual([null, "2"], scan.SeenCheckpoints); Assert.Equal(1, scan.Completed); } [Test] public async Task OnlyAHigherLevelInterruptsAndNeverTheJobsOwnFollowUps() { await using var db = await OpenAsync(); var release = new SemaphoreSlim(0); var queued = new TaskCompletionSource(); // The light analysis queues a deep analysis of the same sequence, which the user put higher. var light = new CallbackHandler(JobKind.LightAnalyse, async c => { queued.TrySetResult(await c.EnqueueFollowUpAsync(JobKind.DeepAnalyse, [Guid.NewGuid()], "Roof", null, JobPriority.Background)); await release.WaitAsync(c.Cancellation); }); var deep = new GatedHandler(JobKind.DeepAnalyse, release); var thumbs = new GatedHandler(JobKind.Thumbs, release); var queue = new JobPriorityQueue { Enabled = true, Levels = new Dictionary { [JobKind.DeepAnalyse] = JobKindPriority.High, [JobKind.Thumbs] = JobKindPriority.Normal }, }; await using var engine = new JobEngine(db, [light, deep, thumbs], new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50), PriorityQueue = () => queue }); await engine.StartAsync(); var analysing = await engine.EnqueueAsync(JobKind.LightAnalyse, []); var followUp = Assert.NotNull(await queued.Task.WaitAsync(TimeSpan.FromSeconds(10))); // A job of the same level waits too. var drawing = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.User); await Task.Delay(400); Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(analysing.Id))!.State); Assert.Equal((0, 0), (Volatile.Read(ref deep.Started), Volatile.Read(ref thumbs.Started))); Assert.False(await engine.ReviewAsync(context.Cancellation)); // Another deep analysis, not its own, interrupts it. var other = await engine.EnqueueAsync(JobKind.DeepAnalyse, [Guid.NewGuid()], priority: JobPriority.User); await UntilAsync(() => Volatile.Read(ref deep.Running) == 1, "a higher job runs"); Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(analysing.Id))!.State); release.Release(20); foreach (var id in new[] { analysing.Id, followUp.Id, drawing.Id, other.Id }) await WaitForStateAsync(db, id, JobState.Done, TimeSpan.FromSeconds(10)); } [Test] public async Task RequeueingAYieldedJobLeavesPausesAndOtherNodesAlone() { await using var db = await OpenAsync(); var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow }; await db.Jobs.EnqueueAsync(job); Assert.NotNull(await db.Jobs.ClaimNextAsync("pc")); Assert.False(await db.Jobs.RequeueAsync(job.Id, "other")); await db.Jobs.SetStateAsync(job.Id, JobState.Paused); Assert.False(await db.Jobs.RequeueAsync(job.Id, "pc")); Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(job.Id))!.State); await db.Jobs.SetStateAsync(job.Id, JobState.Queued); Assert.NotNull(await db.Jobs.ClaimNextAsync("pc")); Assert.True(await db.Jobs.RequeueAsync(job.Id, "pc")); Assert.Equal((JobState.Queued, (string?)null), ((await db.Jobs.GetAsync(job.Id))!.State, (await db.Jobs.GetAsync(job.Id))!.NodeId)); } [Test] public async Task HandlerFailureMarksJobFailedWithMessage() { await using var db = await OpenAsync(); var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); await engine.StartAsync(); var job = await engine.EnqueueAsync(JobKind.Render, []); var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10)); Assert.Equal("encoder lost", failed.Error); Assert.Contains("encoder lost", failed.LogTail); } }