A conversion counted as deleted when no frame of the sidecar held it, no tombstone ended its chain and every frame was hashed. A sidecar rebuilt by adoption meets that after a routine verify although it never knew the conversion, so the originals were erased, the dry run calling them the last copy, while the damaged conversion was still in the folder. A sequence found nowhere and missing from this index counted as gone in the same way, even when the walk had only skipped its folder (photos without a sidecar), and its live conversion was never hashed. A conversion is now deleted only on positive evidence: a Deleted tombstone ending its chain in the sequence's sidecar, the sequence in the trash, or the trash's record of erasing it (.chronos/trash/.erased/<id>.json, written before a deleted sequence's folder goes while a replaced entry of it stays, swept once none does). Anything else keeps the entry, and its job log line says what settles it: verify the sequence (a frame never hashed), scan the archive (a sidecar missing or unreadable), rebuild the index (a row the archive does not bear out), or restore the entry to keep its photos. Also: - a dry run or an emptying walks the archive once for every sequence that moved, and again only when a folder it found no longer holds it; - a download queues the passes a frame swap cleared whatever stops the job (JobContext.EnqueueFollowUpAsync with a token), as a restore does; - the confirmation's detail takes Tab and scrolls from the keyboard when it is longer than its card. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
521 lines
26 KiB
C#
521 lines
26 KiB
C#
using Chronos.Core.Model;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Jobs;
|
|
|
|
[Category("jobs")]
|
|
public sealed class JobEngineTests(TestContext context)
|
|
{
|
|
private sealed class CountingHandler(JobKind kind, int steps, Func<int, CancellationToken, Task>? onStep = null) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
public List<string?> SeenCheckpoints { get; } = [];
|
|
public int Completed;
|
|
|
|
public async Task RunAsync(JobContext c)
|
|
{
|
|
lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint);
|
|
var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0;
|
|
for (var i = start; i < steps; i++)
|
|
{
|
|
c.Cancellation.ThrowIfCancellationRequested();
|
|
if (onStep is not null) await onStep(i, c.Cancellation);
|
|
await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps);
|
|
}
|
|
Interlocked.Increment(ref Completed);
|
|
}
|
|
}
|
|
|
|
/// <summary>Takes only jobs whose parameters say they are its own.</summary>
|
|
private sealed class SelectiveHandler(JobKind kind) : ISelectiveJobHandler
|
|
{
|
|
public const string Mine = """{"mine":true}""";
|
|
|
|
public JobKind Kind => kind;
|
|
|
|
public ValueTask<bool> AcceptsAsync(Job job, CancellationToken cancellationToken) => ValueTask.FromResult(job.ParametersJson == Mine);
|
|
|
|
public Task RunAsync(JobContext c) => Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>Cannot judge jobs without parameters (as a render handler with a malformed preset) and fails them when run.</summary>
|
|
private sealed class FragileHandler(JobKind kind) : ISelectiveJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
|
|
public ValueTask<bool> AcceptsAsync(Job job, CancellationToken cancellationToken) =>
|
|
job.ParametersJson?.Length > 0 ? ValueTask.FromResult(true) : throw new ArgumentNullException(nameof(job));
|
|
|
|
public Task RunAsync(JobContext c) =>
|
|
c.Job.ParametersJson is null ? throw new InvalidDataException("no parameters") : Task.CompletedTask;
|
|
}
|
|
|
|
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
|
|
public Task RunAsync(JobContext context) => run(context);
|
|
}
|
|
|
|
private async Task<ChronosDatabase> OpenAsync() =>
|
|
await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
|
|
|
|
private static async Task<Job> WaitForStateAsync(ChronosDatabase db, Guid id, JobState state, TimeSpan timeout)
|
|
{
|
|
var until = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < until)
|
|
{
|
|
var job = await db.Jobs.GetAsync(id);
|
|
if (job?.State == state) return job;
|
|
await Task.Delay(20);
|
|
}
|
|
Assert.Fail($"Job {id} did not reach {state}");
|
|
return null!;
|
|
}
|
|
|
|
[Test]
|
|
public async Task RunsQueuedJobsToCompletion()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var handler = new CountingHandler(JobKind.Verify, 5);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
|
var progress = 0;
|
|
engine.Progress += _ => Interlocked.Increment(ref progress);
|
|
await engine.StartAsync();
|
|
var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A");
|
|
var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B");
|
|
var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(2, handler.Completed);
|
|
Assert.Equal(5L, doneA.ProgressDone);
|
|
Assert.Greater(progress, 0);
|
|
}
|
|
|
|
[Test]
|
|
public async Task CancelStopsARunningJob()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var started = new TaskCompletionSource();
|
|
var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) =>
|
|
{
|
|
if (i == 0) started.TrySetResult();
|
|
await Task.Delay(10, ct);
|
|
});
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await started.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
await engine.CancelAsync(job.Id);
|
|
await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5));
|
|
Assert.Equal(0, handler.Completed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task PauseAndResumeContinueFromCheckpoint()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var gate = new SemaphoreSlim(0);
|
|
var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) =>
|
|
{
|
|
if (i == 10) await gate.WaitAsync(ct);
|
|
});
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Proxies, []);
|
|
var until = DateTime.UtcNow.AddSeconds(10);
|
|
while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20);
|
|
await engine.PauseAsync(job.Id);
|
|
var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5));
|
|
Assert.Equal("10", paused.Checkpoint);
|
|
gate.Release(10);
|
|
await engine.ResumeAsync(job.Id);
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("10", handler.SeenCheckpoints[^1]);
|
|
}
|
|
|
|
[Test]
|
|
public async Task CrashedJobsResumeAfterRestart()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(job);
|
|
var claimed = await db.Jobs.ClaimNextAsync("t");
|
|
Assert.NotNull(claimed);
|
|
await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null);
|
|
// Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it.
|
|
var handler = new CountingHandler(JobKind.Thumbs, 12);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("7", handler.SeenCheckpoints.Single());
|
|
}
|
|
|
|
[Test]
|
|
public async Task JobsOfNumberedWorkersResumeAfterRestart()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(job);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc#1"));
|
|
var other = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(other);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc2#0"));
|
|
// Restarting node "pc" (two workers: pc#0, pc#1) requeues pc#1's job, never another node's.
|
|
var handler = new CountingHandler(JobKind.Thumbs, 3);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "pc" });
|
|
await engine.StartAsync();
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(other.Id))!.State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OnlyKindsWithAHandlerAreClaimed()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
await using (var idle = new JobEngine(db, [], new JobEngineOptions { Workers = 1, WorkerId = "idle" }))
|
|
{
|
|
await idle.StartAsync();
|
|
var render = await idle.EnqueueAsync(JobKind.Render, []);
|
|
await Task.Delay(300);
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(render.Id))!.State);
|
|
}
|
|
var handler = new CountingHandler(JobKind.Thumbs, 1);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Kinds = [JobKind.Thumbs, JobKind.Render] });
|
|
await engine.StartAsync();
|
|
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
|
|
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
var pending = (await db.Jobs.ListAsync(JobState.Queued)).Items;
|
|
Assert.Count(1, pending);
|
|
Assert.Equal(JobKind.Render, pending[0].Kind);
|
|
}
|
|
|
|
[Test]
|
|
public async Task SelectiveHandlersLeaveDeclinedJobsQueuedWithoutHoldingUpOthers()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
await using var engine = new JobEngine(db, [new SelectiveHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
|
|
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
var declined = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: """{"mine":false}""", priority: JobPriority.User);
|
|
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.Background);
|
|
var accepted = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: SelectiveHandler.Mine, priority: JobPriority.User);
|
|
await engine.StartAsync();
|
|
|
|
var render = await WaitForStateAsync(db, accepted.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
var other = await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.True(render.Started <= other.Started, "claim order: priority first, across selective and other kinds");
|
|
await Task.Delay(200);
|
|
var waiting = (await db.Jobs.GetAsync(declined.Id))!;
|
|
Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASelectiveHandlerThatThrowsDoesNotStopTheWorkers()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
await using var engine = new JobEngine(db, [new FragileHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
|
|
new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
var broken = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
|
|
await engine.StartAsync();
|
|
|
|
var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("no parameters", failed.Error);
|
|
// The workers are still there for every kind.
|
|
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
|
|
var render = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: "{}");
|
|
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, render.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
[Test]
|
|
public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var target = Guid.CreateVersion7();
|
|
Job? thumbs = null, light = null;
|
|
var scan = new CallbackHandler(JobKind.Scan, async c =>
|
|
{
|
|
thumbs = await c.EnqueueFollowUpAsync(JobKind.Thumbs, [target], "Roof", null, JobPriority.Background);
|
|
light = await c.EnqueueFollowUpAsync(JobKind.LightAnalyse, [target], "Roof", null, JobPriority.Background);
|
|
});
|
|
await using var engine = new JobEngine(db, [scan, new CountingHandler(JobKind.Thumbs, 1)],
|
|
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
await engine.StartAsync();
|
|
var user = Guid.CreateVersion7();
|
|
var job = await engine.EnqueueAsync(JobKind.Scan, [], createdBy: user);
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
|
|
Assert.Null(light);
|
|
var followUp = await WaitForStateAsync(db, Assert.NotNull(thumbs).Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal((user, JobPriority.Background, "Roof"), (followUp.CreatedBy!.Value, followUp.Priority, followUp.TargetLabel!));
|
|
Assert.SequenceEqual([target], followUp.TargetIds);
|
|
Assert.Empty((await db.Jobs.ListAsync(null, JobKind.LightAnalyse)).Items);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A follow-up of work already done (a sync download's frame swapped to another encoding, whose thumbnails it cleared) is
|
|
/// queued whatever stops its job meanwhile: a cancel arriving right after the work leaves it queued, and it runs.
|
|
/// </summary>
|
|
[Test]
|
|
public async Task AFollowUpQueuedWhateverStopsItsJobOutlivesACancel()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var target = Guid.CreateVersion7();
|
|
var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var queued = new TaskCompletionSource<Job?>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var download = new CallbackHandler(JobKind.Scan, async c =>
|
|
{
|
|
started.TrySetResult();
|
|
try
|
|
{
|
|
await Task.Delay(Timeout.InfiniteTimeSpan, c.Cancellation);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
// The work is done; the cancel came before its follow-up was queued.
|
|
}
|
|
try
|
|
{
|
|
queued.TrySetResult(await c.EnqueueFollowUpAsync(JobKind.Thumbs, [target], "Roof", null, JobPriority.Background, CancellationToken.None));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
queued.TrySetException(ex);
|
|
}
|
|
c.Cancellation.ThrowIfCancellationRequested();
|
|
});
|
|
await using var engine = new JobEngine(db, [download, new CountingHandler(JobKind.Thumbs, 1)],
|
|
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await started.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
|
|
await engine.CancelAsync(job.Id);
|
|
|
|
var followUp = Assert.NotNull(await queued.Task.WaitAsync(TimeSpan.FromSeconds(10)), "queued although its job was cancelled");
|
|
await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5));
|
|
Assert.SequenceEqual([target], (await WaitForStateAsync(db, followUp.Id, JobState.Done, TimeSpan.FromSeconds(10))).TargetIds);
|
|
}
|
|
|
|
/// <summary>Waits until <paramref name="condition"/> holds, failing with <paramref name="what"/> after ten seconds.</summary>
|
|
private static async Task UntilAsync(Func<bool> condition, string what)
|
|
{
|
|
var until = DateTime.UtcNow.AddSeconds(10);
|
|
while (!condition())
|
|
{
|
|
Assert.True(DateTime.UtcNow < until, what);
|
|
await Task.Delay(20);
|
|
}
|
|
}
|
|
|
|
/// <summary>A handler whose jobs run until released, counting those running and started, by kind.</summary>
|
|
private sealed class GatedHandler(JobKind kind, SemaphoreSlim release) : IJobHandler
|
|
{
|
|
public int Running;
|
|
public int Started;
|
|
|
|
public JobKind Kind => kind;
|
|
|
|
public async Task RunAsync(JobContext c)
|
|
{
|
|
Interlocked.Increment(ref Running);
|
|
Interlocked.Increment(ref Started);
|
|
try
|
|
{
|
|
await release.WaitAsync(c.Cancellation);
|
|
}
|
|
finally
|
|
{
|
|
Interlocked.Decrement(ref Running);
|
|
}
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task JobsRunOneAtATimeAndPausingStartsTheNext()
|
|
{
|
|
// Owner review 1 (R2): one job at a time by default; pausing the running job lets the next one start.
|
|
await using var db = await OpenAsync();
|
|
var release = new SemaphoreSlim(0);
|
|
var handler = new GatedHandler(JobKind.Thumbs, release);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
Assert.Equal(1, engine.Options.Workers);
|
|
await engine.StartAsync();
|
|
var jobs = new List<Job>();
|
|
for (var i = 0; i < 3; i++) jobs.Add(await engine.EnqueueAsync(JobKind.Thumbs, []));
|
|
|
|
await UntilAsync(() => Volatile.Read(ref handler.Running) == 1, "the first job runs");
|
|
await Task.Delay(300);
|
|
Assert.Equal((1, 1), (Volatile.Read(ref handler.Running), Volatile.Read(ref handler.Started)));
|
|
Assert.Count(1, engine.RunningJobIds);
|
|
var first = engine.RunningJobIds.First();
|
|
Assert.Equal("pc", (await db.Jobs.GetAsync(first))!.NodeId);
|
|
|
|
await engine.PauseAsync(first);
|
|
await UntilAsync(() => Volatile.Read(ref handler.Started) == 2 && Volatile.Read(ref handler.Running) == 1, "the next job starts");
|
|
Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(first))!.State);
|
|
|
|
release.Release(10);
|
|
foreach (var job in jobs.Where(j => j.Id != first)) await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(first))!.State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task WithoutThePriorityQueueARunningJobIsNeverInterrupted()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var release = new SemaphoreSlim(0);
|
|
var scan = new GatedHandler(JobKind.Scan, release);
|
|
var render = new GatedHandler(JobKind.Render, release);
|
|
await using var engine = new JobEngine(db, [scan, render], new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
await engine.StartAsync();
|
|
var scanning = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await UntilAsync(() => Volatile.Read(ref scan.Running) == 1, "the scan runs");
|
|
var rendering = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
|
|
await Task.Delay(400);
|
|
Assert.Equal((1, 0), (Volatile.Read(ref scan.Running), Volatile.Read(ref render.Started)));
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(rendering.Id))!.State);
|
|
|
|
release.Release(10);
|
|
await WaitForStateAsync(db, scanning.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, rendering.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(1, Volatile.Read(ref scan.Started));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AHigherJobPausesTheRunningOneWhichResumesFromItsCheckpoint()
|
|
{
|
|
// Owner review 1 (R2): with render at the highest level, starting a render while a scan runs pauses the scan
|
|
// without asking, renders, and resumes the scan by itself where it was.
|
|
await using var db = await OpenAsync();
|
|
var checkpointed = new TaskCompletionSource();
|
|
var go = new SemaphoreSlim(0);
|
|
var scan = new CountingHandler(JobKind.Scan, 4, async (i, ct) =>
|
|
{
|
|
if (i < 2) return;
|
|
checkpointed.TrySetResult();
|
|
await go.WaitAsync(ct);
|
|
});
|
|
var renderRelease = new SemaphoreSlim(0);
|
|
var render = new GatedHandler(JobKind.Render, renderRelease);
|
|
var queue = new JobPriorityQueue { Enabled = true };
|
|
Assert.Equal((JobKindPriority.High, JobKindPriority.Normal), (queue.LevelOf(JobKind.Render), queue.LevelOf(JobKind.Scan)));
|
|
await using var engine = new JobEngine(db, [scan, render],
|
|
new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50), PriorityQueue = () => queue });
|
|
await engine.StartAsync();
|
|
var scanning = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await checkpointed.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
|
|
var rendering = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
|
|
await UntilAsync(() => Volatile.Read(ref render.Running) == 1, "the render starts at once");
|
|
var paused = await db.Jobs.GetAsync(scanning.Id);
|
|
Assert.Equal((JobState.Queued, "2", (string?)null), (paused!.State, paused.Checkpoint, paused.NodeId));
|
|
Assert.Contains(Chronos.Core.Localization.Loc.L("Jobs.Kind.Render"), paused.LogTail);
|
|
|
|
// The scan waits while the render runs, then resumes from its checkpoint without being asked.
|
|
go.Release(10);
|
|
await Task.Delay(300);
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(scanning.Id))!.State);
|
|
renderRelease.Release();
|
|
await WaitForStateAsync(db, rendering.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, scanning.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
lock (scan.SeenCheckpoints) Assert.SequenceEqual([null, "2"], scan.SeenCheckpoints);
|
|
Assert.Equal(1, scan.Completed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OnlyAHigherLevelInterruptsAndNeverTheJobsOwnFollowUps()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var release = new SemaphoreSlim(0);
|
|
var queued = new TaskCompletionSource<Job?>();
|
|
// The light analysis queues a deep analysis of the same sequence, which the user put higher.
|
|
var light = new CallbackHandler(JobKind.LightAnalyse, async c =>
|
|
{
|
|
queued.TrySetResult(await c.EnqueueFollowUpAsync(JobKind.DeepAnalyse, [Guid.NewGuid()], "Roof", null, JobPriority.Background));
|
|
await release.WaitAsync(c.Cancellation);
|
|
});
|
|
var deep = new GatedHandler(JobKind.DeepAnalyse, release);
|
|
var thumbs = new GatedHandler(JobKind.Thumbs, release);
|
|
var queue = new JobPriorityQueue
|
|
{
|
|
Enabled = true,
|
|
Levels = new Dictionary<JobKind, JobKindPriority> { [JobKind.DeepAnalyse] = JobKindPriority.High, [JobKind.Thumbs] = JobKindPriority.Normal },
|
|
};
|
|
await using var engine = new JobEngine(db, [light, deep, thumbs],
|
|
new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50), PriorityQueue = () => queue });
|
|
await engine.StartAsync();
|
|
var analysing = await engine.EnqueueAsync(JobKind.LightAnalyse, []);
|
|
var followUp = Assert.NotNull(await queued.Task.WaitAsync(TimeSpan.FromSeconds(10)));
|
|
// A job of the same level waits too.
|
|
var drawing = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.User);
|
|
await Task.Delay(400);
|
|
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(analysing.Id))!.State);
|
|
Assert.Equal((0, 0), (Volatile.Read(ref deep.Started), Volatile.Read(ref thumbs.Started)));
|
|
Assert.False(await engine.ReviewAsync(context.Cancellation));
|
|
|
|
// Another deep analysis, not its own, interrupts it.
|
|
var other = await engine.EnqueueAsync(JobKind.DeepAnalyse, [Guid.NewGuid()], priority: JobPriority.User);
|
|
await UntilAsync(() => Volatile.Read(ref deep.Running) == 1, "a higher job runs");
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(analysing.Id))!.State);
|
|
release.Release(20);
|
|
foreach (var id in new[] { analysing.Id, followUp.Id, drawing.Id, other.Id }) await WaitForStateAsync(db, id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
[Test]
|
|
public async Task RequeueingAYieldedJobLeavesPausesAndOtherNodesAlone()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(job);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc"));
|
|
Assert.False(await db.Jobs.RequeueAsync(job.Id, "other"));
|
|
await db.Jobs.SetStateAsync(job.Id, JobState.Paused);
|
|
Assert.False(await db.Jobs.RequeueAsync(job.Id, "pc"));
|
|
Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(job.Id))!.State);
|
|
await db.Jobs.SetStateAsync(job.Id, JobState.Queued);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc"));
|
|
Assert.True(await db.Jobs.RequeueAsync(job.Id, "pc"));
|
|
Assert.Equal((JobState.Queued, (string?)null), ((await db.Jobs.GetAsync(job.Id))!.State, (await db.Jobs.GetAsync(job.Id))!.NodeId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task HandlerFailureMarksJobFailedWithMessage()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Render, []);
|
|
var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("encoder lost", failed.Error);
|
|
Assert.Contains("encoder lost", failed.LogTail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task StartingKeepsTheHistoryAndKnowsItsKindsBefore()
|
|
{
|
|
// The housekeeping schedule's job-history step purges finished jobs now (owner review 2, R10), not the engine's start.
|
|
await using var db = await OpenAsync();
|
|
var old = new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = JobKind.Scan,
|
|
State = JobState.Done,
|
|
Created = DateTimeOffset.UtcNow.AddDays(-400),
|
|
Finished = DateTimeOffset.UtcNow.AddDays(-400),
|
|
};
|
|
await db.Jobs.EnqueueAsync(old);
|
|
await using var engine = new JobEngine(db, [new CountingHandler(JobKind.Scan, 1), new CountingHandler(JobKind.Verify, 1)],
|
|
new JobEngineOptions { WorkerId = "t", Kinds = [JobKind.Scan] });
|
|
Assert.True(engine.CanRun(JobKind.Scan));
|
|
Assert.False(engine.CanRun(JobKind.Verify), "filtered out");
|
|
Assert.False(engine.CanRun(JobKind.Render), "no handler");
|
|
await engine.StartAsync();
|
|
Assert.NotNull(await db.Jobs.GetAsync(old.Id));
|
|
|
|
var scheduled = await engine.EnqueueAsync(JobKind.Scan, [], null, null, JobPriority.Background, null, Guid.CreateVersion7());
|
|
Assert.Equal(scheduled.ScheduleId, (await db.Jobs.GetAsync(scheduled.Id))!.ScheduleId);
|
|
}
|
|
}
|