Owner review 6 asks Chronos to recognise the same time-lapse held under different ids on the server and on this PC, compare the copies, and keep them apart or integrate and sync them. This lays the shared foundation the later parcels build on, without any UI or detection yet. - Migration 0012 adds found_copies (local-server and local-local pairs with evidence, scores, answer, policy and the 404 record) and copy_probes; every existing link with a local sequence becomes a Linked pair, so a server deletion and the per-link policy have a place. - FoundCopyStore keeps the pairs: upserts that never touch the answer, a compare-and-set of the state with an Integrating backstop (a failed or cancelled job reads as the previous state), link promotion and demotion, the once-per-refresh 404 count, the merge's links stage (session keys, links reset, pairs moved, refused when both are linked on one server), the local overlap queries and the probes. SyncLinkStore.RekeyAsync moves pairs and probes with the links in the same transaction. - Contracts for detection, review, plans, the Integrate and MergeSequences jobs (kinds 25/26, High level, new detail steps) and the Upload all rule that keeps suspected, kept-apart, deleted, busy and server-born sequences here. - Wire: POST photos/lookup, uploads into a target (IntoTarget/Copied, trailing and optional), frames/delete on the client, capabilities photo-lookup and upload-into, cached per remote. - The trash plans and deletes accept the job that holds the sequence (its own hold is not Busy) and a frames version checked under the lease; a Background job never interrupts a running one. - Settings for linking, the photo policy and projects; the en and it keys of design section 9. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
549 lines
28 KiB
C#
549 lines
28 KiB
C#
using Chronos.Core.Model;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Jobs;
|
|
|
|
[Category("jobs")]
|
|
public sealed class JobEngineTests(TestContext context)
|
|
{
|
|
private sealed class CountingHandler(JobKind kind, int steps, Func<int, CancellationToken, Task>? onStep = null) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
public List<string?> SeenCheckpoints { get; } = [];
|
|
public int Completed;
|
|
|
|
public async Task RunAsync(JobContext c)
|
|
{
|
|
lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint);
|
|
var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0;
|
|
for (var i = start; i < steps; i++)
|
|
{
|
|
c.Cancellation.ThrowIfCancellationRequested();
|
|
if (onStep is not null) await onStep(i, c.Cancellation);
|
|
await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps);
|
|
}
|
|
Interlocked.Increment(ref Completed);
|
|
}
|
|
}
|
|
|
|
/// <summary>Takes only jobs whose parameters say they are its own.</summary>
|
|
private sealed class SelectiveHandler(JobKind kind) : ISelectiveJobHandler
|
|
{
|
|
public const string Mine = """{"mine":true}""";
|
|
|
|
public JobKind Kind => kind;
|
|
|
|
public ValueTask<bool> AcceptsAsync(Job job, CancellationToken cancellationToken) => ValueTask.FromResult(job.ParametersJson == Mine);
|
|
|
|
public Task RunAsync(JobContext c) => Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>Cannot judge jobs without parameters (as a render handler with a malformed preset) and fails them when run.</summary>
|
|
private sealed class FragileHandler(JobKind kind) : ISelectiveJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
|
|
public ValueTask<bool> AcceptsAsync(Job job, CancellationToken cancellationToken) =>
|
|
job.ParametersJson?.Length > 0 ? ValueTask.FromResult(true) : throw new ArgumentNullException(nameof(job));
|
|
|
|
public Task RunAsync(JobContext c) =>
|
|
c.Job.ParametersJson is null ? throw new InvalidDataException("no parameters") : Task.CompletedTask;
|
|
}
|
|
|
|
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
|
|
public Task RunAsync(JobContext context) => run(context);
|
|
}
|
|
|
|
private async Task<ChronosDatabase> OpenAsync() =>
|
|
await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
|
|
|
|
private static async Task<Job> WaitForStateAsync(ChronosDatabase db, Guid id, JobState state, TimeSpan timeout)
|
|
{
|
|
var until = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < until)
|
|
{
|
|
var job = await db.Jobs.GetAsync(id);
|
|
if (job?.State == state) return job;
|
|
await Task.Delay(20);
|
|
}
|
|
Assert.Fail($"Job {id} did not reach {state}");
|
|
return null!;
|
|
}
|
|
|
|
[Test]
|
|
public async Task RunsQueuedJobsToCompletion()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var handler = new CountingHandler(JobKind.Verify, 5);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
|
var progress = 0;
|
|
engine.Progress += _ => Interlocked.Increment(ref progress);
|
|
await engine.StartAsync();
|
|
var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A");
|
|
var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B");
|
|
var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(2, handler.Completed);
|
|
Assert.Equal(5L, doneA.ProgressDone);
|
|
Assert.Greater(progress, 0);
|
|
}
|
|
|
|
[Test]
|
|
public async Task CancelStopsARunningJob()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var started = new TaskCompletionSource();
|
|
var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) =>
|
|
{
|
|
if (i == 0) started.TrySetResult();
|
|
await Task.Delay(10, ct);
|
|
});
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await started.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
await engine.CancelAsync(job.Id);
|
|
await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5));
|
|
Assert.Equal(0, handler.Completed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task PauseAndResumeContinueFromCheckpoint()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var gate = new SemaphoreSlim(0);
|
|
var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) =>
|
|
{
|
|
if (i == 10) await gate.WaitAsync(ct);
|
|
});
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Proxies, []);
|
|
var until = DateTime.UtcNow.AddSeconds(10);
|
|
while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20);
|
|
await engine.PauseAsync(job.Id);
|
|
var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5));
|
|
Assert.Equal("10", paused.Checkpoint);
|
|
gate.Release(10);
|
|
await engine.ResumeAsync(job.Id);
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("10", handler.SeenCheckpoints[^1]);
|
|
}
|
|
|
|
[Test]
|
|
public async Task CrashedJobsResumeAfterRestart()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(job);
|
|
var claimed = await db.Jobs.ClaimNextAsync("t");
|
|
Assert.NotNull(claimed);
|
|
await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null);
|
|
// Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it.
|
|
var handler = new CountingHandler(JobKind.Thumbs, 12);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("7", handler.SeenCheckpoints.Single());
|
|
}
|
|
|
|
[Test]
|
|
public async Task JobsOfNumberedWorkersResumeAfterRestart()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(job);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc#1"));
|
|
var other = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(other);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc2#0"));
|
|
// Restarting node "pc" (two workers: pc#0, pc#1) requeues pc#1's job, never another node's.
|
|
var handler = new CountingHandler(JobKind.Thumbs, 3);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "pc" });
|
|
await engine.StartAsync();
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(other.Id))!.State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OnlyKindsWithAHandlerAreClaimed()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
await using (var idle = new JobEngine(db, [], new JobEngineOptions { Workers = 1, WorkerId = "idle" }))
|
|
{
|
|
await idle.StartAsync();
|
|
var render = await idle.EnqueueAsync(JobKind.Render, []);
|
|
await Task.Delay(300);
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(render.Id))!.State);
|
|
}
|
|
var handler = new CountingHandler(JobKind.Thumbs, 1);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Kinds = [JobKind.Thumbs, JobKind.Render] });
|
|
await engine.StartAsync();
|
|
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
|
|
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
var pending = (await db.Jobs.ListAsync(JobState.Queued)).Items;
|
|
Assert.Count(1, pending);
|
|
Assert.Equal(JobKind.Render, pending[0].Kind);
|
|
}
|
|
|
|
[Test]
|
|
public async Task SelectiveHandlersLeaveDeclinedJobsQueuedWithoutHoldingUpOthers()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
await using var engine = new JobEngine(db, [new SelectiveHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
|
|
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
var declined = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: """{"mine":false}""", priority: JobPriority.User);
|
|
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.Background);
|
|
var accepted = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: SelectiveHandler.Mine, priority: JobPriority.User);
|
|
await engine.StartAsync();
|
|
|
|
var render = await WaitForStateAsync(db, accepted.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
var other = await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.True(render.Started <= other.Started, "claim order: priority first, across selective and other kinds");
|
|
await Task.Delay(200);
|
|
var waiting = (await db.Jobs.GetAsync(declined.Id))!;
|
|
Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASelectiveHandlerThatThrowsDoesNotStopTheWorkers()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
await using var engine = new JobEngine(db, [new FragileHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
|
|
new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
var broken = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
|
|
await engine.StartAsync();
|
|
|
|
var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("no parameters", failed.Error);
|
|
// The workers are still there for every kind.
|
|
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
|
|
var render = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: "{}");
|
|
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, render.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
[Test]
|
|
public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var target = Guid.CreateVersion7();
|
|
Job? thumbs = null, light = null;
|
|
var scan = new CallbackHandler(JobKind.Scan, async c =>
|
|
{
|
|
thumbs = await c.EnqueueFollowUpAsync(JobKind.Thumbs, [target], "Roof", null, JobPriority.Background);
|
|
light = await c.EnqueueFollowUpAsync(JobKind.LightAnalyse, [target], "Roof", null, JobPriority.Background);
|
|
});
|
|
await using var engine = new JobEngine(db, [scan, new CountingHandler(JobKind.Thumbs, 1)],
|
|
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
await engine.StartAsync();
|
|
var user = Guid.CreateVersion7();
|
|
var job = await engine.EnqueueAsync(JobKind.Scan, [], createdBy: user);
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
|
|
Assert.Null(light);
|
|
var followUp = await WaitForStateAsync(db, Assert.NotNull(thumbs).Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal((user, JobPriority.Background, "Roof"), (followUp.CreatedBy!.Value, followUp.Priority, followUp.TargetLabel!));
|
|
Assert.SequenceEqual([target], followUp.TargetIds);
|
|
Assert.Empty((await db.Jobs.ListAsync(null, JobKind.LightAnalyse)).Items);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A follow-up of work already done (a sync download's frame swapped to another encoding, whose thumbnails it cleared) is
|
|
/// queued whatever stops its job meanwhile: a cancel arriving right after the work leaves it queued, and it runs.
|
|
/// </summary>
|
|
[Test]
|
|
public async Task AFollowUpQueuedWhateverStopsItsJobOutlivesACancel()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var target = Guid.CreateVersion7();
|
|
var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var queued = new TaskCompletionSource<Job?>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var download = new CallbackHandler(JobKind.Scan, async c =>
|
|
{
|
|
started.TrySetResult();
|
|
try
|
|
{
|
|
await Task.Delay(Timeout.InfiniteTimeSpan, c.Cancellation);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
// The work is done; the cancel came before its follow-up was queued.
|
|
}
|
|
try
|
|
{
|
|
queued.TrySetResult(await c.EnqueueFollowUpAsync(JobKind.Thumbs, [target], "Roof", null, JobPriority.Background, CancellationToken.None));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
queued.TrySetException(ex);
|
|
}
|
|
c.Cancellation.ThrowIfCancellationRequested();
|
|
});
|
|
await using var engine = new JobEngine(db, [download, new CountingHandler(JobKind.Thumbs, 1)],
|
|
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await started.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
|
|
await engine.CancelAsync(job.Id);
|
|
|
|
var followUp = Assert.NotNull(await queued.Task.WaitAsync(TimeSpan.FromSeconds(10)), "queued although its job was cancelled");
|
|
await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5));
|
|
Assert.SequenceEqual([target], (await WaitForStateAsync(db, followUp.Id, JobState.Done, TimeSpan.FromSeconds(10))).TargetIds);
|
|
}
|
|
|
|
/// <summary>Waits until <paramref name="condition"/> holds, failing with <paramref name="what"/> after ten seconds.</summary>
|
|
private static async Task UntilAsync(Func<bool> condition, string what)
|
|
{
|
|
var until = DateTime.UtcNow.AddSeconds(10);
|
|
while (!condition())
|
|
{
|
|
Assert.True(DateTime.UtcNow < until, what);
|
|
await Task.Delay(20);
|
|
}
|
|
}
|
|
|
|
/// <summary>A handler whose jobs run until released, counting those running and started, by kind.</summary>
|
|
private sealed class GatedHandler(JobKind kind, SemaphoreSlim release) : IJobHandler
|
|
{
|
|
public int Running;
|
|
public int Started;
|
|
|
|
public JobKind Kind => kind;
|
|
|
|
public async Task RunAsync(JobContext c)
|
|
{
|
|
Interlocked.Increment(ref Running);
|
|
Interlocked.Increment(ref Started);
|
|
try
|
|
{
|
|
await release.WaitAsync(c.Cancellation);
|
|
}
|
|
finally
|
|
{
|
|
Interlocked.Decrement(ref Running);
|
|
}
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task JobsRunOneAtATimeAndPausingStartsTheNext()
|
|
{
|
|
// Owner review 1 (R2): one job at a time by default; pausing the running job lets the next one start.
|
|
await using var db = await OpenAsync();
|
|
var release = new SemaphoreSlim(0);
|
|
var handler = new GatedHandler(JobKind.Thumbs, release);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
Assert.Equal(1, engine.Options.Workers);
|
|
await engine.StartAsync();
|
|
var jobs = new List<Job>();
|
|
for (var i = 0; i < 3; i++) jobs.Add(await engine.EnqueueAsync(JobKind.Thumbs, []));
|
|
|
|
await UntilAsync(() => Volatile.Read(ref handler.Running) == 1, "the first job runs");
|
|
await Task.Delay(300);
|
|
Assert.Equal((1, 1), (Volatile.Read(ref handler.Running), Volatile.Read(ref handler.Started)));
|
|
Assert.Count(1, engine.RunningJobIds);
|
|
var first = engine.RunningJobIds.First();
|
|
Assert.Equal("pc", (await db.Jobs.GetAsync(first))!.NodeId);
|
|
|
|
await engine.PauseAsync(first);
|
|
await UntilAsync(() => Volatile.Read(ref handler.Started) == 2 && Volatile.Read(ref handler.Running) == 1, "the next job starts");
|
|
Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(first))!.State);
|
|
|
|
release.Release(10);
|
|
foreach (var job in jobs.Where(j => j.Id != first)) await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(first))!.State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task WithoutThePriorityQueueARunningJobIsNeverInterrupted()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var release = new SemaphoreSlim(0);
|
|
var scan = new GatedHandler(JobKind.Scan, release);
|
|
var render = new GatedHandler(JobKind.Render, release);
|
|
await using var engine = new JobEngine(db, [scan, render], new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
await engine.StartAsync();
|
|
var scanning = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await UntilAsync(() => Volatile.Read(ref scan.Running) == 1, "the scan runs");
|
|
var rendering = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
|
|
await Task.Delay(400);
|
|
Assert.Equal((1, 0), (Volatile.Read(ref scan.Running), Volatile.Read(ref render.Started)));
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(rendering.Id))!.State);
|
|
|
|
release.Release(10);
|
|
await WaitForStateAsync(db, scanning.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, rendering.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(1, Volatile.Read(ref scan.Started));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AHigherJobPausesTheRunningOneWhichResumesFromItsCheckpoint()
|
|
{
|
|
// Owner review 1 (R2): with render at the highest level, starting a render while a scan runs pauses the scan
|
|
// without asking, renders, and resumes the scan by itself where it was.
|
|
await using var db = await OpenAsync();
|
|
var checkpointed = new TaskCompletionSource();
|
|
var go = new SemaphoreSlim(0);
|
|
var scan = new CountingHandler(JobKind.Scan, 4, async (i, ct) =>
|
|
{
|
|
if (i < 2) return;
|
|
checkpointed.TrySetResult();
|
|
await go.WaitAsync(ct);
|
|
});
|
|
var renderRelease = new SemaphoreSlim(0);
|
|
var render = new GatedHandler(JobKind.Render, renderRelease);
|
|
var queue = new JobPriorityQueue { Enabled = true };
|
|
Assert.Equal((JobKindPriority.High, JobKindPriority.Normal), (queue.LevelOf(JobKind.Render), queue.LevelOf(JobKind.Scan)));
|
|
await using var engine = new JobEngine(db, [scan, render],
|
|
new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50), PriorityQueue = () => queue });
|
|
await engine.StartAsync();
|
|
var scanning = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await checkpointed.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
|
|
var rendering = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
|
|
await UntilAsync(() => Volatile.Read(ref render.Running) == 1, "the render starts at once");
|
|
var paused = await db.Jobs.GetAsync(scanning.Id);
|
|
Assert.Equal((JobState.Queued, "2", (string?)null), (paused!.State, paused.Checkpoint, paused.NodeId));
|
|
Assert.Contains(Chronos.Core.Localization.Loc.L("Jobs.Kind.Render"), paused.LogTail);
|
|
|
|
// The scan waits while the render runs, then resumes from its checkpoint without being asked.
|
|
go.Release(10);
|
|
await Task.Delay(300);
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(scanning.Id))!.State);
|
|
renderRelease.Release();
|
|
await WaitForStateAsync(db, rendering.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, scanning.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
lock (scan.SeenCheckpoints) Assert.SequenceEqual([null, "2"], scan.SeenCheckpoints);
|
|
Assert.Equal(1, scan.Completed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OnlyAHigherLevelInterruptsAndNeverTheJobsOwnFollowUps()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var release = new SemaphoreSlim(0);
|
|
var queued = new TaskCompletionSource<Job?>();
|
|
// The light analysis queues a deep analysis of the same sequence, which the user put higher.
|
|
var light = new CallbackHandler(JobKind.LightAnalyse, async c =>
|
|
{
|
|
queued.TrySetResult(await c.EnqueueFollowUpAsync(JobKind.DeepAnalyse, [Guid.NewGuid()], "Roof", null, JobPriority.Background));
|
|
await release.WaitAsync(c.Cancellation);
|
|
});
|
|
var deep = new GatedHandler(JobKind.DeepAnalyse, release);
|
|
var thumbs = new GatedHandler(JobKind.Thumbs, release);
|
|
var queue = new JobPriorityQueue
|
|
{
|
|
Enabled = true,
|
|
Levels = new Dictionary<JobKind, JobKindPriority> { [JobKind.DeepAnalyse] = JobKindPriority.High, [JobKind.Thumbs] = JobKindPriority.Normal },
|
|
};
|
|
await using var engine = new JobEngine(db, [light, deep, thumbs],
|
|
new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50), PriorityQueue = () => queue });
|
|
await engine.StartAsync();
|
|
var analysing = await engine.EnqueueAsync(JobKind.LightAnalyse, []);
|
|
var followUp = Assert.NotNull(await queued.Task.WaitAsync(TimeSpan.FromSeconds(10)));
|
|
// A job of the same level waits too.
|
|
var drawing = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.User);
|
|
await Task.Delay(400);
|
|
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(analysing.Id))!.State);
|
|
Assert.Equal((0, 0), (Volatile.Read(ref deep.Started), Volatile.Read(ref thumbs.Started)));
|
|
Assert.False(await engine.ReviewAsync(context.Cancellation));
|
|
|
|
// Another deep analysis, not its own, interrupts it.
|
|
var other = await engine.EnqueueAsync(JobKind.DeepAnalyse, [Guid.NewGuid()], priority: JobPriority.User);
|
|
await UntilAsync(() => Volatile.Read(ref deep.Running) == 1, "a higher job runs");
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(analysing.Id))!.State);
|
|
release.Release(20);
|
|
foreach (var id in new[] { analysing.Id, followUp.Id, drawing.Id, other.Id }) await WaitForStateAsync(db, id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ABackgroundJobNeverInterruptsEvenAtAHigherLevel()
|
|
{
|
|
// Owner review 6 (#73): an Integrate queued by itself at the end of a sync is High but Background; it waits for the
|
|
// running job instead of pausing it.
|
|
await using var db = await OpenAsync();
|
|
var release = new SemaphoreSlim(0);
|
|
var scan = new GatedHandler(JobKind.Scan, release);
|
|
var download = new GatedHandler(JobKind.Download, release);
|
|
var queue = new JobPriorityQueue { Enabled = true };
|
|
Assert.True(queue.Interrupts(JobKind.Download, JobKind.Scan));
|
|
await using var engine = new JobEngine(db, [scan, download],
|
|
new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50), PriorityQueue = () => queue });
|
|
await engine.StartAsync();
|
|
var scanning = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await UntilAsync(() => Volatile.Read(ref scan.Running) == 1, "the scan runs");
|
|
var downloading = await engine.EnqueueAsync(JobKind.Download, [], priority: JobPriority.Background);
|
|
await Task.Delay(400);
|
|
Assert.Equal((1, 0), (Volatile.Read(ref scan.Running), Volatile.Read(ref download.Started)));
|
|
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(scanning.Id))!.State);
|
|
Assert.False(await engine.ReviewAsync(context.Cancellation));
|
|
|
|
release.Release(10);
|
|
await WaitForStateAsync(db, scanning.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, downloading.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(1, Volatile.Read(ref scan.Started));
|
|
}
|
|
|
|
[Test]
|
|
public async Task RequeueingAYieldedJobLeavesPausesAndOtherNodesAlone()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(job);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc"));
|
|
Assert.False(await db.Jobs.RequeueAsync(job.Id, "other"));
|
|
await db.Jobs.SetStateAsync(job.Id, JobState.Paused);
|
|
Assert.False(await db.Jobs.RequeueAsync(job.Id, "pc"));
|
|
Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(job.Id))!.State);
|
|
await db.Jobs.SetStateAsync(job.Id, JobState.Queued);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc"));
|
|
Assert.True(await db.Jobs.RequeueAsync(job.Id, "pc"));
|
|
Assert.Equal((JobState.Queued, (string?)null), ((await db.Jobs.GetAsync(job.Id))!.State, (await db.Jobs.GetAsync(job.Id))!.NodeId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task HandlerFailureMarksJobFailedWithMessage()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Render, []);
|
|
var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("encoder lost", failed.Error);
|
|
Assert.Contains("encoder lost", failed.LogTail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task StartingKeepsTheHistoryAndKnowsItsKindsBefore()
|
|
{
|
|
// The housekeeping schedule's job-history step purges finished jobs now (owner review 2, R10), not the engine's start.
|
|
await using var db = await OpenAsync();
|
|
var old = new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = JobKind.Scan,
|
|
State = JobState.Done,
|
|
Created = DateTimeOffset.UtcNow.AddDays(-400),
|
|
Finished = DateTimeOffset.UtcNow.AddDays(-400),
|
|
};
|
|
await db.Jobs.EnqueueAsync(old);
|
|
await using var engine = new JobEngine(db, [new CountingHandler(JobKind.Scan, 1), new CountingHandler(JobKind.Verify, 1)],
|
|
new JobEngineOptions { WorkerId = "t", Kinds = [JobKind.Scan] });
|
|
Assert.True(engine.CanRun(JobKind.Scan));
|
|
Assert.False(engine.CanRun(JobKind.Verify), "filtered out");
|
|
Assert.False(engine.CanRun(JobKind.Render), "no handler");
|
|
await engine.StartAsync();
|
|
Assert.NotNull(await db.Jobs.GetAsync(old.Id));
|
|
|
|
var scheduled = await engine.EnqueueAsync(JobKind.Scan, [], null, null, JobPriority.Background, null, Guid.CreateVersion7());
|
|
Assert.Equal(scheduled.ScheduleId, (await db.Jobs.GetAsync(scheduled.Id))!.ScheduleId);
|
|
}
|
|
}
|