Files
Chronos/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs
T
Alby96andClaude Opus 5.5 8ea25228eb fix(convert): reserve the photos still writing, keep photo ids out of the job row, never copy a source twice
Stage A's space guard reserved, before each photo, only that photo and the
ones after it, so at the end of a sequence it counted a single photo while
up to twice the parallelism were still converting: a volume filling
meanwhile could go into the reserve. It now reserves the photo and every
photo started whose converted file has not landed yet (a file lands whole,
by a rename, before it is verified), so nothing in flight is missed and
nothing landed is counted twice, at the end of a sequence as anywhere.

The confirmed photo ids (about 21 MB for a million photos) lived in the
job's parameters, read by every dry run's busy check, by each sequence turn
of the conversion and by the job lists. They are now written whole, under a
job id chosen before the job is queued, into convert-<job>.photos beside
the journal, and go with it; the row keeps each sequence's digest, so the
job still converts exactly the photos confirmed (without the file, the
whole plan made again must match). The busy check reads only ids, kinds and
targets (and a verify's parameters), never the asking job's row; a
conversion's turn reads only its creation time; the server's job list and
dashboard and the engine's queue peeks read jobs without parameters.

A retried New sequence conversion could make a second copy. The journal now
names the copy as soon as its photos are filed; a retry finishes that copy
whatever changed in the source since, never remakes one deleted since, and
needs no space again for what it holds; and a source another conversion
copied in the same format after this one was confirmed is done with, with
no copy made (new outcome Superseded, with its job log line).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 09:36:00 +01:00

341 lines
19 KiB
C#

using System.Collections.Concurrent;
using Chronos.Core.Model;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Testing;
namespace Chronos.Data.Tests.Repositories;
public abstract class JobRepositoryTests(TestContext context) : RepositoryTest(context)
{
[Test]
public async Task JobsRoundTripEveryField()
{
var db = await OpenChronosAsync();
var job = TestData.Job(JobKind.Render, JobPriority.User) with
{
TargetIds = [Guid.CreateVersion7(), Guid.CreateVersion7()],
TargetLabel = "Harbour sunset",
ParametersJson = "{\"preset\": \"4K\"}",
ProgressDone = 5,
ProgressTotal = 10,
ItemsPerSecond = 2.5,
Eta = TimeSpan.FromSeconds(2),
Checkpoint = "frame:5",
LogTail = "line 1\nline 2",
CreatedBy = Guid.CreateVersion7(),
NodeId = "node-a",
Attempts = 1,
Error = "boom",
State = JobState.Failed,
Started = TestData.Now.AddMinutes(1),
Finished = TestData.Now.AddMinutes(2),
ScheduleId = Guid.CreateVersion7(),
};
await db.Jobs.EnqueueAsync(job, Ct);
var read = Assert.NotNull(await db.Jobs.GetAsync(job.Id, Ct));
Assert.SequenceEqual(job.TargetIds, read.TargetIds);
Assert.Equal(job with { TargetIds = read.TargetIds }, read);
Assert.Null((await db.Jobs.GetAsync((await EnqueueAsync(db, TestData.Job())).Id, Ct))!.ScheduleId);
}
[Test]
public async Task JobsOfAScheduleAreListedNewestFirst()
{
var db = await OpenChronosAsync();
var schedule = Guid.CreateVersion7();
var older = await EnqueueAsync(db, TestData.Job(created: TestData.Now.AddMinutes(-5)) with { ScheduleId = schedule });
var newer = await EnqueueAsync(db, TestData.Job() with { ScheduleId = schedule });
await EnqueueAsync(db, TestData.Job());
Assert.SequenceEqual([newer.Id, older.Id], (await db.Jobs.ListByScheduleAsync(schedule, cancellationToken: Ct)).Select(j => j.Id));
Assert.Empty(await db.Jobs.ListByScheduleAsync(Guid.CreateVersion7(), cancellationToken: Ct));
}
[Test]
public async Task JobsOfAScheduleArePagedNewestFirst()
{
// The runs a schedule's row lists in the admin panel (owner review 2, R10).
var db = await OpenChronosAsync();
var schedule = Guid.CreateVersion7();
var jobs = new List<Job>();
for (var i = 0; i < 3; i++) jobs.Add(await EnqueueAsync(db, TestData.Job(created: TestData.Now.AddMinutes(i)) with { ScheduleId = schedule }));
await EnqueueAsync(db, TestData.Job(created: TestData.Now.AddMinutes(5)));
var first = await db.Jobs.PageByScheduleAsync(schedule, 2, null, Ct);
Assert.SequenceEqual([jobs[2].Id, jobs[1].Id], first.Items.Select(j => j.Id));
var second = await db.Jobs.PageByScheduleAsync(schedule, 2, Assert.NotNull(first.NextCursor), Ct);
Assert.SequenceEqual([jobs[0].Id], second.Items.Select(j => j.Id));
Assert.Null(second.NextCursor);
Assert.Empty((await db.Jobs.PageByScheduleAsync(Guid.CreateVersion7(), cancellationToken: Ct)).Items);
// A cursor of another listing is refused.
var other = await db.Jobs.ListAsync(null, 1, null, Ct);
await Assert.ThrowsAsync<FormatException>(() => db.Jobs.PageByScheduleAsync(schedule, 2, Assert.NotNull(other.NextCursor), Ct));
}
[Test]
public async Task PendingJobsAreFoundByKindAndTarget()
{
// A schedule queues nothing while a job doing the same work waits, runs or is paused (owner review 2, R10).
var db = await OpenChronosAsync();
Guid archive = Guid.CreateVersion7(), a = Guid.CreateVersion7(), b = Guid.CreateVersion7(), c = Guid.CreateVersion7();
await EnqueueAsync(db, TestData.Job(JobKind.Scan) with { TargetIds = [archive] });
await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [a, b], State = JobState.Paused });
await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [c], State = JobState.Done, Finished = TestData.Now });
await EnqueueAsync(db, TestData.Job(JobKind.Verify) with { TargetIds = [archive], State = JobState.Failed, Finished = TestData.Now });
Assert.True(await db.Jobs.HasPendingAsync([JobKind.Scan, JobKind.Rebuild], [archive], Ct));
Assert.False(await db.Jobs.HasPendingAsync([JobKind.Rebuild], [archive], Ct));
Assert.False(await db.Jobs.HasPendingAsync([JobKind.Verify], [archive], Ct), "a failed job is not pending");
Assert.True(await db.Jobs.HasPendingAsync([JobKind.Thumbs], [b], Ct), "the second of a job's targets");
Assert.True(await db.Jobs.HasPendingAsync([JobKind.Thumbs], null, Ct));
Assert.False(await db.Jobs.HasPendingAsync([JobKind.Thumbs], [c], Ct), "a done job is not pending");
Assert.False(await db.Jobs.HasPendingAsync([JobKind.Scan], [], Ct));
var claimed = await db.Jobs.ClaimNextAsync("node", [JobKind.Scan], Ct);
Assert.Equal(JobState.Running, claimed!.State);
Assert.True(await db.Jobs.HasPendingAsync([JobKind.Scan], [archive], Ct), "a running job is pending");
Assert.SequenceEqual(new[] { a, b }.Order(), (await db.Jobs.ListPendingTargetsAsync(JobKind.Thumbs, Ct)).Order());
Assert.Empty(await db.Jobs.ListPendingTargetsAsync(JobKind.Verify, Ct));
}
private async Task<Job> EnqueueAsync(ChronosDatabase db, Job job)
{
await db.Jobs.EnqueueAsync(job, Ct);
return job;
}
[Test]
public async Task HoldsNameTheTargetsOfRunningAndPausedJobsAndOnlyAVerifysParameters()
{
// What a dry run counts as busy, read at every one and at every sequence a conversion comes to: never the parameters
// a conversion or an import keeps, nor the row of the job asking.
var db = await OpenChronosAsync();
Guid a = Guid.CreateVersion7(), b = Guid.CreateVersion7(), archive = Guid.CreateVersion7();
var convert = await EnqueueAsync(db, TestData.Job(JobKind.Convert) with { TargetIds = [a, b], ParametersJson = "{\"fingerprint\": \"f\"}", State = JobState.Running });
var verify = await EnqueueAsync(db, TestData.Job(JobKind.Verify) with { TargetIds = [archive], ParametersJson = "{\"sequenceIds\": []}", State = JobState.Paused });
await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [a] });
await EnqueueAsync(db, TestData.Job(JobKind.Scan) with { TargetIds = [archive], State = JobState.Done, Finished = TestData.Now });
var holds = await db.Jobs.ListHoldsAsync(cancellationToken: Ct);
Assert.SequenceEqual(new[] { convert.Id, verify.Id }.Order(), holds.Select(h => h.Id).Order());
var running = holds.Single(h => h.Id == convert.Id);
Assert.Equal((JobKind.Convert, (string?)null), (running.Kind, running.ParametersJson));
Assert.SequenceEqual([a, b], running.TargetIds);
var paused = holds.Single(h => h.Id == verify.Id);
Assert.Equal((JobKind.Verify, verify.ParametersJson), (paused.Kind, paused.ParametersJson));
Assert.SequenceEqual([verify.Id], (await db.Jobs.ListHoldsAsync(convert.Id, Ct)).Select(h => h.Id));
Assert.Equal(convert.Created, await db.Jobs.GetCreatedAsync(convert.Id, Ct));
Assert.Null(await db.Jobs.GetCreatedAsync(Guid.CreateVersion7(), Ct));
}
[Test]
public async Task BriefListsReadEveryFieldButTheParameters()
{
var db = await OpenChronosAsync();
var job = TestData.Job(JobKind.Convert, JobPriority.User) with
{
TargetIds = [Guid.CreateVersion7(), Guid.CreateVersion7()],
TargetLabel = "Harbour sunset",
ParametersJson = "{\"fingerprint\": \"f\"}",
LogTail = "line 1",
CreatedBy = Guid.CreateVersion7(),
ScheduleId = Guid.CreateVersion7(),
};
await db.Jobs.EnqueueAsync(job, Ct);
var brief = Assert.NotNull((await db.Jobs.ListBriefAsync(JobState.Queued, JobKind.Convert, 10, null, Ct)).Items.SingleOrDefault());
Assert.SequenceEqual(job.TargetIds, brief.TargetIds);
Assert.Equal(job with { TargetIds = brief.TargetIds, ParametersJson = null }, brief);
Assert.Equal(job.ParametersJson, (await db.Jobs.ListAsync(JobState.Queued, JobKind.Convert, 10, null, Ct)).Items.Single().ParametersJson);
Assert.Empty((await db.Jobs.ListBriefAsync(JobState.Running, null, 10, null, Ct)).Items);
var next = Assert.NotNull((await db.Jobs.ListQueuedBriefAsync([JobKind.Convert], 10, Chronos.Data.Jobs.JobPriorityQueue.Off, Ct)).SingleOrDefault());
Assert.Equal((job.Id, (string?)null), (next.Id, next.ParametersJson));
Assert.Empty(await db.Jobs.ListQueuedBriefAsync([], 10, Chronos.Data.Jobs.JobPriorityQueue.Off, Ct));
}
[Test]
public async Task ThePriorityQueueClaimsTheHigherKindsFirst()
{
// Owner review 1 (R2): on, the level of the kind comes before priority and age; off, the strict queue stays.
var db = await OpenChronosAsync();
var oldVerify = TestData.Job(JobKind.Verify, JobPriority.User, TestData.Now.AddMinutes(-30));
var backgroundScan = TestData.Job(JobKind.Scan, JobPriority.Background, TestData.Now.AddMinutes(-20));
var userThumbs = TestData.Job(JobKind.Thumbs, JobPriority.User, TestData.Now.AddMinutes(-10));
var render = TestData.Job(JobKind.Render, JobPriority.Normal, TestData.Now);
foreach (var job in new[] { oldVerify, backgroundScan, userThumbs, render }) await db.Jobs.EnqueueAsync(job, Ct);
var queue = new Chronos.Data.Jobs.JobPriorityQueue { Enabled = true };
JobKind[] kinds = [JobKind.Verify, JobKind.Scan, JobKind.Thumbs, JobKind.Render];
Assert.SequenceEqual([render.Id, userThumbs.Id, backgroundScan.Id, oldVerify.Id], (await db.Jobs.ListQueuedAsync(kinds, 10, queue, Ct)).Select(j => j.Id));
Assert.SequenceEqual([oldVerify.Id, userThumbs.Id, render.Id, backgroundScan.Id], (await db.Jobs.ListQueuedAsync(kinds, 10, Ct)).Select(j => j.Id));
Assert.Equal(userThumbs.Id, (await db.Jobs.ClaimNextAsync("server", [JobKind.Scan, JobKind.Thumbs, JobKind.Verify], queue, Ct))?.Id);
var order = new List<Guid>();
while (await db.Jobs.ClaimNextAsync("server", null, queue, Ct) is { } claimed) order.Add(claimed.Id);
Assert.SequenceEqual([render.Id, backgroundScan.Id, oldVerify.Id], order);
}
[Test]
public async Task ClaimsByPriorityThenAgeAndKind()
{
var db = await OpenChronosAsync();
var oldBackground = TestData.Job(JobKind.Thumbs, JobPriority.Background, TestData.Now.AddMinutes(-10));
var newUser = TestData.Job(JobKind.Render, JobPriority.User, TestData.Now);
var oldUser = TestData.Job(JobKind.Import, JobPriority.User, TestData.Now.AddMinutes(-1));
var normal = TestData.Job(JobKind.Scan, JobPriority.Normal, TestData.Now.AddMinutes(-5));
foreach (var job in new[] { oldBackground, newUser, oldUser, normal }) await db.Jobs.EnqueueAsync(job, Ct);
var renderOnly = await db.Jobs.ClaimNextAsync("node-gpu", [JobKind.Render], Ct);
Assert.Equal(newUser.Id, renderOnly?.Id);
Assert.Equal(JobState.Running, renderOnly!.State);
Assert.Equal("node-gpu", renderOnly.NodeId);
Assert.Equal(1, renderOnly.Attempts);
Assert.NotNull(renderOnly.Started);
Assert.Null(await db.Jobs.ClaimNextAsync("node-gpu", [JobKind.Render], Ct));
var order = new List<Guid>();
while (await db.Jobs.ClaimNextAsync("server", cancellationToken: Ct) is { } claimed) order.Add(claimed.Id);
Assert.SequenceEqual([oldUser.Id, normal.Id, oldBackground.Id], order);
}
[Test]
public async Task SelectiveClaimersListQueuedKindsInClaimOrderAndClaimById()
{
var db = await OpenChronosAsync(multiConnection: true);
var oldBackground = TestData.Job(JobKind.Render, JobPriority.Background, TestData.Now.AddMinutes(-10));
var newUser = TestData.Job(JobKind.Thumbs, JobPriority.User, TestData.Now);
var oldUser = TestData.Job(JobKind.Render, JobPriority.User, TestData.Now.AddMinutes(-1));
var otherKind = TestData.Job(JobKind.Scan, JobPriority.User, TestData.Now.AddMinutes(-20));
foreach (var job in new[] { oldBackground, newUser, oldUser, otherKind }) await db.Jobs.EnqueueAsync(job, Ct);
var queued = await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 10, Ct);
Assert.SequenceEqual([oldUser.Id, newUser.Id, oldBackground.Id], queued.Select(j => j.Id));
Assert.Count(1, await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 1, Ct));
Assert.Empty(await db.Jobs.ListQueuedAsync([], 10, Ct));
// Workers racing for the same listed job: exactly one gets it, and it leaves the listing.
var claims = await Task.WhenAll(Enumerable.Range(0, 6).Select(w => Task.Run(() => db.Jobs.TryClaimAsync(oldUser.Id, "worker-" + w, Ct))));
var claimed = Assert.NotNull(claims.SingleOrDefault(c => c is not null));
Assert.Equal((JobState.Running, 1), (claimed.State, claimed.Attempts));
Assert.SequenceEqual([newUser.Id, oldBackground.Id], (await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 10, Ct)).Select(j => j.Id));
}
[Test]
public async Task ConcurrentClaimersNeverShareAJob()
{
var db = await OpenChronosAsync(multiConnection: true);
var jobs = Enumerable.Range(0, 40).Select(i => TestData.Job(created: TestData.Now.AddSeconds(i))).ToList();
foreach (var job in jobs) await db.Jobs.EnqueueAsync(job, Ct);
var claims = new ConcurrentBag<(string Worker, Guid Job)>();
await Task.WhenAll(Enumerable.Range(0, 8).Select(w => Task.Run(async () =>
{
var worker = "worker-" + w;
while (await db.Jobs.ClaimNextAsync(worker, cancellationToken: Ct) is { } job) claims.Add((worker, job.Id));
})));
Assert.Equal(40, claims.Count);
Assert.Equal(40, claims.Select(c => c.Job).Distinct().Count());
foreach (var (worker, id) in claims)
{
var stored = await db.Jobs.GetAsync(id, Ct);
Assert.Equal(worker, stored!.NodeId);
Assert.Equal(1, stored.Attempts);
}
}
[Test]
public async Task ProgressIsRefusedOnceTheJobStopsRunning()
{
var db = await OpenChronosAsync();
var job = TestData.Job();
await db.Jobs.EnqueueAsync(job, Ct);
Assert.False(await db.Jobs.UpdateProgressAsync(job.Id, 1, 10, null, null, null, null, Ct));
await db.Jobs.ClaimNextAsync("w", cancellationToken: Ct);
Assert.True(await db.Jobs.UpdateProgressAsync(job.Id, 4, 10, 2.0, TimeSpan.FromSeconds(3), "cursor:4", "tail", Ct));
var running = await db.Jobs.GetAsync(job.Id, Ct);
Assert.Equal((4L, 10L, "cursor:4", "tail"), (running!.ProgressDone, running.ProgressTotal, running.Checkpoint, running.LogTail));
Assert.Equal(TimeSpan.FromSeconds(3), running.Eta);
Assert.True(await db.Jobs.SetStateAsync(job.Id, JobState.Cancelled, cancellationToken: Ct));
Assert.False(await db.Jobs.UpdateProgressAsync(job.Id, 5, 10, null, null, "cursor:5", null, Ct));
var cancelled = await db.Jobs.GetAsync(job.Id, Ct);
Assert.Equal(JobState.Cancelled, cancelled!.State);
Assert.NotNull(cancelled.Finished);
Assert.True(await db.Jobs.SetStateAsync(job.Id, JobState.Queued, cancellationToken: Ct));
Assert.Null((await db.Jobs.GetAsync(job.Id, Ct))!.Finished);
Assert.Equal(2, (await db.Jobs.ClaimNextAsync("w", cancellationToken: Ct))!.Attempts);
Assert.False(await db.Jobs.SetStateAsync(Guid.CreateVersion7(), JobState.Done, cancellationToken: Ct));
}
[Test]
public async Task ListingPagesByStateNewestFirst()
{
var db = await OpenChronosAsync();
var jobs = Enumerable.Range(0, 7).Select(i => TestData.Job(created: TestData.Now.AddMinutes(i % 3))).ToList();
foreach (var job in jobs) await db.Jobs.EnqueueAsync(job, Ct);
await db.Jobs.SetStateAsync(jobs[0].Id, JobState.Done, cancellationToken: Ct);
var seen = new List<Guid>();
string? cursor = null;
do
{
var page = await db.Jobs.ListAsync(JobState.Queued, limit: 2, cursor: cursor, cancellationToken: Ct);
seen.AddRange(page.Items.Select(j => j.Id));
cursor = page.NextCursor;
}
while (cursor is not null);
var expected = jobs.Skip(1)
.OrderByDescending(j => j.Created)
.ThenByDescending(j => j.Id.ToString("D"), StringComparer.Ordinal)
.Select(j => j.Id);
Assert.SequenceEqual(expected, seen);
Assert.SequenceEqual([jobs[0].Id], (await db.Jobs.ListAsync(JobState.Done, cancellationToken: Ct)).Items.Select(j => j.Id));
Assert.Equal(7, (await db.Jobs.ListAsync(cancellationToken: Ct)).Items.Count);
}
[Test]
public async Task HistoryPurgeKeepsRecentAndActiveJobs()
{
var db = await OpenChronosAsync();
var old = TestData.Job() with { State = JobState.Done, Finished = TestData.Now.AddDays(-100) };
var oldFailed = TestData.Job() with { State = JobState.Failed, Finished = TestData.Now.AddDays(-91) };
var recent = TestData.Job() with { State = JobState.Done, Finished = TestData.Now.AddDays(-10) };
var queued = TestData.Job(created: TestData.Now.AddDays(-200));
foreach (var job in new[] { old, oldFailed, recent, queued }) await db.Jobs.EnqueueAsync(job, Ct);
Assert.Equal(2, await db.Jobs.PurgeHistoryAsync(TestData.Now.AddDays(-90), Ct));
Assert.Equal(2, (await db.Jobs.ListAsync(cancellationToken: Ct)).Items.Count);
}
[Test]
public async Task RunningJobsAreRequeuedAfterACrash()
{
var db = await OpenChronosAsync();
for (var i = 0; i < 3; i++) await db.Jobs.EnqueueAsync(TestData.Job(created: TestData.Now.AddSeconds(i)), Ct);
var a = await db.Jobs.ClaimNextAsync("node-a", cancellationToken: Ct);
var b = await db.Jobs.ClaimNextAsync("node-b", cancellationToken: Ct);
await db.Jobs.UpdateProgressAsync(a!.Id, 3, 9, null, null, "resume-here", null, Ct);
Assert.Equal(1, await db.Jobs.RequeueRunningAsync("node-a", Ct));
var requeued = await db.Jobs.GetAsync(a.Id, Ct);
Assert.Equal(JobState.Queued, requeued!.State);
Assert.Equal("resume-here", requeued.Checkpoint);
Assert.Null(requeued.NodeId);
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(b!.Id, Ct))!.State);
Assert.Equal(1, await db.Jobs.RequeueRunningAsync(cancellationToken: Ct));
Assert.Equal(0, await db.Jobs.RequeueRunningAsync(cancellationToken: Ct));
}
}
public sealed class SqliteJobRepositoryTests(TestContext context) : JobRepositoryTests(context)
{
protected override Backend Backend => Backend.SqliteMemory;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresJobRepositoryTests(TestContext context) : JobRepositoryTests(context)
{
protected override Backend Backend => Backend.Postgres;
}