using System.Collections.Concurrent; using Chronos.Core.Model; using Chronos.Data.Tests.Infrastructure; using Chronos.Testing; namespace Chronos.Data.Tests.Repositories; public abstract class JobRepositoryTests(TestContext context) : RepositoryTest(context) { [Test] public async Task JobsRoundTripEveryField() { var db = await OpenChronosAsync(); var job = TestData.Job(JobKind.Render, JobPriority.User) with { TargetIds = [Guid.CreateVersion7(), Guid.CreateVersion7()], TargetLabel = "Harbour sunset", ParametersJson = "{\"preset\": \"4K\"}", ProgressDone = 5, ProgressTotal = 10, ItemsPerSecond = 2.5, Eta = TimeSpan.FromSeconds(2), Checkpoint = "frame:5", LogTail = "line 1\nline 2", CreatedBy = Guid.CreateVersion7(), NodeId = "node-a", Attempts = 1, Error = "boom", State = JobState.Failed, Started = TestData.Now.AddMinutes(1), Finished = TestData.Now.AddMinutes(2), ScheduleId = Guid.CreateVersion7(), Detail = new JobDetail(JobStep.Encode, "Harbour sunset", 5, 10, 1, 2, JobPartUnit.Render), }; await db.Jobs.EnqueueAsync(job, Ct); var read = Assert.NotNull(await db.Jobs.GetAsync(job.Id, Ct)); Assert.SequenceEqual(job.TargetIds, read.TargetIds); Assert.Equal(job with { TargetIds = read.TargetIds }, read); var plain = (await db.Jobs.GetAsync((await EnqueueAsync(db, TestData.Job())).Id, Ct))!; Assert.Null(plain.ScheduleId); Assert.Null(plain.Detail); } [Test] public async Task TheProgressDetailIsSavedWithTheProgressAndListedWithTheJob() { var db = await OpenChronosAsync(); var job = TestData.Job(JobKind.Verify); await db.Jobs.EnqueueAsync(job, Ct); await db.Jobs.ClaimNextAsync("w", cancellationToken: Ct); var detail = new JobDetail(JobStep.Verify, "Harbour sunset ยท 2024", 812, 2400, 3, 41, JobPartUnit.Sequence); Assert.True(await db.Jobs.UpdateProgressAsync(job.Id, 4, 10, null, null, "cursor:4", null, detail, Ct)); Assert.Equal(detail, (await db.Jobs.GetAsync(job.Id, Ct))!.Detail); // Lists read it too: a paused job tells where it stopped. Assert.SequenceEqual([detail], (await db.Jobs.ListBriefAsync(JobState.Running, null, cancellationToken: Ct)).Items.Select(j => j.Detail)); // Without a detail the column is cleared; a node's progress keeps or replaces it. Assert.True(await db.Jobs.UpdateProgressAsync(job.Id, 5, 10, null, null, "cursor:5", null, Ct)); Assert.Null((await db.Jobs.GetAsync(job.Id, Ct))!.Detail); var node = new JobDetail(JobStep.Encode, null, 120, 3600); Assert.True(await db.Jobs.UpdateNodeProgressAsync(job.Id, "w", 6, 10, null, null, null, node, Ct)); Assert.True(await db.Jobs.UpdateNodeProgressAsync(job.Id, "w", 7, 10, null, null, "line", Ct)); Assert.Equal(node, (await db.Jobs.GetAsync(job.Id, Ct))!.Detail); Assert.True(await db.Jobs.UpdateNodeProgressAsync(job.Id, "w", 8, 10, null, null, null, null, Ct)); Assert.Null((await db.Jobs.GetAsync(job.Id, Ct))!.Detail); // A cell this build cannot read is no detail, never an error. foreach (var unreadable in new[] { "not json", "42", "[]", "{\"step\": 7}" }) { await using (var session = await db.Database.OpenAsync(Ct)) await session.ExecuteAsync("UPDATE jobs SET progress_detail = $1 WHERE id = $2", [unreadable, job.Id], Ct); Assert.Null((await db.Jobs.GetAsync(job.Id, Ct))!.Detail); } // One of a newer build: its step and unit read as none, the counts stay. await using (var session = await db.Database.OpenAsync(Ct)) await session.ExecuteAsync("UPDATE jobs SET progress_detail = $1 WHERE id = $2", ["""{"step":"teleport","itemDone":3,"itemTotal":9,"part":2,"parts":4,"partUnit":"galaxy","future":true}""", job.Id], Ct); Assert.Equal(new JobDetail(JobStep.None, null, 3, 9, 2, 4), (await db.Jobs.GetAsync(job.Id, Ct))!.Detail); } [Test] public async Task JobsOfAScheduleAreListedNewestFirst() { var db = await OpenChronosAsync(); var schedule = Guid.CreateVersion7(); var older = await EnqueueAsync(db, TestData.Job(created: TestData.Now.AddMinutes(-5)) with { ScheduleId = schedule }); var newer = await EnqueueAsync(db, TestData.Job() with { ScheduleId = schedule }); await EnqueueAsync(db, TestData.Job()); Assert.SequenceEqual([newer.Id, older.Id], (await db.Jobs.ListByScheduleAsync(schedule, cancellationToken: Ct)).Select(j => j.Id)); Assert.Empty(await db.Jobs.ListByScheduleAsync(Guid.CreateVersion7(), cancellationToken: Ct)); } [Test] public async Task JobsOfAScheduleArePagedNewestFirst() { // The runs a schedule's row lists in the admin panel (owner review 2, R10). var db = await OpenChronosAsync(); var schedule = Guid.CreateVersion7(); var jobs = new List(); for (var i = 0; i < 3; i++) jobs.Add(await EnqueueAsync(db, TestData.Job(created: TestData.Now.AddMinutes(i)) with { ScheduleId = schedule })); await EnqueueAsync(db, TestData.Job(created: TestData.Now.AddMinutes(5))); var first = await db.Jobs.PageByScheduleAsync(schedule, 2, null, Ct); Assert.SequenceEqual([jobs[2].Id, jobs[1].Id], first.Items.Select(j => j.Id)); var second = await db.Jobs.PageByScheduleAsync(schedule, 2, Assert.NotNull(first.NextCursor), Ct); Assert.SequenceEqual([jobs[0].Id], second.Items.Select(j => j.Id)); Assert.Null(second.NextCursor); Assert.Empty((await db.Jobs.PageByScheduleAsync(Guid.CreateVersion7(), cancellationToken: Ct)).Items); // A cursor of another listing is refused. var other = await db.Jobs.ListAsync(null, 1, null, Ct); await Assert.ThrowsAsync(() => db.Jobs.PageByScheduleAsync(schedule, 2, Assert.NotNull(other.NextCursor), Ct)); } [Test] public async Task PendingJobsAreFoundByKindAndTarget() { // A schedule queues nothing while a job doing the same work waits, runs or is paused (owner review 2, R10). var db = await OpenChronosAsync(); Guid archive = Guid.CreateVersion7(), a = Guid.CreateVersion7(), b = Guid.CreateVersion7(), c = Guid.CreateVersion7(); await EnqueueAsync(db, TestData.Job(JobKind.Scan) with { TargetIds = [archive] }); await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [a, b], State = JobState.Paused }); await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [c], State = JobState.Done, Finished = TestData.Now }); await EnqueueAsync(db, TestData.Job(JobKind.Verify) with { TargetIds = [archive], State = JobState.Failed, Finished = TestData.Now }); Assert.True(await db.Jobs.HasPendingAsync([JobKind.Scan, JobKind.Rebuild], [archive], Ct)); Assert.False(await db.Jobs.HasPendingAsync([JobKind.Rebuild], [archive], Ct)); Assert.False(await db.Jobs.HasPendingAsync([JobKind.Verify], [archive], Ct), "a failed job is not pending"); Assert.True(await db.Jobs.HasPendingAsync([JobKind.Thumbs], [b], Ct), "the second of a job's targets"); Assert.True(await db.Jobs.HasPendingAsync([JobKind.Thumbs], null, Ct)); Assert.False(await db.Jobs.HasPendingAsync([JobKind.Thumbs], [c], Ct), "a done job is not pending"); Assert.False(await db.Jobs.HasPendingAsync([JobKind.Scan], [], Ct)); var claimed = await db.Jobs.ClaimNextAsync("node", [JobKind.Scan], Ct); Assert.Equal(JobState.Running, claimed!.State); Assert.True(await db.Jobs.HasPendingAsync([JobKind.Scan], [archive], Ct), "a running job is pending"); Assert.SequenceEqual(new[] { a, b }.Order(), (await db.Jobs.ListPendingTargetsAsync(JobKind.Thumbs, Ct)).Order()); Assert.Empty(await db.Jobs.ListPendingTargetsAsync(JobKind.Verify, Ct)); } private async Task EnqueueAsync(ChronosDatabase db, Job job) { await db.Jobs.EnqueueAsync(job, Ct); return job; } [Test] public async Task HoldsNameTheTargetsOfRunningAndPausedJobsAndOnlyAVerifysParameters() { // What a dry run counts as busy, read at every one and at every sequence a conversion comes to: never the parameters // a conversion or an import keeps, nor the row of the job asking. var db = await OpenChronosAsync(); Guid a = Guid.CreateVersion7(), b = Guid.CreateVersion7(), archive = Guid.CreateVersion7(); var convert = await EnqueueAsync(db, TestData.Job(JobKind.Convert) with { TargetIds = [a, b], ParametersJson = "{\"fingerprint\": \"f\"}", State = JobState.Running }); var verify = await EnqueueAsync(db, TestData.Job(JobKind.Verify) with { TargetIds = [archive], ParametersJson = "{\"sequenceIds\": []}", State = JobState.Paused }); await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [a] }); await EnqueueAsync(db, TestData.Job(JobKind.Scan) with { TargetIds = [archive], State = JobState.Done, Finished = TestData.Now }); var holds = await db.Jobs.ListHoldsAsync(cancellationToken: Ct); Assert.SequenceEqual(new[] { convert.Id, verify.Id }.Order(), holds.Select(h => h.Id).Order()); var running = holds.Single(h => h.Id == convert.Id); Assert.Equal((JobKind.Convert, (string?)null), (running.Kind, running.ParametersJson)); Assert.SequenceEqual([a, b], running.TargetIds); var paused = holds.Single(h => h.Id == verify.Id); Assert.Equal((JobKind.Verify, verify.ParametersJson), (paused.Kind, paused.ParametersJson)); Assert.SequenceEqual([verify.Id], (await db.Jobs.ListHoldsAsync(convert.Id, Ct)).Select(h => h.Id)); Assert.Equal(convert.Created, await db.Jobs.GetCreatedAsync(convert.Id, Ct)); Assert.Null(await db.Jobs.GetCreatedAsync(Guid.CreateVersion7(), Ct)); } [Test] public async Task BriefListsReadEveryFieldButTheParameters() { var db = await OpenChronosAsync(); var job = TestData.Job(JobKind.Convert, JobPriority.User) with { TargetIds = [Guid.CreateVersion7(), Guid.CreateVersion7()], TargetLabel = "Harbour sunset", ParametersJson = "{\"fingerprint\": \"f\"}", LogTail = "line 1", CreatedBy = Guid.CreateVersion7(), ScheduleId = Guid.CreateVersion7(), }; await db.Jobs.EnqueueAsync(job, Ct); var brief = Assert.NotNull((await db.Jobs.ListBriefAsync(JobState.Queued, JobKind.Convert, 10, null, Ct)).Items.SingleOrDefault()); Assert.SequenceEqual(job.TargetIds, brief.TargetIds); Assert.Equal(job with { TargetIds = brief.TargetIds, ParametersJson = null }, brief); Assert.Equal(job.ParametersJson, (await db.Jobs.ListAsync(JobState.Queued, JobKind.Convert, 10, null, Ct)).Items.Single().ParametersJson); Assert.Empty((await db.Jobs.ListBriefAsync(JobState.Running, null, 10, null, Ct)).Items); var next = Assert.NotNull((await db.Jobs.ListQueuedBriefAsync([JobKind.Convert], 10, Chronos.Data.Jobs.JobPriorityQueue.Off, Ct)).SingleOrDefault()); Assert.Equal((job.Id, (string?)null), (next.Id, next.ParametersJson)); Assert.Empty(await db.Jobs.ListQueuedBriefAsync([], 10, Chronos.Data.Jobs.JobPriorityQueue.Off, Ct)); } [Test] public async Task ThePriorityQueueClaimsTheHigherKindsFirst() { // Owner review 1 (R2): on, the level of the kind comes before priority and age; off, the strict queue stays. var db = await OpenChronosAsync(); var oldVerify = TestData.Job(JobKind.Verify, JobPriority.User, TestData.Now.AddMinutes(-30)); var backgroundScan = TestData.Job(JobKind.Scan, JobPriority.Background, TestData.Now.AddMinutes(-20)); var userThumbs = TestData.Job(JobKind.Thumbs, JobPriority.User, TestData.Now.AddMinutes(-10)); var render = TestData.Job(JobKind.Render, JobPriority.Normal, TestData.Now); foreach (var job in new[] { oldVerify, backgroundScan, userThumbs, render }) await db.Jobs.EnqueueAsync(job, Ct); var queue = new Chronos.Data.Jobs.JobPriorityQueue { Enabled = true }; JobKind[] kinds = [JobKind.Verify, JobKind.Scan, JobKind.Thumbs, JobKind.Render]; Assert.SequenceEqual([render.Id, userThumbs.Id, backgroundScan.Id, oldVerify.Id], (await db.Jobs.ListQueuedAsync(kinds, 10, queue, Ct)).Select(j => j.Id)); Assert.SequenceEqual([oldVerify.Id, userThumbs.Id, render.Id, backgroundScan.Id], (await db.Jobs.ListQueuedAsync(kinds, 10, Ct)).Select(j => j.Id)); Assert.Equal(userThumbs.Id, (await db.Jobs.ClaimNextAsync("server", [JobKind.Scan, JobKind.Thumbs, JobKind.Verify], queue, Ct))?.Id); var order = new List(); while (await db.Jobs.ClaimNextAsync("server", null, queue, Ct) is { } claimed) order.Add(claimed.Id); Assert.SequenceEqual([render.Id, backgroundScan.Id, oldVerify.Id], order); } [Test] public async Task ClaimsByPriorityThenAgeAndKind() { var db = await OpenChronosAsync(); var oldBackground = TestData.Job(JobKind.Thumbs, JobPriority.Background, TestData.Now.AddMinutes(-10)); var newUser = TestData.Job(JobKind.Render, JobPriority.User, TestData.Now); var oldUser = TestData.Job(JobKind.Import, JobPriority.User, TestData.Now.AddMinutes(-1)); var normal = TestData.Job(JobKind.Scan, JobPriority.Normal, TestData.Now.AddMinutes(-5)); foreach (var job in new[] { oldBackground, newUser, oldUser, normal }) await db.Jobs.EnqueueAsync(job, Ct); var renderOnly = await db.Jobs.ClaimNextAsync("node-gpu", [JobKind.Render], Ct); Assert.Equal(newUser.Id, renderOnly?.Id); Assert.Equal(JobState.Running, renderOnly!.State); Assert.Equal("node-gpu", renderOnly.NodeId); Assert.Equal(1, renderOnly.Attempts); Assert.NotNull(renderOnly.Started); Assert.Null(await db.Jobs.ClaimNextAsync("node-gpu", [JobKind.Render], Ct)); var order = new List(); while (await db.Jobs.ClaimNextAsync("server", cancellationToken: Ct) is { } claimed) order.Add(claimed.Id); Assert.SequenceEqual([oldUser.Id, normal.Id, oldBackground.Id], order); } [Test] public async Task SelectiveClaimersListQueuedKindsInClaimOrderAndClaimById() { var db = await OpenChronosAsync(multiConnection: true); var oldBackground = TestData.Job(JobKind.Render, JobPriority.Background, TestData.Now.AddMinutes(-10)); var newUser = TestData.Job(JobKind.Thumbs, JobPriority.User, TestData.Now); var oldUser = TestData.Job(JobKind.Render, JobPriority.User, TestData.Now.AddMinutes(-1)); var otherKind = TestData.Job(JobKind.Scan, JobPriority.User, TestData.Now.AddMinutes(-20)); foreach (var job in new[] { oldBackground, newUser, oldUser, otherKind }) await db.Jobs.EnqueueAsync(job, Ct); var queued = await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 10, Ct); Assert.SequenceEqual([oldUser.Id, newUser.Id, oldBackground.Id], queued.Select(j => j.Id)); Assert.Count(1, await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 1, Ct)); Assert.Empty(await db.Jobs.ListQueuedAsync([], 10, Ct)); // Workers racing for the same listed job: exactly one gets it, and it leaves the listing. var claims = await Task.WhenAll(Enumerable.Range(0, 6).Select(w => Task.Run(() => db.Jobs.TryClaimAsync(oldUser.Id, "worker-" + w, Ct)))); var claimed = Assert.NotNull(claims.SingleOrDefault(c => c is not null)); Assert.Equal((JobState.Running, 1), (claimed.State, claimed.Attempts)); Assert.SequenceEqual([newUser.Id, oldBackground.Id], (await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 10, Ct)).Select(j => j.Id)); } [Test] public async Task ConcurrentClaimersNeverShareAJob() { var db = await OpenChronosAsync(multiConnection: true); var jobs = Enumerable.Range(0, 40).Select(i => TestData.Job(created: TestData.Now.AddSeconds(i))).ToList(); foreach (var job in jobs) await db.Jobs.EnqueueAsync(job, Ct); var claims = new ConcurrentBag<(string Worker, Guid Job)>(); await Task.WhenAll(Enumerable.Range(0, 8).Select(w => Task.Run(async () => { var worker = "worker-" + w; while (await db.Jobs.ClaimNextAsync(worker, cancellationToken: Ct) is { } job) claims.Add((worker, job.Id)); }))); Assert.Equal(40, claims.Count); Assert.Equal(40, claims.Select(c => c.Job).Distinct().Count()); foreach (var (worker, id) in claims) { var stored = await db.Jobs.GetAsync(id, Ct); Assert.Equal(worker, stored!.NodeId); Assert.Equal(1, stored.Attempts); } } [Test] public async Task ProgressIsRefusedOnceTheJobStopsRunning() { var db = await OpenChronosAsync(); var job = TestData.Job(); await db.Jobs.EnqueueAsync(job, Ct); Assert.False(await db.Jobs.UpdateProgressAsync(job.Id, 1, 10, null, null, null, null, Ct)); await db.Jobs.ClaimNextAsync("w", cancellationToken: Ct); Assert.True(await db.Jobs.UpdateProgressAsync(job.Id, 4, 10, 2.0, TimeSpan.FromSeconds(3), "cursor:4", "tail", Ct)); var running = await db.Jobs.GetAsync(job.Id, Ct); Assert.Equal((4L, 10L, "cursor:4", "tail"), (running!.ProgressDone, running.ProgressTotal, running.Checkpoint, running.LogTail)); Assert.Equal(TimeSpan.FromSeconds(3), running.Eta); Assert.True(await db.Jobs.SetStateAsync(job.Id, JobState.Cancelled, cancellationToken: Ct)); Assert.False(await db.Jobs.UpdateProgressAsync(job.Id, 5, 10, null, null, "cursor:5", null, Ct)); var cancelled = await db.Jobs.GetAsync(job.Id, Ct); Assert.Equal(JobState.Cancelled, cancelled!.State); Assert.NotNull(cancelled.Finished); Assert.True(await db.Jobs.SetStateAsync(job.Id, JobState.Queued, cancellationToken: Ct)); Assert.Null((await db.Jobs.GetAsync(job.Id, Ct))!.Finished); Assert.Equal(2, (await db.Jobs.ClaimNextAsync("w", cancellationToken: Ct))!.Attempts); Assert.False(await db.Jobs.SetStateAsync(Guid.CreateVersion7(), JobState.Done, cancellationToken: Ct)); } [Test] public async Task ListingPagesByStateNewestFirst() { var db = await OpenChronosAsync(); var jobs = Enumerable.Range(0, 7).Select(i => TestData.Job(created: TestData.Now.AddMinutes(i % 3))).ToList(); foreach (var job in jobs) await db.Jobs.EnqueueAsync(job, Ct); await db.Jobs.SetStateAsync(jobs[0].Id, JobState.Done, cancellationToken: Ct); var seen = new List(); string? cursor = null; do { var page = await db.Jobs.ListAsync(JobState.Queued, limit: 2, cursor: cursor, cancellationToken: Ct); seen.AddRange(page.Items.Select(j => j.Id)); cursor = page.NextCursor; } while (cursor is not null); var expected = jobs.Skip(1) .OrderByDescending(j => j.Created) .ThenByDescending(j => j.Id.ToString("D"), StringComparer.Ordinal) .Select(j => j.Id); Assert.SequenceEqual(expected, seen); Assert.SequenceEqual([jobs[0].Id], (await db.Jobs.ListAsync(JobState.Done, cancellationToken: Ct)).Items.Select(j => j.Id)); Assert.Equal(7, (await db.Jobs.ListAsync(cancellationToken: Ct)).Items.Count); } [Test] public async Task HistoryPurgeKeepsRecentAndActiveJobs() { var db = await OpenChronosAsync(); var old = TestData.Job() with { State = JobState.Done, Finished = TestData.Now.AddDays(-100) }; var oldFailed = TestData.Job() with { State = JobState.Failed, Finished = TestData.Now.AddDays(-91) }; var recent = TestData.Job() with { State = JobState.Done, Finished = TestData.Now.AddDays(-10) }; var queued = TestData.Job(created: TestData.Now.AddDays(-200)); foreach (var job in new[] { old, oldFailed, recent, queued }) await db.Jobs.EnqueueAsync(job, Ct); Assert.Equal(2, await db.Jobs.PurgeHistoryAsync(TestData.Now.AddDays(-90), Ct)); Assert.Equal(2, (await db.Jobs.ListAsync(cancellationToken: Ct)).Items.Count); } [Test] public async Task RunningJobsAreRequeuedAfterACrash() { var db = await OpenChronosAsync(); for (var i = 0; i < 3; i++) await db.Jobs.EnqueueAsync(TestData.Job(created: TestData.Now.AddSeconds(i)), Ct); var a = await db.Jobs.ClaimNextAsync("node-a", cancellationToken: Ct); var b = await db.Jobs.ClaimNextAsync("node-b", cancellationToken: Ct); await db.Jobs.UpdateProgressAsync(a!.Id, 3, 9, null, null, "resume-here", null, Ct); Assert.Equal(1, await db.Jobs.RequeueRunningAsync("node-a", Ct)); var requeued = await db.Jobs.GetAsync(a.Id, Ct); Assert.Equal(JobState.Queued, requeued!.State); Assert.Equal("resume-here", requeued.Checkpoint); Assert.Null(requeued.NodeId); Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(b!.Id, Ct))!.State); Assert.Equal(1, await db.Jobs.RequeueRunningAsync(cancellationToken: Ct)); Assert.Equal(0, await db.Jobs.RequeueRunningAsync(cancellationToken: Ct)); } } public sealed class SqliteJobRepositoryTests(TestContext context) : JobRepositoryTests(context) { protected override Backend Backend => Backend.SqliteMemory; } [RequiresEnvironment(PgTestServer.Variable)] public sealed class PostgresJobRepositoryTests(TestContext context) : JobRepositoryTests(context) { protected override Backend Backend => Backend.Postgres; }