CI / Build and test (push) Successful in 7m0s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 13m28s
CI / PostgreSQL 16 integration (push) Successful in 5m52s
CI / PostgreSQL 17 integration (push) Successful in 4m50s
CI / Docker image (push) Successful in 1m4s
Release / Server image (push) Successful in 1m35s
- Jobs > Scheduled on the desktop and the admin panel. Every timer that queued work is now a visible system schedule: archive scan, upload clean-up, index backup, job-history and audit clean-ups, desktop housekeeping. New actions: verify, thumbnails, proxies, analysis, maintenance, backup, clean-ups, reports. - The scheduler: migration 0007, in-house recurrence, compare-and-set claim, catch-up, scan at start. - Log settings: levels, retention, size parts, total limit, daily or hourly rotation, audit retention, diagnostics share; the server writes Debug/Trace to its files again; logging never fails a job. Upgrade: CHRONOS_LOG_LEVEL set in an existing container locks both levels; migration 0007 is one-way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
291 lines
16 KiB
C#
291 lines
16 KiB
C#
using System.Collections.Concurrent;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Tests.Infrastructure;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Repositories;
|
|
|
|
public abstract class JobRepositoryTests(TestContext context) : RepositoryTest(context)
|
|
{
|
|
[Test]
|
|
public async Task JobsRoundTripEveryField()
|
|
{
|
|
var db = await OpenChronosAsync();
|
|
var job = TestData.Job(JobKind.Render, JobPriority.User) with
|
|
{
|
|
TargetIds = [Guid.CreateVersion7(), Guid.CreateVersion7()],
|
|
TargetLabel = "Harbour sunset",
|
|
ParametersJson = "{\"preset\": \"4K\"}",
|
|
ProgressDone = 5,
|
|
ProgressTotal = 10,
|
|
ItemsPerSecond = 2.5,
|
|
Eta = TimeSpan.FromSeconds(2),
|
|
Checkpoint = "frame:5",
|
|
LogTail = "line 1\nline 2",
|
|
CreatedBy = Guid.CreateVersion7(),
|
|
NodeId = "node-a",
|
|
Attempts = 1,
|
|
Error = "boom",
|
|
State = JobState.Failed,
|
|
Started = TestData.Now.AddMinutes(1),
|
|
Finished = TestData.Now.AddMinutes(2),
|
|
ScheduleId = Guid.CreateVersion7(),
|
|
};
|
|
await db.Jobs.EnqueueAsync(job, Ct);
|
|
var read = Assert.NotNull(await db.Jobs.GetAsync(job.Id, Ct));
|
|
Assert.SequenceEqual(job.TargetIds, read.TargetIds);
|
|
Assert.Equal(job with { TargetIds = read.TargetIds }, read);
|
|
Assert.Null((await db.Jobs.GetAsync((await EnqueueAsync(db, TestData.Job())).Id, Ct))!.ScheduleId);
|
|
}
|
|
|
|
[Test]
|
|
public async Task JobsOfAScheduleAreListedNewestFirst()
|
|
{
|
|
var db = await OpenChronosAsync();
|
|
var schedule = Guid.CreateVersion7();
|
|
var older = await EnqueueAsync(db, TestData.Job(created: TestData.Now.AddMinutes(-5)) with { ScheduleId = schedule });
|
|
var newer = await EnqueueAsync(db, TestData.Job() with { ScheduleId = schedule });
|
|
await EnqueueAsync(db, TestData.Job());
|
|
Assert.SequenceEqual([newer.Id, older.Id], (await db.Jobs.ListByScheduleAsync(schedule, cancellationToken: Ct)).Select(j => j.Id));
|
|
Assert.Empty(await db.Jobs.ListByScheduleAsync(Guid.CreateVersion7(), cancellationToken: Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task JobsOfAScheduleArePagedNewestFirst()
|
|
{
|
|
// The runs a schedule's row lists in the admin panel (owner review 2, R10).
|
|
var db = await OpenChronosAsync();
|
|
var schedule = Guid.CreateVersion7();
|
|
var jobs = new List<Job>();
|
|
for (var i = 0; i < 3; i++) jobs.Add(await EnqueueAsync(db, TestData.Job(created: TestData.Now.AddMinutes(i)) with { ScheduleId = schedule }));
|
|
await EnqueueAsync(db, TestData.Job(created: TestData.Now.AddMinutes(5)));
|
|
|
|
var first = await db.Jobs.PageByScheduleAsync(schedule, 2, null, Ct);
|
|
Assert.SequenceEqual([jobs[2].Id, jobs[1].Id], first.Items.Select(j => j.Id));
|
|
var second = await db.Jobs.PageByScheduleAsync(schedule, 2, Assert.NotNull(first.NextCursor), Ct);
|
|
Assert.SequenceEqual([jobs[0].Id], second.Items.Select(j => j.Id));
|
|
Assert.Null(second.NextCursor);
|
|
Assert.Empty((await db.Jobs.PageByScheduleAsync(Guid.CreateVersion7(), cancellationToken: Ct)).Items);
|
|
// A cursor of another listing is refused.
|
|
var other = await db.Jobs.ListAsync(null, 1, null, Ct);
|
|
await Assert.ThrowsAsync<FormatException>(() => db.Jobs.PageByScheduleAsync(schedule, 2, Assert.NotNull(other.NextCursor), Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PendingJobsAreFoundByKindAndTarget()
|
|
{
|
|
// A schedule queues nothing while a job doing the same work waits, runs or is paused (owner review 2, R10).
|
|
var db = await OpenChronosAsync();
|
|
Guid archive = Guid.CreateVersion7(), a = Guid.CreateVersion7(), b = Guid.CreateVersion7(), c = Guid.CreateVersion7();
|
|
await EnqueueAsync(db, TestData.Job(JobKind.Scan) with { TargetIds = [archive] });
|
|
await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [a, b], State = JobState.Paused });
|
|
await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [c], State = JobState.Done, Finished = TestData.Now });
|
|
await EnqueueAsync(db, TestData.Job(JobKind.Verify) with { TargetIds = [archive], State = JobState.Failed, Finished = TestData.Now });
|
|
|
|
Assert.True(await db.Jobs.HasPendingAsync([JobKind.Scan, JobKind.Rebuild], [archive], Ct));
|
|
Assert.False(await db.Jobs.HasPendingAsync([JobKind.Rebuild], [archive], Ct));
|
|
Assert.False(await db.Jobs.HasPendingAsync([JobKind.Verify], [archive], Ct), "a failed job is not pending");
|
|
Assert.True(await db.Jobs.HasPendingAsync([JobKind.Thumbs], [b], Ct), "the second of a job's targets");
|
|
Assert.True(await db.Jobs.HasPendingAsync([JobKind.Thumbs], null, Ct));
|
|
Assert.False(await db.Jobs.HasPendingAsync([JobKind.Thumbs], [c], Ct), "a done job is not pending");
|
|
Assert.False(await db.Jobs.HasPendingAsync([JobKind.Scan], [], Ct));
|
|
|
|
var claimed = await db.Jobs.ClaimNextAsync("node", [JobKind.Scan], Ct);
|
|
Assert.Equal(JobState.Running, claimed!.State);
|
|
Assert.True(await db.Jobs.HasPendingAsync([JobKind.Scan], [archive], Ct), "a running job is pending");
|
|
Assert.SequenceEqual(new[] { a, b }.Order(), (await db.Jobs.ListPendingTargetsAsync(JobKind.Thumbs, Ct)).Order());
|
|
Assert.Empty(await db.Jobs.ListPendingTargetsAsync(JobKind.Verify, Ct));
|
|
}
|
|
|
|
private async Task<Job> EnqueueAsync(ChronosDatabase db, Job job)
|
|
{
|
|
await db.Jobs.EnqueueAsync(job, Ct);
|
|
return job;
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThePriorityQueueClaimsTheHigherKindsFirst()
|
|
{
|
|
// Owner review 1 (R2): on, the level of the kind comes before priority and age; off, the strict queue stays.
|
|
var db = await OpenChronosAsync();
|
|
var oldVerify = TestData.Job(JobKind.Verify, JobPriority.User, TestData.Now.AddMinutes(-30));
|
|
var backgroundScan = TestData.Job(JobKind.Scan, JobPriority.Background, TestData.Now.AddMinutes(-20));
|
|
var userThumbs = TestData.Job(JobKind.Thumbs, JobPriority.User, TestData.Now.AddMinutes(-10));
|
|
var render = TestData.Job(JobKind.Render, JobPriority.Normal, TestData.Now);
|
|
foreach (var job in new[] { oldVerify, backgroundScan, userThumbs, render }) await db.Jobs.EnqueueAsync(job, Ct);
|
|
var queue = new Chronos.Data.Jobs.JobPriorityQueue { Enabled = true };
|
|
JobKind[] kinds = [JobKind.Verify, JobKind.Scan, JobKind.Thumbs, JobKind.Render];
|
|
|
|
Assert.SequenceEqual([render.Id, userThumbs.Id, backgroundScan.Id, oldVerify.Id], (await db.Jobs.ListQueuedAsync(kinds, 10, queue, Ct)).Select(j => j.Id));
|
|
Assert.SequenceEqual([oldVerify.Id, userThumbs.Id, render.Id, backgroundScan.Id], (await db.Jobs.ListQueuedAsync(kinds, 10, Ct)).Select(j => j.Id));
|
|
Assert.Equal(userThumbs.Id, (await db.Jobs.ClaimNextAsync("server", [JobKind.Scan, JobKind.Thumbs, JobKind.Verify], queue, Ct))?.Id);
|
|
var order = new List<Guid>();
|
|
while (await db.Jobs.ClaimNextAsync("server", null, queue, Ct) is { } claimed) order.Add(claimed.Id);
|
|
Assert.SequenceEqual([render.Id, backgroundScan.Id, oldVerify.Id], order);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ClaimsByPriorityThenAgeAndKind()
|
|
{
|
|
var db = await OpenChronosAsync();
|
|
var oldBackground = TestData.Job(JobKind.Thumbs, JobPriority.Background, TestData.Now.AddMinutes(-10));
|
|
var newUser = TestData.Job(JobKind.Render, JobPriority.User, TestData.Now);
|
|
var oldUser = TestData.Job(JobKind.Import, JobPriority.User, TestData.Now.AddMinutes(-1));
|
|
var normal = TestData.Job(JobKind.Scan, JobPriority.Normal, TestData.Now.AddMinutes(-5));
|
|
foreach (var job in new[] { oldBackground, newUser, oldUser, normal }) await db.Jobs.EnqueueAsync(job, Ct);
|
|
|
|
var renderOnly = await db.Jobs.ClaimNextAsync("node-gpu", [JobKind.Render], Ct);
|
|
Assert.Equal(newUser.Id, renderOnly?.Id);
|
|
Assert.Equal(JobState.Running, renderOnly!.State);
|
|
Assert.Equal("node-gpu", renderOnly.NodeId);
|
|
Assert.Equal(1, renderOnly.Attempts);
|
|
Assert.NotNull(renderOnly.Started);
|
|
Assert.Null(await db.Jobs.ClaimNextAsync("node-gpu", [JobKind.Render], Ct));
|
|
|
|
var order = new List<Guid>();
|
|
while (await db.Jobs.ClaimNextAsync("server", cancellationToken: Ct) is { } claimed) order.Add(claimed.Id);
|
|
Assert.SequenceEqual([oldUser.Id, normal.Id, oldBackground.Id], order);
|
|
}
|
|
|
|
[Test]
|
|
public async Task SelectiveClaimersListQueuedKindsInClaimOrderAndClaimById()
|
|
{
|
|
var db = await OpenChronosAsync(multiConnection: true);
|
|
var oldBackground = TestData.Job(JobKind.Render, JobPriority.Background, TestData.Now.AddMinutes(-10));
|
|
var newUser = TestData.Job(JobKind.Thumbs, JobPriority.User, TestData.Now);
|
|
var oldUser = TestData.Job(JobKind.Render, JobPriority.User, TestData.Now.AddMinutes(-1));
|
|
var otherKind = TestData.Job(JobKind.Scan, JobPriority.User, TestData.Now.AddMinutes(-20));
|
|
foreach (var job in new[] { oldBackground, newUser, oldUser, otherKind }) await db.Jobs.EnqueueAsync(job, Ct);
|
|
|
|
var queued = await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 10, Ct);
|
|
Assert.SequenceEqual([oldUser.Id, newUser.Id, oldBackground.Id], queued.Select(j => j.Id));
|
|
Assert.Count(1, await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 1, Ct));
|
|
Assert.Empty(await db.Jobs.ListQueuedAsync([], 10, Ct));
|
|
|
|
// Workers racing for the same listed job: exactly one gets it, and it leaves the listing.
|
|
var claims = await Task.WhenAll(Enumerable.Range(0, 6).Select(w => Task.Run(() => db.Jobs.TryClaimAsync(oldUser.Id, "worker-" + w, Ct))));
|
|
var claimed = Assert.NotNull(claims.SingleOrDefault(c => c is not null));
|
|
Assert.Equal((JobState.Running, 1), (claimed.State, claimed.Attempts));
|
|
Assert.SequenceEqual([newUser.Id, oldBackground.Id], (await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 10, Ct)).Select(j => j.Id));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ConcurrentClaimersNeverShareAJob()
|
|
{
|
|
var db = await OpenChronosAsync(multiConnection: true);
|
|
var jobs = Enumerable.Range(0, 40).Select(i => TestData.Job(created: TestData.Now.AddSeconds(i))).ToList();
|
|
foreach (var job in jobs) await db.Jobs.EnqueueAsync(job, Ct);
|
|
|
|
var claims = new ConcurrentBag<(string Worker, Guid Job)>();
|
|
await Task.WhenAll(Enumerable.Range(0, 8).Select(w => Task.Run(async () =>
|
|
{
|
|
var worker = "worker-" + w;
|
|
while (await db.Jobs.ClaimNextAsync(worker, cancellationToken: Ct) is { } job) claims.Add((worker, job.Id));
|
|
})));
|
|
|
|
Assert.Equal(40, claims.Count);
|
|
Assert.Equal(40, claims.Select(c => c.Job).Distinct().Count());
|
|
foreach (var (worker, id) in claims)
|
|
{
|
|
var stored = await db.Jobs.GetAsync(id, Ct);
|
|
Assert.Equal(worker, stored!.NodeId);
|
|
Assert.Equal(1, stored.Attempts);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProgressIsRefusedOnceTheJobStopsRunning()
|
|
{
|
|
var db = await OpenChronosAsync();
|
|
var job = TestData.Job();
|
|
await db.Jobs.EnqueueAsync(job, Ct);
|
|
Assert.False(await db.Jobs.UpdateProgressAsync(job.Id, 1, 10, null, null, null, null, Ct));
|
|
await db.Jobs.ClaimNextAsync("w", cancellationToken: Ct);
|
|
Assert.True(await db.Jobs.UpdateProgressAsync(job.Id, 4, 10, 2.0, TimeSpan.FromSeconds(3), "cursor:4", "tail", Ct));
|
|
var running = await db.Jobs.GetAsync(job.Id, Ct);
|
|
Assert.Equal((4L, 10L, "cursor:4", "tail"), (running!.ProgressDone, running.ProgressTotal, running.Checkpoint, running.LogTail));
|
|
Assert.Equal(TimeSpan.FromSeconds(3), running.Eta);
|
|
|
|
Assert.True(await db.Jobs.SetStateAsync(job.Id, JobState.Cancelled, cancellationToken: Ct));
|
|
Assert.False(await db.Jobs.UpdateProgressAsync(job.Id, 5, 10, null, null, "cursor:5", null, Ct));
|
|
var cancelled = await db.Jobs.GetAsync(job.Id, Ct);
|
|
Assert.Equal(JobState.Cancelled, cancelled!.State);
|
|
Assert.NotNull(cancelled.Finished);
|
|
|
|
Assert.True(await db.Jobs.SetStateAsync(job.Id, JobState.Queued, cancellationToken: Ct));
|
|
Assert.Null((await db.Jobs.GetAsync(job.Id, Ct))!.Finished);
|
|
Assert.Equal(2, (await db.Jobs.ClaimNextAsync("w", cancellationToken: Ct))!.Attempts);
|
|
Assert.False(await db.Jobs.SetStateAsync(Guid.CreateVersion7(), JobState.Done, cancellationToken: Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ListingPagesByStateNewestFirst()
|
|
{
|
|
var db = await OpenChronosAsync();
|
|
var jobs = Enumerable.Range(0, 7).Select(i => TestData.Job(created: TestData.Now.AddMinutes(i % 3))).ToList();
|
|
foreach (var job in jobs) await db.Jobs.EnqueueAsync(job, Ct);
|
|
await db.Jobs.SetStateAsync(jobs[0].Id, JobState.Done, cancellationToken: Ct);
|
|
|
|
var seen = new List<Guid>();
|
|
string? cursor = null;
|
|
do
|
|
{
|
|
var page = await db.Jobs.ListAsync(JobState.Queued, limit: 2, cursor: cursor, cancellationToken: Ct);
|
|
seen.AddRange(page.Items.Select(j => j.Id));
|
|
cursor = page.NextCursor;
|
|
}
|
|
while (cursor is not null);
|
|
|
|
var expected = jobs.Skip(1)
|
|
.OrderByDescending(j => j.Created)
|
|
.ThenByDescending(j => j.Id.ToString("D"), StringComparer.Ordinal)
|
|
.Select(j => j.Id);
|
|
Assert.SequenceEqual(expected, seen);
|
|
Assert.SequenceEqual([jobs[0].Id], (await db.Jobs.ListAsync(JobState.Done, cancellationToken: Ct)).Items.Select(j => j.Id));
|
|
Assert.Equal(7, (await db.Jobs.ListAsync(cancellationToken: Ct)).Items.Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task HistoryPurgeKeepsRecentAndActiveJobs()
|
|
{
|
|
var db = await OpenChronosAsync();
|
|
var old = TestData.Job() with { State = JobState.Done, Finished = TestData.Now.AddDays(-100) };
|
|
var oldFailed = TestData.Job() with { State = JobState.Failed, Finished = TestData.Now.AddDays(-91) };
|
|
var recent = TestData.Job() with { State = JobState.Done, Finished = TestData.Now.AddDays(-10) };
|
|
var queued = TestData.Job(created: TestData.Now.AddDays(-200));
|
|
foreach (var job in new[] { old, oldFailed, recent, queued }) await db.Jobs.EnqueueAsync(job, Ct);
|
|
Assert.Equal(2, await db.Jobs.PurgeHistoryAsync(TestData.Now.AddDays(-90), Ct));
|
|
Assert.Equal(2, (await db.Jobs.ListAsync(cancellationToken: Ct)).Items.Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task RunningJobsAreRequeuedAfterACrash()
|
|
{
|
|
var db = await OpenChronosAsync();
|
|
for (var i = 0; i < 3; i++) await db.Jobs.EnqueueAsync(TestData.Job(created: TestData.Now.AddSeconds(i)), Ct);
|
|
var a = await db.Jobs.ClaimNextAsync("node-a", cancellationToken: Ct);
|
|
var b = await db.Jobs.ClaimNextAsync("node-b", cancellationToken: Ct);
|
|
await db.Jobs.UpdateProgressAsync(a!.Id, 3, 9, null, null, "resume-here", null, Ct);
|
|
|
|
Assert.Equal(1, await db.Jobs.RequeueRunningAsync("node-a", Ct));
|
|
var requeued = await db.Jobs.GetAsync(a.Id, Ct);
|
|
Assert.Equal(JobState.Queued, requeued!.State);
|
|
Assert.Equal("resume-here", requeued.Checkpoint);
|
|
Assert.Null(requeued.NodeId);
|
|
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(b!.Id, Ct))!.State);
|
|
Assert.Equal(1, await db.Jobs.RequeueRunningAsync(cancellationToken: Ct));
|
|
Assert.Equal(0, await db.Jobs.RequeueRunningAsync(cancellationToken: Ct));
|
|
}
|
|
}
|
|
|
|
public sealed class SqliteJobRepositoryTests(TestContext context) : JobRepositoryTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.SqliteMemory;
|
|
}
|
|
|
|
[RequiresEnvironment(PgTestServer.Variable)]
|
|
public sealed class PostgresJobRepositoryTests(TestContext context) : JobRepositoryTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.Postgres;
|
|
}
|