Owner review 1, phase 2 (docs/requests/2026-10-15-owner-review-1.md, R2). Jobs now run one at a time on the desktop and on the server. Parallelism (Settings > Rendering, TITANO_PARALLELISM) is how many photos the running job processes at once: thumbnail and proxy passes draw that many photos together and still write the pack and checkpoints in frame order, decode tiers are probed that many at a time, the deep analysis measures that many frames and a render decodes that many ahead. Wherever originals are developed whole (TIFF, PNG, RAW without a usable preview), fewer run at once within the memory budget, which the running job now has whole; the server's frame window is sized to the host instead of 1 GiB. A new job waits its turn and pausing the running job starts the next one. An optional priority queue gives each job kind a level (renders, imports and downloads high, verifies and proxies low by default): a queued job of a higher level pauses the running one without asking, and that job goes back to the queue with its checkpoint and resumes by itself afterwards. Jobs of the same level, and a job's own follow-ups, never interrupt it. The Jobs screen and the admin panel show the parallelism and the queue; Settings > Rendering and the admin panel's Settings set them, at once. TITANO_WORKERS and the Workers key of server.json still set the parallelism (1 to 64, with a warning at start for the variable). Titano.Tools gains i18n prune, which removed the renamed keys from the 36 machine-translated languages; the new strings are translated in all 38. Found while testing: uploads and imports failed when a pass read the sidecar while they wrote it (atomic writes now retry the rename while a reader holds the file, and never for a read-only one), and taking a lease just released while another process read it failed on Windows (the create now waits for the reader). Five adversarial review rounds checked the phase before this commit. Also updates BRIEF (12.8, 14.6, 14.8, 14.9, 24), DECISIONS, CHANGELOG, ARCHITECTURE, STATUS, the request file and the compose/Unraid templates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
224 lines
12 KiB
C#
224 lines
12 KiB
C#
using System.Collections.Concurrent;
|
|
using Titano.Core.Model;
|
|
using Titano.Data.Tests.Infrastructure;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Data.Tests.Repositories;
|
|
|
|
public abstract class JobRepositoryTests(TestContext context) : RepositoryTest(context)
|
|
{
|
|
[Test]
|
|
public async Task JobsRoundTripEveryField()
|
|
{
|
|
var db = await OpenTitanoAsync();
|
|
var job = TestData.Job(JobKind.Render, JobPriority.User) with
|
|
{
|
|
TargetIds = [Guid.CreateVersion7(), Guid.CreateVersion7()],
|
|
TargetLabel = "Harbour sunset",
|
|
ParametersJson = "{\"preset\": \"4K\"}",
|
|
ProgressDone = 5,
|
|
ProgressTotal = 10,
|
|
ItemsPerSecond = 2.5,
|
|
Eta = TimeSpan.FromSeconds(2),
|
|
Checkpoint = "frame:5",
|
|
LogTail = "line 1\nline 2",
|
|
CreatedBy = Guid.CreateVersion7(),
|
|
NodeId = "node-a",
|
|
Attempts = 1,
|
|
Error = "boom",
|
|
State = JobState.Failed,
|
|
Started = TestData.Now.AddMinutes(1),
|
|
Finished = TestData.Now.AddMinutes(2),
|
|
};
|
|
await db.Jobs.EnqueueAsync(job, Ct);
|
|
var read = Assert.NotNull(await db.Jobs.GetAsync(job.Id, Ct));
|
|
Assert.SequenceEqual(job.TargetIds, read.TargetIds);
|
|
Assert.Equal(job with { TargetIds = read.TargetIds }, read);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThePriorityQueueClaimsTheHigherKindsFirst()
|
|
{
|
|
// Owner review 1 (R2): on, the level of the kind comes before priority and age; off, the strict queue stays.
|
|
var db = await OpenTitanoAsync();
|
|
var oldVerify = TestData.Job(JobKind.Verify, JobPriority.User, TestData.Now.AddMinutes(-30));
|
|
var backgroundScan = TestData.Job(JobKind.Scan, JobPriority.Background, TestData.Now.AddMinutes(-20));
|
|
var userThumbs = TestData.Job(JobKind.Thumbs, JobPriority.User, TestData.Now.AddMinutes(-10));
|
|
var render = TestData.Job(JobKind.Render, JobPriority.Normal, TestData.Now);
|
|
foreach (var job in new[] { oldVerify, backgroundScan, userThumbs, render }) await db.Jobs.EnqueueAsync(job, Ct);
|
|
var queue = new Titano.Data.Jobs.JobPriorityQueue { Enabled = true };
|
|
JobKind[] kinds = [JobKind.Verify, JobKind.Scan, JobKind.Thumbs, JobKind.Render];
|
|
|
|
Assert.SequenceEqual([render.Id, userThumbs.Id, backgroundScan.Id, oldVerify.Id], (await db.Jobs.ListQueuedAsync(kinds, 10, queue, Ct)).Select(j => j.Id));
|
|
Assert.SequenceEqual([oldVerify.Id, userThumbs.Id, render.Id, backgroundScan.Id], (await db.Jobs.ListQueuedAsync(kinds, 10, Ct)).Select(j => j.Id));
|
|
Assert.Equal(userThumbs.Id, (await db.Jobs.ClaimNextAsync("server", [JobKind.Scan, JobKind.Thumbs, JobKind.Verify], queue, Ct))?.Id);
|
|
var order = new List<Guid>();
|
|
while (await db.Jobs.ClaimNextAsync("server", null, queue, Ct) is { } claimed) order.Add(claimed.Id);
|
|
Assert.SequenceEqual([render.Id, backgroundScan.Id, oldVerify.Id], order);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ClaimsByPriorityThenAgeAndKind()
|
|
{
|
|
var db = await OpenTitanoAsync();
|
|
var oldBackground = TestData.Job(JobKind.Thumbs, JobPriority.Background, TestData.Now.AddMinutes(-10));
|
|
var newUser = TestData.Job(JobKind.Render, JobPriority.User, TestData.Now);
|
|
var oldUser = TestData.Job(JobKind.Import, JobPriority.User, TestData.Now.AddMinutes(-1));
|
|
var normal = TestData.Job(JobKind.Scan, JobPriority.Normal, TestData.Now.AddMinutes(-5));
|
|
foreach (var job in new[] { oldBackground, newUser, oldUser, normal }) await db.Jobs.EnqueueAsync(job, Ct);
|
|
|
|
var renderOnly = await db.Jobs.ClaimNextAsync("node-gpu", [JobKind.Render], Ct);
|
|
Assert.Equal(newUser.Id, renderOnly?.Id);
|
|
Assert.Equal(JobState.Running, renderOnly!.State);
|
|
Assert.Equal("node-gpu", renderOnly.NodeId);
|
|
Assert.Equal(1, renderOnly.Attempts);
|
|
Assert.NotNull(renderOnly.Started);
|
|
Assert.Null(await db.Jobs.ClaimNextAsync("node-gpu", [JobKind.Render], Ct));
|
|
|
|
var order = new List<Guid>();
|
|
while (await db.Jobs.ClaimNextAsync("server", cancellationToken: Ct) is { } claimed) order.Add(claimed.Id);
|
|
Assert.SequenceEqual([oldUser.Id, normal.Id, oldBackground.Id], order);
|
|
}
|
|
|
|
[Test]
|
|
public async Task SelectiveClaimersListQueuedKindsInClaimOrderAndClaimById()
|
|
{
|
|
var db = await OpenTitanoAsync(multiConnection: true);
|
|
var oldBackground = TestData.Job(JobKind.Render, JobPriority.Background, TestData.Now.AddMinutes(-10));
|
|
var newUser = TestData.Job(JobKind.Thumbs, JobPriority.User, TestData.Now);
|
|
var oldUser = TestData.Job(JobKind.Render, JobPriority.User, TestData.Now.AddMinutes(-1));
|
|
var otherKind = TestData.Job(JobKind.Scan, JobPriority.User, TestData.Now.AddMinutes(-20));
|
|
foreach (var job in new[] { oldBackground, newUser, oldUser, otherKind }) await db.Jobs.EnqueueAsync(job, Ct);
|
|
|
|
var queued = await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 10, Ct);
|
|
Assert.SequenceEqual([oldUser.Id, newUser.Id, oldBackground.Id], queued.Select(j => j.Id));
|
|
Assert.Count(1, await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 1, Ct));
|
|
Assert.Empty(await db.Jobs.ListQueuedAsync([], 10, Ct));
|
|
|
|
// Workers racing for the same listed job: exactly one gets it, and it leaves the listing.
|
|
var claims = await Task.WhenAll(Enumerable.Range(0, 6).Select(w => Task.Run(() => db.Jobs.TryClaimAsync(oldUser.Id, "worker-" + w, Ct))));
|
|
var claimed = Assert.NotNull(claims.SingleOrDefault(c => c is not null));
|
|
Assert.Equal((JobState.Running, 1), (claimed.State, claimed.Attempts));
|
|
Assert.SequenceEqual([newUser.Id, oldBackground.Id], (await db.Jobs.ListQueuedAsync([JobKind.Render, JobKind.Thumbs], 10, Ct)).Select(j => j.Id));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ConcurrentClaimersNeverShareAJob()
|
|
{
|
|
var db = await OpenTitanoAsync(multiConnection: true);
|
|
var jobs = Enumerable.Range(0, 40).Select(i => TestData.Job(created: TestData.Now.AddSeconds(i))).ToList();
|
|
foreach (var job in jobs) await db.Jobs.EnqueueAsync(job, Ct);
|
|
|
|
var claims = new ConcurrentBag<(string Worker, Guid Job)>();
|
|
await Task.WhenAll(Enumerable.Range(0, 8).Select(w => Task.Run(async () =>
|
|
{
|
|
var worker = "worker-" + w;
|
|
while (await db.Jobs.ClaimNextAsync(worker, cancellationToken: Ct) is { } job) claims.Add((worker, job.Id));
|
|
})));
|
|
|
|
Assert.Equal(40, claims.Count);
|
|
Assert.Equal(40, claims.Select(c => c.Job).Distinct().Count());
|
|
foreach (var (worker, id) in claims)
|
|
{
|
|
var stored = await db.Jobs.GetAsync(id, Ct);
|
|
Assert.Equal(worker, stored!.NodeId);
|
|
Assert.Equal(1, stored.Attempts);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProgressIsRefusedOnceTheJobStopsRunning()
|
|
{
|
|
var db = await OpenTitanoAsync();
|
|
var job = TestData.Job();
|
|
await db.Jobs.EnqueueAsync(job, Ct);
|
|
Assert.False(await db.Jobs.UpdateProgressAsync(job.Id, 1, 10, null, null, null, null, Ct));
|
|
await db.Jobs.ClaimNextAsync("w", cancellationToken: Ct);
|
|
Assert.True(await db.Jobs.UpdateProgressAsync(job.Id, 4, 10, 2.0, TimeSpan.FromSeconds(3), "cursor:4", "tail", Ct));
|
|
var running = await db.Jobs.GetAsync(job.Id, Ct);
|
|
Assert.Equal((4L, 10L, "cursor:4", "tail"), (running!.ProgressDone, running.ProgressTotal, running.Checkpoint, running.LogTail));
|
|
Assert.Equal(TimeSpan.FromSeconds(3), running.Eta);
|
|
|
|
Assert.True(await db.Jobs.SetStateAsync(job.Id, JobState.Cancelled, cancellationToken: Ct));
|
|
Assert.False(await db.Jobs.UpdateProgressAsync(job.Id, 5, 10, null, null, "cursor:5", null, Ct));
|
|
var cancelled = await db.Jobs.GetAsync(job.Id, Ct);
|
|
Assert.Equal(JobState.Cancelled, cancelled!.State);
|
|
Assert.NotNull(cancelled.Finished);
|
|
|
|
Assert.True(await db.Jobs.SetStateAsync(job.Id, JobState.Queued, cancellationToken: Ct));
|
|
Assert.Null((await db.Jobs.GetAsync(job.Id, Ct))!.Finished);
|
|
Assert.Equal(2, (await db.Jobs.ClaimNextAsync("w", cancellationToken: Ct))!.Attempts);
|
|
Assert.False(await db.Jobs.SetStateAsync(Guid.CreateVersion7(), JobState.Done, cancellationToken: Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ListingPagesByStateNewestFirst()
|
|
{
|
|
var db = await OpenTitanoAsync();
|
|
var jobs = Enumerable.Range(0, 7).Select(i => TestData.Job(created: TestData.Now.AddMinutes(i % 3))).ToList();
|
|
foreach (var job in jobs) await db.Jobs.EnqueueAsync(job, Ct);
|
|
await db.Jobs.SetStateAsync(jobs[0].Id, JobState.Done, cancellationToken: Ct);
|
|
|
|
var seen = new List<Guid>();
|
|
string? cursor = null;
|
|
do
|
|
{
|
|
var page = await db.Jobs.ListAsync(JobState.Queued, limit: 2, cursor: cursor, cancellationToken: Ct);
|
|
seen.AddRange(page.Items.Select(j => j.Id));
|
|
cursor = page.NextCursor;
|
|
}
|
|
while (cursor is not null);
|
|
|
|
var expected = jobs.Skip(1)
|
|
.OrderByDescending(j => j.Created)
|
|
.ThenByDescending(j => j.Id.ToString("D"), StringComparer.Ordinal)
|
|
.Select(j => j.Id);
|
|
Assert.SequenceEqual(expected, seen);
|
|
Assert.SequenceEqual([jobs[0].Id], (await db.Jobs.ListAsync(JobState.Done, cancellationToken: Ct)).Items.Select(j => j.Id));
|
|
Assert.Equal(7, (await db.Jobs.ListAsync(cancellationToken: Ct)).Items.Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task HistoryPurgeKeepsRecentAndActiveJobs()
|
|
{
|
|
var db = await OpenTitanoAsync();
|
|
var old = TestData.Job() with { State = JobState.Done, Finished = TestData.Now.AddDays(-100) };
|
|
var oldFailed = TestData.Job() with { State = JobState.Failed, Finished = TestData.Now.AddDays(-91) };
|
|
var recent = TestData.Job() with { State = JobState.Done, Finished = TestData.Now.AddDays(-10) };
|
|
var queued = TestData.Job(created: TestData.Now.AddDays(-200));
|
|
foreach (var job in new[] { old, oldFailed, recent, queued }) await db.Jobs.EnqueueAsync(job, Ct);
|
|
Assert.Equal(2, await db.Jobs.PurgeHistoryAsync(TestData.Now.AddDays(-90), Ct));
|
|
Assert.Equal(2, (await db.Jobs.ListAsync(cancellationToken: Ct)).Items.Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task RunningJobsAreRequeuedAfterACrash()
|
|
{
|
|
var db = await OpenTitanoAsync();
|
|
for (var i = 0; i < 3; i++) await db.Jobs.EnqueueAsync(TestData.Job(created: TestData.Now.AddSeconds(i)), Ct);
|
|
var a = await db.Jobs.ClaimNextAsync("node-a", cancellationToken: Ct);
|
|
var b = await db.Jobs.ClaimNextAsync("node-b", cancellationToken: Ct);
|
|
await db.Jobs.UpdateProgressAsync(a!.Id, 3, 9, null, null, "resume-here", null, Ct);
|
|
|
|
Assert.Equal(1, await db.Jobs.RequeueRunningAsync("node-a", Ct));
|
|
var requeued = await db.Jobs.GetAsync(a.Id, Ct);
|
|
Assert.Equal(JobState.Queued, requeued!.State);
|
|
Assert.Equal("resume-here", requeued.Checkpoint);
|
|
Assert.Null(requeued.NodeId);
|
|
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(b!.Id, Ct))!.State);
|
|
Assert.Equal(1, await db.Jobs.RequeueRunningAsync(cancellationToken: Ct));
|
|
Assert.Equal(0, await db.Jobs.RequeueRunningAsync(cancellationToken: Ct));
|
|
}
|
|
}
|
|
|
|
public sealed class SqliteJobRepositoryTests(TestContext context) : JobRepositoryTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.SqliteMemory;
|
|
}
|
|
|
|
[RequiresEnvironment(PgTestServer.Variable)]
|
|
public sealed class PostgresJobRepositoryTests(TestContext context) : JobRepositoryTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.Postgres;
|
|
}
|