Owner review 1, phase 2 (docs/requests/2026-10-15-owner-review-1.md, R2). Jobs now run one at a time on the desktop and on the server. Parallelism (Settings > Rendering, TITANO_PARALLELISM) is how many photos the running job processes at once: thumbnail and proxy passes draw that many photos together and still write the pack and checkpoints in frame order, decode tiers are probed that many at a time, the deep analysis measures that many frames and a render decodes that many ahead. Wherever originals are developed whole (TIFF, PNG, RAW without a usable preview), fewer run at once within the memory budget, which the running job now has whole; the server's frame window is sized to the host instead of 1 GiB. A new job waits its turn and pausing the running job starts the next one. An optional priority queue gives each job kind a level (renders, imports and downloads high, verifies and proxies low by default): a queued job of a higher level pauses the running one without asking, and that job goes back to the queue with its checkpoint and resumes by itself afterwards. Jobs of the same level, and a job's own follow-ups, never interrupt it. The Jobs screen and the admin panel show the parallelism and the queue; Settings > Rendering and the admin panel's Settings set them, at once. TITANO_WORKERS and the Workers key of server.json still set the parallelism (1 to 64, with a warning at start for the variable). Titano.Tools gains i18n prune, which removed the renamed keys from the 36 machine-translated languages; the new strings are translated in all 38. Found while testing: uploads and imports failed when a pass read the sidecar while they wrote it (atomic writes now retry the rename while a reader holds the file, and never for a read-only one), and taking a lease just released while another process read it failed on Windows (the create now waits for the reader). Five adversarial review rounds checked the phase before this commit. Also updates BRIEF (12.8, 14.6, 14.8, 14.9, 24), DECISIONS, CHANGELOG, ARCHITECTURE, STATUS, the request file and the compose/Unraid templates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
98 lines
5.3 KiB
C#
98 lines
5.3 KiB
C#
using Titano.Core.Archiving;
|
|
using Titano.Data.Archiving;
|
|
using Titano.Testing;
|
|
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Titano.Data.Tests.Archiving;
|
|
|
|
/// <summary>
|
|
/// Thumbnail and proxy passes working on several photos at once (owner review 1, R2: the parallelism decides how many
|
|
/// photos the running job processes at once): the same images as one at a time, written in frame order, and checkpoints
|
|
/// that still mean every frame before them is done.
|
|
/// </summary>
|
|
[Category("archive")]
|
|
public sealed class ParallelPassTests(TestContext context)
|
|
{
|
|
[Test]
|
|
public async Task SeveralPhotosAtOnceDrawTheSameThumbnailsInFrameOrder()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
// The pass commits its pack and checkpoints every 200 frames.
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 430);
|
|
await fx.ScanAsync();
|
|
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single();
|
|
var sequence = (await fx.Db.Sequences.GetAsync(location.Id))!;
|
|
var folder = fx.Paths.Sequence("a");
|
|
|
|
var parallel = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", Parallelism = () => 4 });
|
|
var checkpoints = new List<(int Next, int Total)>();
|
|
var result = await parallel.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, (next, total) =>
|
|
{
|
|
checkpoints.Add((next, total));
|
|
return ValueTask.CompletedTask;
|
|
}, fx.Ct);
|
|
Assert.Equal((430, 0, 0), (result.Built, result.Reused, result.Undecodable));
|
|
Assert.SequenceEqual([(200, 430), (400, 430)], checkpoints);
|
|
|
|
Dictionary<int, byte[]> drawn;
|
|
using (var pack = ThumbPackReader.Open(folder.ThumbsPackPath))
|
|
{
|
|
var entries = pack.Entries.Values.OrderBy(e => e.FrameIndex).ToList();
|
|
Assert.Count(430, entries);
|
|
// Appended in frame order, as one at a time.
|
|
Assert.SequenceEqual(entries.Select(e => e.Offset).Order(), entries.Select(e => e.Offset));
|
|
drawn = entries.ToDictionary(e => e.FrameIndex, e => pack.Read(e.FrameIndex)!);
|
|
}
|
|
|
|
var single = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", Parallelism = () => 1 });
|
|
await single.Images.BuildThumbnailsAsync(fx.Archive, sequence, force: true, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
using (var pack = ThumbPackReader.Open(folder.ThumbsPackPath))
|
|
{
|
|
foreach (var (index, jpeg) in drawn) Assert.SequenceEqual(jpeg, pack.Read(index)!);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task SeveralPhotosAtOnceWriteEveryProxyAndAResumedPassChecksWhatTheEarlierRunDid()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 24);
|
|
await fx.ScanAsync();
|
|
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single();
|
|
var sequence = (await fx.Db.Sequences.GetAsync(location.Id))!;
|
|
var folder = fx.Paths.Sequence("a");
|
|
var parallel = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", Parallelism = () => 6 });
|
|
|
|
// Resumed at frame 10: it makes the proxies from there on, and those before it, which the earlier run should have
|
|
// made, are missing, so the sequence is not built.
|
|
var resumed = await parallel.Images.BuildProxiesAsync(fx.Archive, sequence, false, 10, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.Equal((14, 0, 0), (resumed.Built, resumed.Reused, resumed.Undecodable));
|
|
Assert.FileMissing(folder.Proxy(0));
|
|
Assert.FileExists(folder.Proxy(10));
|
|
Assert.False((await fx.Db.Sequences.GetAsync(location.Id))!.Status.HasFlag(Titano.Core.Model.SequenceStatus.ProxiesBuilt), "not built");
|
|
|
|
var full = await parallel.Images.BuildProxiesAsync(fx.Archive, sequence, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.Equal((10, 14), (full.Built, full.Reused));
|
|
foreach (var frame in ManifestStore.ReadSequence(folder.ManifestPath).Frames)
|
|
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, null), $"proxy of {frame.File}");
|
|
Assert.True((await fx.Db.Sequences.GetAsync(location.Id))!.Status.HasFlag(Titano.Core.Model.SequenceStatus.ProxiesBuilt), "built");
|
|
}
|
|
|
|
[Test]
|
|
public void LargeOriginalsDecodeFewerAtOnceWithinTheMemoryBudget()
|
|
{
|
|
// 24 MP takes some 768 MB while it is decoded.
|
|
const long sensor24 = 6000 * 4000;
|
|
Assert.Equal(2, DecodeBudget.PhotosAtOnce(8, 2L << 30, sensor24));
|
|
Assert.Equal(1, DecodeBudget.PhotosAtOnce(8, 512L << 20, sensor24));
|
|
Assert.Equal(8, DecodeBudget.PhotosAtOnce(8, 64L << 30, sensor24));
|
|
Assert.Equal(1, DecodeBudget.PhotosAtOnce(1, 64L << 30, sensor24));
|
|
// Small photos all fit; an unknown size counts as 24 MP.
|
|
Assert.Equal(8, DecodeBudget.PhotosAtOnce(8, 2L << 30, 48 * 32));
|
|
Assert.Equal(2, DecodeBudget.PhotosAtOnce(8, 2L << 30, 0));
|
|
// The size a damaged file claims never overflows: one decode at a time.
|
|
Assert.Equal(1, DecodeBudget.PhotosAtOnce(8, 64L << 30, 1L << 59));
|
|
Assert.Equal(1, DecodeBudget.PhotosAtOnce(8, 64L << 30, long.MaxValue));
|
|
}
|
|
}
|