From fba05c2840734c681ac5acade8e5a5bf33fb9682 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 15:58:37 +0000 Subject: [PATCH 1/6] fix(archive): scans leave sequences waiting for a folder alone and build server images A sequence created through POST /api/v1/sequences keeps an empty folder until its first frame. Scans no longer adopt loose photos at the archive root under its id, and a rebuild, the orphans report and fix, verify and the scan follow-ups no longer treat it as the root sequence, as missing or as a sequence to analyse. All of them use ArchiveIngest.AwaitsFolder. On the server (ArchiveServiceOptions.ImagesAfterScan), a scan that indexes sequences without thumbnails or proxies queues background Thumbs and Proxies passes for them, as an upload does. A scan that finds photos added or removed clears ThumbsBuilt and ProxiesBuilt together with AnalyzedLight. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/ArchiveIndexer.cs | 56 ++++++++--- .../Archiving/ArchiveMaintenance.cs | 12 ++- src/Titano.Data/Archiving/ArchiveService.cs | 4 +- src/Titano.Data/Archiving/ArchiveVerifier.cs | 8 +- .../Archiving/Jobs/ScanJobHandler.cs | 49 +++++++++- src/Titano.Data/Archiving/SequenceWriter.cs | 7 ++ src/Titano.Server/Storage/ArchiveModule.cs | 7 +- .../Archiving/ArchiveIngestTests.cs | 92 +++++++++++++++++++ .../Archiving/ArchiveJobTests.cs | 54 +++++++++++ .../Storage/ArchiveModuleTests.cs | 30 +++++- 10 files changed, 290 insertions(+), 29 deletions(-) diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index a85265da..ece6e735 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -20,7 +20,9 @@ public enum ScanMode public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed); /// Folders that could not be indexed, with the reason; they are retried by the next scan. -public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed); +/// The sequences this run indexed ( of them), in folder order. +public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed, + IReadOnlyList Sequences); /// /// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts @@ -49,11 +51,15 @@ public sealed class ArchiveIndexer if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false); var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken); var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false); - var locations = new Locations(await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)); + var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + // A sequence waiting for its first frame has no folder yet: the archive root, adopted or re-adopted, never takes its id. + var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false); + var locations = new Locations(rows.Where(r => !awaiting.Contains(r.Id))); var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null; using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null; - int indexed = 0, adopted = 0; + int adopted = 0; + var indexed = new List(); var failed = new List<(string, string)>(); for (var i = 0; i < folders.Count; i++) { @@ -67,14 +73,14 @@ public sealed class ArchiveIndexer try { var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos); - var bytes = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false); + var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false); if (snapshotWriter is not null) { var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath; snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value, bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false)); } - indexed++; + indexed.Add(id); if (folder.Sidecar is null) adopted++; } catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) @@ -88,7 +94,7 @@ public sealed class ArchiveIndexer var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0; if (failed.Count == 0) snapshotWriter?.Commit(); - return new ScanSummary(folders.Count, indexed, adopted, removed, failed); + return new ScanSummary(folders.Count, indexed.Count, adopted, removed, failed, indexed); } /// @@ -156,8 +162,8 @@ public sealed class ArchiveIndexer } } - /// Indexes one folder; returns the sidecar bytes when they were read and not rewritten since (for the snapshot). - private async Task IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot, + /// Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot). + private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot, Locations locations, CancellationToken cancellationToken) { var sequence = paths.Sequence(folder.RelativeFolder); @@ -189,10 +195,11 @@ public sealed class ArchiveIndexer } if (reconcile || File.Exists(sequence.IngestJournalPath)) { - // Photos added, gone or back: the light analysis no longer describes the frames, so the scan's follow-up runs it again. + // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, so the scan's + // follow-ups make them again (the images on a server, ArchiveServiceOptions.ImagesAfterScan). manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled - ? reconciled with { Status = reconciled.Status & ~SequenceStatus.AnalyzedLight } + ? reconciled with { Status = reconciled.Status & ~(SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt) } : null, cancellationToken).ConfigureAwait(false); bytes = null; @@ -208,7 +215,23 @@ public sealed class ArchiveIndexer await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); } locations.Set(manifest.Id, folder.RelativeFolder); - return bytes; + return (manifest.Id, bytes); + } + + /// + /// The sequences among created without photos that still wait for their first frame + /// (). Their empty folder is not the archive root: scans do not adopt the root + /// under their id, a rebuild keeps them although they have no sidecar yet, and orphan checks do not count them missing. + /// + internal async Task> AwaitingFolderAsync(ArchivePaths paths, IEnumerable locations, CancellationToken cancellationToken) + { + var awaiting = new HashSet(); + foreach (var location in locations.Where(l => l.Folder.Length == 0)) + { + if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is { } sequence && ArchiveIngest.AwaitsFolder(sequence, paths)) + awaiting.Add(sequence.Id); + } + return awaiting; } /// The index holds a project for this sequence and other frames than the sidecar: its positions may move. @@ -261,16 +284,21 @@ public sealed class ArchiveIndexer UpdatedAt = DateTimeOffset.UtcNow, }; - /// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). + /// + /// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence + /// waiting for its first frame has no sidecar yet and stays. + /// private async Task RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList folders, Locations locations, CancellationToken cancellationToken) { ArchiveVolume.EnsureWritable(paths); var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal); + var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false); var removed = 0; - foreach (var location in await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)) + foreach (var location in rows) { - if (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id) continue; + if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue; if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++; await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false); } diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs index c3cc7dc7..6a75cde9 100644 --- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -71,7 +71,10 @@ public sealed class ArchiveMaintenance { var paths = new ArchivePaths(archive.RootPath); if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline); - var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + // A sequence waiting for its first frame has no folder yet: none of it is missing, and the archive root is not its folder. + var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false); + var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList(); var sequences = new List(); var missing = new List(); foreach (var location in locations) @@ -91,7 +94,10 @@ public sealed class ArchiveMaintenance return new OrphanReport(sequences, missing, unindexed); } - /// Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept. + /// + /// Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept, and + /// so are sequences waiting for their first frame (), which have none yet. + /// public async Task RemoveMissingSequencesAsync(Archive archive, IReadOnlyList sequenceIds, CancellationToken cancellationToken = default) { var paths = new ArchivePaths(archive.RootPath); @@ -100,7 +106,7 @@ public sealed class ArchiveMaintenance foreach (var id in sequenceIds) { if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue; - if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath)) continue; + if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath) || ArchiveIngest.AwaitsFolder(sequence, paths)) continue; if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++; await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false); } diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs index 562d9e42..0d43f48e 100644 --- a/src/Titano.Data/Archiving/ArchiveService.cs +++ b/src/Titano.Data/Archiving/ArchiveService.cs @@ -54,8 +54,8 @@ public sealed class ArchiveService : IJobHandlerProvider /// Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the . public IReadOnlyList CreateJobHandlers() => [ - new ScanJobHandler(_db, Indexer, JobKind.Scan), - new ScanJobHandler(_db, Indexer, JobKind.Rebuild), + new ScanJobHandler(_db, Indexer, JobKind.Scan, _writer.Options.ImagesAfterScan), + new ScanJobHandler(_db, Indexer, JobKind.Rebuild, _writer.Options.ImagesAfterScan), new VerifyJobHandler(_db, Verifier), new ImportJobHandler(_db, Imports), new ImageJobHandler(_db, Images, JobKind.Thumbs), diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs index db6bcd8c..3cdf58a7 100644 --- a/src/Titano.Data/Archiving/ArchiveVerifier.cs +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -44,7 +44,7 @@ public sealed class ArchiveVerifier { var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); - var targets = await TargetsAsync(archive, sequenceIds, cancellationToken).ConfigureAwait(false); + var targets = await TargetsAsync(archive, paths, sequenceIds, cancellationToken).ConfigureAwait(false); var bytesTotal = targets.Sum(t => t.TotalBytes); var bytesDone = targets.Take(resume.Sequence).Sum(t => t.TotalBytes); long checkedFrames = 0, ok = 0, hashed = 0; @@ -82,13 +82,15 @@ public sealed class ArchiveVerifier return new VerifyReport(targets.Count, checkedFrames, ok, hashed, problems); } - private async Task> TargetsAsync(Archive archive, IReadOnlyList? sequenceIds, CancellationToken cancellationToken) + /// The sequences to check; one waiting for its first frame () has no sidecar to check yet. + private async Task> TargetsAsync(Archive archive, ArchivePaths paths, IReadOnlyList? sequenceIds, CancellationToken cancellationToken) { var ids = sequenceIds ?? (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id).ToList(); var targets = new List(ids.Count); foreach (var id in ids) { - if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id) targets.Add(s); + if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id && !ArchiveIngest.AwaitsFolder(s, paths)) + targets.Add(s); } return targets; } diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs index e0812277..2b77e95a 100644 --- a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs +++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs @@ -1,7 +1,9 @@ +using Titano.Core.Archiving; using Titano.Core.Localization; using Titano.Core.Model; using Titano.Data.Jobs; using Titano.Data.Processing; +using Titano.Data.Repositories; namespace Titano.Data.Archiving.Jobs; @@ -18,9 +20,12 @@ internal static class JobArchive /// (incremental, or full with ) and /// . Checkpoint: after:<folder>, the last folder indexed. When anything was /// indexed, the sequences without a current light analysis (adopted folders, new sidecars, photos added to a folder) are -/// queued for it, and for a background deep analysis too when asks. +/// queued for it, and for a background deep analysis too when asks. On the +/// server (, ) the sequences it indexed +/// without thumbnails or proxies are queued for them too, as after an upload. Sequences waiting for their first frame +/// () have nothing to analyse or draw yet and are left out. /// -internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler +internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind, bool imagesAfterScan = false) : IJobHandler { private const string Prefix = "after:"; @@ -42,9 +47,47 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, // A resumed scan may have indexed its folders before the interruption. if (summary.Indexed > 0 || resume is not null) { - var unanalysed = await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false); + var awaiting = await indexer.AwaitingFolderAsync(new ArchivePaths(archive.RootPath), + await db.Sequences.ListLocationsAsync(archive.Id, context.Cancellation).ConfigureAwait(false), context.Cancellation).ConfigureAwait(false); + var unanalysed = (await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false)) + .Where(id => !awaiting.Contains(id)).ToList(); await AnalysisFollowUp.QueueAsync(db, context, unanalysed).ConfigureAwait(false); if (kind == JobKind.Scan && parameters.DeepAnalysis) await AnalysisFollowUp.QueueDeepAsync(db, context, unanalysed).ConfigureAwait(false); + if (imagesAfterScan) + { + var resumed = resume is not null; + await QueueImagesAsync(context, archive, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, summary.Sequences, resumed, awaiting).ConfigureAwait(false); + // An archive whose policy makes no proxies would get a pass that does nothing. + if (archive.ProxyLongEdge != ProxyPolicy.Disabled) + await QueueImagesAsync(context, archive, JobKind.Proxies, SequenceStatus.ProxiesBuilt, summary.Sequences, resumed, awaiting).ConfigureAwait(false); + } } } + + /// + /// Queues one background (thumbnails or proxies) of the sequences this run indexed whose + /// flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the + /// flag then). After a resumed run, which may have indexed sequences before the interruption, every sequence of the + /// archive without the flag but those in . Sequences already waiting in a queued pass of that + /// kind are left out. + /// + private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList indexed, + bool resumed, IReadOnlySet awaiting) + { + var missing = new HashSet(); + var query = new SequenceQuery { ArchiveId = archive.Id, WithoutStatus = built, Sort = SequenceSort.Oldest, Limit = 1000 }; + do + { + var page = await db.Sequences.ListAsync(query, context.Cancellation).ConfigureAwait(false); + missing.UnionWith(page.Items.Select(s => s.Id)); + query = query with { Cursor = page.NextCursor }; + } + while (query.Cursor is not null); + missing.ExceptWith(awaiting); + var waiting = (await db.Jobs.ListQueuedAsync([pass], 1000, context.Cancellation).ConfigureAwait(false)).SelectMany(j => j.TargetIds).ToHashSet(); + var targets = (resumed ? missing : indexed.Where(missing.Contains)).Where(id => !waiting.Contains(id)).Distinct().ToList(); + if (targets.Count == 0) return; + var request = pass == JobKind.Thumbs ? ArchiveJobs.Thumbnails(targets, context.Job.TargetLabel) : ArchiveJobs.Proxies(targets, context.Job.TargetLabel); + await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false); + } } diff --git a/src/Titano.Data/Archiving/SequenceWriter.cs b/src/Titano.Data/Archiving/SequenceWriter.cs index fe636cf4..d91283ef 100644 --- a/src/Titano.Data/Archiving/SequenceWriter.cs +++ b/src/Titano.Data/Archiving/SequenceWriter.cs @@ -17,6 +17,13 @@ public sealed record ArchiveServiceOptions public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2); public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration; + + /// + /// Scans queue thumbnails and proxies of the sequences they index without them, as an upload does (the server, whose + /// clients load both from it: BRIEF §4.3, §14.4). The desktop decodes its thumbnails on demand and makes proxies when + /// asked. + /// + public bool ImagesAfterScan { get; init; } } /// diff --git a/src/Titano.Server/Storage/ArchiveModule.cs b/src/Titano.Server/Storage/ArchiveModule.cs index ad5bebc2..ee318bf0 100644 --- a/src/Titano.Server/Storage/ArchiveModule.cs +++ b/src/Titano.Server/Storage/ArchiveModule.cs @@ -8,8 +8,9 @@ namespace Titano.Server.Storage; /// /// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and /// thumbnails, its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance) and the periodic -/// incremental scan (). Decoding is managed only (BRIEF §4.2). The daily backup of an -/// SQLite index runs with it (). +/// incremental scan (); a scan queues thumbnails and proxies of what it indexes, as an upload +/// does. Decoding is managed only (BRIEF §4.2). The daily backup of an SQLite index runs with it +/// (). /// public static class ArchiveModule { @@ -19,7 +20,7 @@ public static class ArchiveModule public static void Register(IServiceCollection services) { services.AddSingleton(sp => new ArchiveService(sp.GetRequiredService(), - new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = Owner })); + new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = Owner, ImagesAfterScan = true })); services.AddSingleton(sp => sp.GetRequiredService().Ingest); services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs index 548f799c..c881a068 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -17,6 +17,24 @@ public sealed class ArchiveIngestTests(TestContext context) Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") => new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null); + /// A sequence as POST /api/v1/sequences creates it: a row without a folder or frames. + private static async Task CreateWithoutPhotosAsync(ArchiveFixture fx, string title) + { + var created = new Sequence + { + Id = Guid.CreateVersion7(), + ArchiveId = fx.Archive.Id, + Title = title, + Folder = string.Empty, + Rating = 4, + Origin = SequenceOrigin.Server, + OwnerUserId = Guid.CreateVersion7(), + UpdatedAt = DateTimeOffset.UtcNow, + }; + await fx.Db.Sequences.UpsertAsync(created, fx.Ct); + return created; + } + [Test] public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates() { @@ -242,6 +260,80 @@ public sealed class ArchiveIngestTests(TestContext context) Assert.FileExists(staged.Path); } + [Test] + public async Task ASequenceWaitingForItsFirstFrameNeverLendsItsIdToTheArchiveRoot() + { + await using var fx = await CreateAsync(context); + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + // Loose photos at the root, which a scan adopts in place. + Photos.WriteSeries(fx.Archive.RootPath, Start.AddDays(-1), 2); + + await fx.ScanAsync(); + + var root = ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath); + Assert.NotEqual(waiting.Id, root.Id); + var rootRow = (await fx.Db.Sequences.GetAsync(root.Id))!; + Assert.Equal((string.Empty, 2), (rootRow.Folder, rootRow.FrameCount)); + var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal((string.Empty, 0, "Harbour sunset"), (row.Folder, row.FrameCount, row.Title)); + + // Its first frame still gets a folder of its own, and the root keeps its photos. + var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), sequenceId: waiting.Id), fx.Ct); + Assert.Equal(waiting.Id, first.SequenceId); + var placed = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal(("2026/2026-09-28 Canon EOS R5", 1), (placed.Folder, placed.FrameCount)); + Assert.Equal(root.Id, ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath).Id); + Assert.Equal(2, (await fx.Db.Sequences.GetAsync(root.Id))!.FrameCount); + } + + [Test] + public async Task ARootSequenceWhoseSidecarIsGoneIsAdoptedAgainUnderItsOwnId() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var adopted = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault()); + // Created later, so its id sorts after the root sequence's. + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + File.Delete(fx.Paths.Sequence(string.Empty).ManifestPath); + + await fx.ScanAsync(); + + Assert.Equal(adopted.Id, ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath).Id); + Assert.Equal(2, (await fx.Db.Sequences.GetAsync(adopted.Id))!.FrameCount); + var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal((string.Empty, 0), (row.Folder, row.FrameCount)); + } + + [Test] + public async Task RebuildOrphansAndVerifyLeaveASequenceWaitingForItsFirstFrame() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2); + await fx.ScanAsync(); + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + + // It has no sidecar yet, which is no reason to drop it. + var rebuilt = await fx.ScanAsync(ScanMode.Rebuild); + Assert.Equal(0, rebuilt.Removed); + var orphans = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct); + Assert.Empty(orphans.MissingSequences); + Assert.Empty(orphans.UnindexedFolders); + Assert.Equal(0, await fx.Service.RemoveMissingSequencesAsync(fx.Archive.Id, [waiting.Id], fx.Ct)); + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + Assert.Equal((1, 2L), (report.Sequences, report.Checked)); + var kept = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id)); + Assert.Equal(("Harbour sunset", 4), (kept.Title, kept.Rating)); + + // The archive root is not its folder: loose photos there are a folder the index does not know yet. + Photos.WriteSeries(fx.Archive.RootPath, Start.AddDays(-1), 1, firstSeed: 50); + Assert.SequenceEqual([string.Empty], (await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).UnindexedFolders); + + var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct); + Assert.Equal(waiting.Id, first.SequenceId); + Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder); + } + [Test] public async Task FramesInsertedBeforeOthersTakeTheProjectPositionsAlong() { diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs index 1a50f5d3..f6d5e090 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs @@ -3,6 +3,7 @@ using Titano.Core.Import; using Titano.Core.Model; using Titano.Data.Archiving; using Titano.Data.Archiving.Jobs; +using Titano.Data.Jobs; using Titano.Data.Repositories; using Titano.Testing; using static Titano.Data.Tests.Archiving.ArchiveFixture; @@ -40,6 +41,9 @@ public sealed class ArchiveJobTests(TestContext context) var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct); await fx.WaitForAsync(rebuild.Id, JobState.Done); Assert.Equal(2, (await LocationsAsync(fx)).Count); + // The desktop decodes its thumbnails on demand and makes proxies when asked. + Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items); + Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items); } [Test] @@ -88,6 +92,56 @@ public sealed class ArchiveJobTests(TestContext context) Assert.Contains("made 3, reused 3", done.LogTail); } + [Test] + public async Task AServerScanQueuesThumbnailsAndProxiesOfTheSequencesItIndexesWithoutThem() + { + await using var fx = await CreateAsync(context); + Populate(fx); + // The server's archive engine (ArchiveModule); the desktop's scans queue no image passes. + var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true }); + await using var engine = new JobEngine(fx.Db, service.CreateJobHandlers(), + new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await engine.StartAsync(fx.Ct); + + var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(scan.Id, JobState.Done); + var locations = await LocationsAsync(fx); + var first = new Dictionary(); + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(); + Assert.SequenceEqual(locations.Select(l => l.Id), pass.TargetIds); + Assert.Equal(JobPriority.Background, pass.Priority); + await fx.WaitForAsync(pass.Id, JobState.Done); + first[kind] = pass.Id; + } + foreach (var location in locations) Assert.True(Built((await fx.Db.Sequences.GetAsync(location.Id))!.Status), location.Folder); + + // Every folder indexed again, but none without its images: nothing is queued. + var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct); + await fx.WaitForAsync(full.Id, JobState.Done); + Assert.Count(1, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items); + Assert.Count(1, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items); + + // A photo added to a folder: its sequence lacks the images of that frame until the passes queued for it alone ran. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(5), 1, firstSeed: 90, firstNumber: 7); + var added = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(added.Id, JobState.Done); + var a = locations.Single(l => l.Folder == "a"); + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first[kind]); + Assert.SequenceEqual([a.Id], pass.TargetIds); + await fx.WaitForAsync(pass.Id, JobState.Done); + } + Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built again"); + using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath); + Assert.Count(7, pack.Entries); + Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies)); + + static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt); + } + [Test] public async Task AThumbnailPassFlagsFramesNothingDecodesAndClearsThemOnceTheyDecode() { diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs index 9112850f..9ca8dbc8 100644 --- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs @@ -68,7 +68,7 @@ public sealed class ArchiveModuleTests(TestContext context) } [Test] - public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPolling() + public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPollingWithThumbnailsAndProxies() { var photo = Photo(); // An adopted folder that already holds photos. @@ -78,8 +78,11 @@ public sealed class ArchiveModuleTests(TestContext context) File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo); }, ArchiveModule.Register); var archive = server.Services.GetRequiredService(); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); await UntilIndexedAsync(server, archive.Id, "roof"); await UntilScannedAsync(server); + // The scan queues the thumbnails and proxies clients load from the server, as an upload does. + await UntilImagesAsync(server, editor, archive.Id, "roof"); // Another writer adds a folder: the next poll queues a scan, but not while one is pending. Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden")); @@ -100,6 +103,7 @@ public sealed class ArchiveModuleTests(TestContext context) var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation)); Assert.Equal(JobPriority.Background, scan.Priority); await UntilIndexedAsync(server, archive.Id, "garden"); + await UntilImagesAsync(server, editor, archive.Id, "garden"); } private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder) @@ -112,6 +116,30 @@ public sealed class ArchiveModuleTests(TestContext context) } } + /// Waits until the first frame of the sequence in has its thumbnail and proxy served. + private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder) + { + var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder); + var frame = (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).First(); + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + foreach (var asset in new[] { "thumb", "proxy" }) + { + while (true) + { + using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}"); + if (response.StatusCode == HttpStatusCode.OK) + { + Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {folder}"); + break; + } + Assert.True(DateTime.UtcNow < until, $"no {asset} of {folder}: {(int)response.StatusCode}"); + await Task.Delay(100); + } + } + var built = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status; + Assert.True(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt), $"{folder}: {built}"); + } + /// Waits for the scan queued at start to finish: its sequences are indexed before it ends. private static async Task UntilScannedAsync(TestServer server) { From e01887ef516ce21f9fe6ae9703d7db45af3913c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 15:58:47 +0000 Subject: [PATCH 2/6] fix(archive): mass rename refuses a folder that gained a sequence after the dry run A scan can index a sequence inside a folder the plan moves after the dry run was shown (photos copied into a subfolder). The folder would then move with it while its index row stayed behind. Apply now lists the index again before anything moves and under each moving step's leases just before the move, and throws StaleRenamePlanException naming the newcomer. A resumed journal drops such a step while its folder has not moved yet. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/MassRenamer.cs | 46 +++++++++++++++-- .../Archiving/MassRenameTests.cs | 51 +++++++++++++++++++ 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/src/Titano.Data/Archiving/MassRenamer.cs b/src/Titano.Data/Archiving/MassRenamer.cs index 6846e70d..e6b1c11e 100644 --- a/src/Titano.Data/Archiving/MassRenamer.cs +++ b/src/Titano.Data/Archiving/MassRenamer.cs @@ -2,10 +2,14 @@ using Titano.Core.Archiving; using Titano.Core.Diagnostics; using Titano.Core.Localization; using Titano.Core.Model; +using Titano.Data.Repositories; namespace Titano.Data.Archiving; -/// The plan no longer matches the archive (a sequence moved or vanished since it was made). +/// +/// The plan no longer matches the archive: a sequence moved or vanished since it was made, or one was indexed inside a +/// folder the plan moves. +/// public sealed class StaleRenamePlanException(Guid sequenceId) : InvalidOperationException(Loc.L("Archive.Error.StalePlan")) { @@ -62,7 +66,8 @@ public sealed class MassRenamer private static RenameSubject Subject(Sequence s) => new(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens); /// Applies a plan; each completed step is journaled before the next starts. - /// A sequence is no longer where the plan expects it; nothing of that step was changed. + /// A sequence is no longer where the plan expects it, or a folder that moves holds + /// a sequence the plan does not take along; nothing of that step was changed. public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress? progress = null, CancellationToken cancellationToken = default) => RunAsync(archive, new RenameJournal(plan, []), resuming: false, progress, cancellationToken); @@ -85,12 +90,17 @@ public sealed class MassRenamer var completed = journal.Completed.ToHashSet(); if (!resuming) { - // A plan made before another change is refused before anything moves. + // A plan made before another change is refused before anything moves: a sequence moved, or one was indexed + // inside a folder that moves (a scan adopted a subfolder copied in since the dry run). foreach (var step in journal.Plan.Steps) { if ((await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false))?.Folder != step.FromFolder) throw new StaleRenamePlanException(step.SequenceId); } + var planned = journal.Plan.Steps.Select(s => s.SequenceId).ToHashSet(); + if (await StrangerInsideAsync(archive, journal.Plan.Steps.Where(s => s.MovesFolder && !s.Carried), planned, cancellationToken) + .ConfigureAwait(false) is { } stranger) + throw new StaleRenamePlanException(stranger.Id); } RenameJournal.Write(paths, journal); var steps = journal.Plan.Steps; @@ -147,6 +157,7 @@ public sealed class MassRenamer { if (await StandsAsync(c, resuming, cancellationToken).ConfigureAwait(false)) follow.Add(c); } + if (!await CarriesAllInsideAsync(archive, paths, step, carried, resuming, cancellationToken).ConfigureAwait(false)) return; RenamePlanner.MoveFolder(paths, step); await RecordAsync(archive, paths, step, cancellationToken).ConfigureAwait(false); // Moved with their carrier: only their sidecars and index rows follow. @@ -170,6 +181,35 @@ public sealed class MassRenamer return false; } + /// + /// Whether every sequence the index has inside the folder of a moving step moves with it as planned, checked again under + /// the step's leases just before the folder moves. One indexed there since the dry run would move without its index + /// row following, and was never shown in the plan: the step is refused, or dropped when resuming a folder that has not + /// moved yet (a folder moved before the interruption goes on; the next scan indexes the newcomer where it ended up). + /// + /// Not resuming, and such a sequence is inside the folder. + private async Task CarriesAllInsideAsync(Archive archive, ArchivePaths paths, RenameStep step, IReadOnlyList carried, bool resuming, + CancellationToken cancellationToken) + { + if (!step.MovesFolder) return true; + var taken = carried.Select(c => c.SequenceId).ToHashSet(); + if (await StrangerInsideAsync(archive, [step], taken, cancellationToken).ConfigureAwait(false) is not { } stranger) return true; + if (!resuming) throw new StaleRenamePlanException(stranger.Id); + if (!Directory.Exists(paths.Sequence(step.FromFolder).Folder)) return true; + Log.Warning("archive", $"Rename of {step.FromFolder} dropped: {stranger.Folder} was indexed inside it after the plan was made"); + return false; + } + + /// A sequence the index has inside the folder of one of that is not in ; null when there is none. + private async Task StrangerInsideAsync(Archive archive, IEnumerable moving, IReadOnlySet taken, + CancellationToken cancellationToken) + { + var folders = moving.Select(s => s.FromFolder).ToList(); + if (folders.Count == 0) return null; + return (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)) + .FirstOrDefault(l => !taken.Contains(l.Id) && folders.Any(f => RenamePlanner.IsInside(f, l.Folder))); + } + /// Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it. private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken) { diff --git a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs index d15b5036..cdcee194 100644 --- a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs +++ b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs @@ -105,6 +105,57 @@ public sealed class MassRenameTests(TestContext context) Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder))); } + [Test] + public async Task ASequenceIndexedInsideAMovingFolderAfterTheDryRunMakesThePlanStale() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips"), Start, 2); + await fx.ScanAsync(); + var trips = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [trips.Id], "moved/{date}", null, fx.Ct); + Assert.Count(1, plan.Steps); + + // Photos copied into a subfolder and indexed by the next scan, after the dry run was shown. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips", "day2"), Start.AddDays(1), 2, firstSeed: 10); + await fx.ScanAsync(); + var day2 = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "trips/day2"); + + var stale = await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct)); + Assert.Equal(day2.Id, stale.SequenceId); + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + Assert.SequenceEqual(["trips|trips|trips/IMG_0001.JPG,trips/IMG_0002.JPG", "trips/day2|day2|trips/day2/IMG_0001.JPG,trips/day2/IMG_0002.JPG"], + await StateAsync(fx, [trips.Id, day2.Id])); + } + + [Test] + public async Task AFolderThatGainedASequenceWhileTheRenameRanStaysWhereItIs() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0], ids[1]], "renamed/{date}", null, fx.Ct); + Assert.SequenceEqual(["inbox/a", "inbox/b"], plan.Steps.Select(s => s.FromFolder)); + + // Once the first folder moved, a scan indexes photos copied into the second one. + var late = Path.Combine(fx.Paths.Root, "inbox", "b", "late"); + var progress = new SyncProgress(done => + { + if (done != 1) return; + Photos.WriteSeries(late, Start.AddDays(5), 2, firstSeed: 40); + fx.ScanAsync().GetAwaiter().GetResult(); + }); + await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, progress, fx.Ct)); + var newcomer = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "inbox/b/late"); + Assert.True(Directory.Exists(late), "the folder did not move"); + Assert.Count(1, RenameJournal.ListPending(fx.Paths).Single().Completed); + + // Resumed, the rest of the plan runs but that step is dropped: every index row still names its files. + Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct)); + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + var state = await StateAsync(fx, [ids[0], ids[1], newcomer.Id]); + Assert.SequenceEqual(["renamed/2026-09-28", "inbox/b", "inbox/b/late"], state.Select(s => s.Split('|')[0])); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } + /// "trips" with photos of its own, "trips/day2" inside it and "trips/day2/dawn" inside that: three nested sequences. private static async Task> NestedAsync(ArchiveFixture fx) { From 98f301f5fd2bf59ed6e6bde84bc15af328d522b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 15:58:48 +0000 Subject: [PATCH 3/6] fix(server): answer every chunk request and out-of-range archive.json policies The chunk endpoint raised the request body limit only before reading a chunk. A chunk sent again after a lost answer, or to an unknown or finished upload, was left unread above Kestrel's 1 MiB default, so Kestrel closed the connection instead of delivering the answer. The limit is now raised before any answer, to the larger of the upload's and the server's chunk size, and never lowered. PUT /api/v1/archive/settings answered 500 when archive.json held a thumbnail size or proxy quality out of range, which the admin API cannot set. It now answers 409 archive_policy with the policy to correct in details (Server.Error.ArchivePolicy). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- i18n/en/Server.resx | 1 + i18n/it/Server.it.resx | 1 + src/Titano.Server/Api/ArchiveEndpoints.cs | 15 +++++++++++- src/Titano.Server/Api/UploadEndpoints.cs | 3 ++- src/Titano.Server/Storage/UploadService.cs | 7 ++++-- tests/Titano.Server.Tests/Api/AdminTests.cs | 16 +++++++++++++ .../Storage/UploadTests.cs | 23 +++++++++++++++++++ 7 files changed, 62 insertions(+), 4 deletions(-) diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 84cf4086..3166661a 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -29,6 +29,7 @@ Another program is changing the archive's settings; try again shortly.long The archive's archive.json is damaged, from a newer Titano or of another archive; check the archive folder.long The archive is offline or read-only; try again when it is back.long + The archive's archive.json has a thumbnail or proxy setting Titano cannot use; correct it in the archive folder.long Its photos are the archive's root folder, which never moves to the trash.long Wrong user name or password.long The request is not valid.long diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index b0aa9af3..fc9c78d1 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -27,6 +27,7 @@ Un altro programma sta modificando le impostazioni dell'archivio; riprova tra poco. Il file archive.json dell'archivio è danneggiato, di un Titano più recente o di un altro archivio; controlla la cartella dell'archivio. L'archivio non è raggiungibile o è di sola lettura; riprova quando torna disponibile. + Il file archive.json dell'archivio ha un'impostazione delle miniature o dei proxy che Titano non può usare; correggila nella cartella dell'archivio. Le sue foto sono la cartella principale dell'archivio, che non va mai nel cestino. Nome utente o password errati. La richiesta non è valida. diff --git a/src/Titano.Server/Api/ArchiveEndpoints.cs b/src/Titano.Server/Api/ArchiveEndpoints.cs index 953df725..c71e1df6 100644 --- a/src/Titano.Server/Api/ArchiveEndpoints.cs +++ b/src/Titano.Server/Api/ArchiveEndpoints.cs @@ -67,7 +67,9 @@ public static class ArchiveEndpoints /// Writes the name, templates and proxy policy into the archive's archive.json through the archive engine (so a /// restart and the proxy jobs see them), then into the index. 503 archive_offline while the archive cannot be /// written; 409 archive_busy while another process holds its settings; 409 archive_manifest when its - /// archive.json is damaged, written by a newer Titano or of another archive (nothing is written). + /// archive.json is damaged, written by a newer Titano or of another archive; 409 archive_policy when it holds a + /// thumbnail or proxy setting this API cannot set and Titano cannot use (details: thumbnailPolicy or + /// proxyPolicy, the property to correct in the file). Nothing is written. /// private static async Task UpdateAsync(HttpContext context, Caller caller, ArchiveSettingsDto request, TitanoDatabase db, ServerArchive archive, [FromServices] ArchiveService? archives, AuditLog audit) @@ -99,6 +101,17 @@ public static class ArchiveEndpoints { return ApiErrors.Conflict(context, "archive_manifest"); } + catch (ArgumentException ex) + { + // The request's fields were checked above: what fails is a value read from archive.json (a thumbnail size, or a + // proxy quality, written by hand or by another tool), which the admin corrects in the file, not in this form. + return ApiErrors.Conflict(context, "archive_policy", ex.ParamName switch + { + "thumbnailLongEdge" => "thumbnailPolicy", + "proxyLongEdge" => "proxyPolicy", + var other => other, + }); + } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { // ArchiveOfflineException included: nothing was written. diff --git a/src/Titano.Server/Api/UploadEndpoints.cs b/src/Titano.Server/Api/UploadEndpoints.cs index 56755b08..74bd66f0 100644 --- a/src/Titano.Server/Api/UploadEndpoints.cs +++ b/src/Titano.Server/Api/UploadEndpoints.cs @@ -54,7 +54,8 @@ public static class UploadEndpoints /// /// Chunks are the only large request bodies; every other endpoint keeps Kestrel's small default limit. The service - /// raises it to the chunk size of this upload (fixed when it started) before it reads the body. + /// raises it before any answer to at least the chunk size of this upload (fixed when it started) and of this server, + /// so a body it leaves unread can still be drained. /// private static async Task ChunkAsync(HttpContext context, Caller caller, Guid id, int index, UploadService service) { diff --git a/src/Titano.Server/Storage/UploadService.cs b/src/Titano.Server/Storage/UploadService.cs index 22f799bc..a468bad2 100644 --- a/src/Titano.Server/Storage/UploadService.cs +++ b/src/Titano.Server/Storage/UploadService.cs @@ -101,14 +101,17 @@ public sealed class UploadService( public async Task> ReceiveChunkAsync(Caller caller, Guid id, int index, HttpRequest request, CancellationToken cancellationToken) { var upload = await db.Uploads.GetAsync(id, cancellationToken); + // A body left unread by an answer below is drained after it: one of a chunk's size must not count as too large, or + // Kestrel would close the connection while the client still sends it and the client would never see the answer. The + // upload keeps the chunk size it started with: a smaller setting since a restart must not refuse its chunks (413). + if (request.HttpContext.Features.Get() is { IsReadOnly: false, MaxRequestBodySize: { } current } limit) + limit.MaxRequestBodySize = Math.Max(current, Math.Max(upload?.ChunkSize ?? 0, options.ChunkSizeBytes)); if (upload is null || upload.UserId != caller.Id) return Fail(404, "not_found", "upload"); if (upload.State != UploadState.Receiving) return Fail(409, "upload_state"); if (index < 0 || index >= upload.ChunkCount) return Fail(400, "chunk_out_of_range"); var expected = Math.Min(upload.ChunkSize, upload.Size - (long)index * upload.ChunkSize); if (request.ContentLength is { } announced && announced != expected) return Fail(400, "chunk_size", expected.ToString(CultureInfo.InvariantCulture)); if (upload.HasChunk(index)) return new ChunkReceived(index, UploadRules.ReceivedChunks(upload)); - // The upload keeps the chunk size it started with: a smaller setting since a restart must not refuse its chunks (413). - if (request.HttpContext.Features.Get() is { IsReadOnly: false } limit) limit.MaxRequestBodySize = upload.ChunkSize; if (_chunksInFlight.AddOrUpdate(caller.Id, 1, (_, n) => n + 1) > options.ConcurrentChunksPerUser) { diff --git a/tests/Titano.Server.Tests/Api/AdminTests.cs b/tests/Titano.Server.Tests/Api/AdminTests.cs index ea8aebd3..744d20a2 100644 --- a/tests/Titano.Server.Tests/Api/AdminTests.cs +++ b/tests/Titano.Server.Tests/Api/AdminTests.cs @@ -177,5 +177,21 @@ public sealed class AdminTests(TestContext context) { await File.WriteAllBytesAsync(path, saved, context.Cancellation); } + + // A policy this form cannot set, written out of range by another tool, is named so the admin can correct the file. + var current = Titano.Core.Archiving.ManifestStore.ReadArchive(path); + foreach (var (odd, property) in new[] + { + (current with { ThumbnailPolicy = new Titano.Core.Archiving.ThumbnailPolicy { LongEdge = 2048 } }, "thumbnailPolicy"), + (current with { ProxyPolicy = current.ProxyPolicy with { Quality = 0 } }, "proxyPolicy"), + }) + { + Titano.Core.Archiving.ManifestStore.WriteArchive(path, odd); + var before = await File.ReadAllBytesAsync(path, context.Cancellation); + using var refused = await admin.PutAsJsonAsync("/api/v1/archive/settings", changed with { ProxyLongEdge = 1280 }, ProtocolJson.Default.ArchiveSettingsDto); + Assert.Equal(property, (await refused.ExpectErrorAsync(HttpStatusCode.Conflict, "archive_policy")).Details); + Assert.SequenceEqual(before, await File.ReadAllBytesAsync(path, context.Cancellation)); + } + await File.WriteAllBytesAsync(path, saved, context.Cancellation); } } diff --git a/tests/Titano.Server.Tests/Storage/UploadTests.cs b/tests/Titano.Server.Tests/Storage/UploadTests.cs index aea2324e..b449139f 100644 --- a/tests/Titano.Server.Tests/Storage/UploadTests.cs +++ b/tests/Titano.Server.Tests/Storage/UploadTests.cs @@ -170,6 +170,29 @@ public sealed class UploadTests(TestContext context) await oversized.ExpectErrorAsync(HttpStatusCode.BadRequest, "chunk_size"); } + [Test] + public async Task AnswersThatLeaveALargeChunkUnreadReachTheClient() + { + // Chunks far above the 1 MiB other requests may send: more than the connection buffers while the server reads nothing. + const int large = 32 * 1024 * 1024; + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = large); + var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader"); + var data = Uploads.Photo(24 * 1024 * 1024, 16); + var id = (await Uploads.StartAsync(uploader, data)).UploadId!.Value; + using (var first = await Uploads.PutChunkAsync(uploader, id, 0, data)) await first.ExpectAsync(HttpStatusCode.OK); + + // The client lost the answer and sends the chunk again: the same answer, not a closed connection. + using (var again = await Uploads.PutChunkAsync(uploader, id, 0, data)) + { + await again.ExpectAsync(HttpStatusCode.OK); + Assert.SequenceEqual([0], (await again.ReadAsync(ProtocolJson.Default.ChunkReceived)).Received); + } + using (var unknown = await Uploads.PutChunkAsync(uploader, Guid.CreateVersion7(), 0, data)) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); + using (var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null)) await complete.ExpectAsync(HttpStatusCode.OK); + using var late = await Uploads.PutChunkAsync(uploader, id, 0, data); + await late.ExpectErrorAsync(HttpStatusCode.Conflict, "upload_state"); + } + [Test] public async Task RenderUploadsNeedTheEditPermission() { From ed5c0c71c8a581ec56549d92d7e2b3b1c7ec5eb1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 16:29:32 +0000 Subject: [PATCH 4/6] fix(archive): an unreadable root sidecar names no sequence waiting for its first frame The index's own predicate for sequences created without photos reads the archive root's sidecar once per call and treats a damaged or unreadable one as naming nobody. A torn root sidecar is therefore rebuilt under the root-adopted sequence instead of a later POST /sequences row, and a sidecar the server may not read fails the root folder alone instead of every scan, rebuild, orphan check and scan follow-up. Maintenance leaves these rows out of temporary-file purging and pack compaction, so the root's thumbs.pack is compacted under the leases of the root's own sequence. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/ArchiveIndexer.cs | 32 ++++- .../Archiving/ArchiveMaintenance.cs | 6 +- .../Archiving/ArchiveIngestTests.cs | 117 ++++++++++++++++++ 3 files changed, 149 insertions(+), 6 deletions(-) diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index ece6e735..f8cfe10e 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -219,21 +219,43 @@ public sealed class ArchiveIndexer } /// - /// The sequences among created without photos that still wait for their first frame - /// (). Their empty folder is not the archive root: scans do not adopt the root - /// under their id, a rebuild keeps them although they have no sidecar yet, and orphan checks do not count them missing. + /// The sequences among created without photos that still wait for their first frame: no + /// folder, no frames, and not the sequence the archive root's sidecar names. Their empty folder is not the archive + /// root: scans do not adopt or rebuild the root under their id, a rebuild keeps them although they have no sidecar yet, + /// orphan checks do not count them missing, and maintenance leaves the root's files to the root's own sequence. Unlike + /// , whose commit reports it, a root sidecar that cannot be read names nobody + /// here: a scan rebuilds a damaged one under the id of the sequence that holds the root's frames, and one it may not + /// read fails the root folder alone rather than the whole scan or rebuild. /// internal async Task> AwaitingFolderAsync(ArchivePaths paths, IEnumerable locations, CancellationToken cancellationToken) { var awaiting = new HashSet(); + (Guid? Id, bool Read) root = (null, false); foreach (var location in locations.Where(l => l.Folder.Length == 0)) { - if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is { } sequence && ArchiveIngest.AwaitsFolder(sequence, paths)) - awaiting.Add(sequence.Id); + if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is not { Folder.Length: 0, FrameCount: 0 } sequence) continue; + // Read once, and only when a row may be waiting: a root sidecar can list a hundred thousand frames. + if (!root.Read) root = (RootSidecarId(paths), true); + if (root.Id != sequence.Id) awaiting.Add(sequence.Id); } return awaiting; } + /// The id the archive root's sidecar names; null when there is none or it cannot be read. + private static Guid? RootSidecarId(ArchivePaths paths) + { + var sidecar = paths.Sequence(string.Empty).ManifestPath; + try + { + return File.Exists(sidecar) ? ManifestStore.ReadSequence(sidecar).Id : null; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + // Indexing the root reports it (a damaged sidecar is rebuilt, an unreadable one fails that folder alone). + return null; + } + } + /// The index holds a project for this sequence and other frames than the sidecar: its positions may move. private async Task MayMovePositionsAsync(SequenceManifest manifest, CancellationToken cancellationToken) => await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { SettingsJson: not null } indexed diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs index 6a75cde9..e9c921c3 100644 --- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -123,7 +123,11 @@ public sealed class ArchiveMaintenance { var paths = new ArchivePaths(archive.RootPath); if (!dryRun) ArchiveVolume.EnsureWritable(paths); - var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + // A sequence waiting for its first frame has no folder yet: the archive root's pack is compacted under the leases of + // the root's own sequence, which its thumbnail passes hold, never under another id's. + var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false); + var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList(); IReadOnlyList leases = [], temps = [], packs = []; int groups = 0, flagChanges = 0; long? reclaimable = null; diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs index c881a068..76033c7a 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -1,3 +1,4 @@ +using System.Runtime.Versioning; using System.Text.Json.Nodes; using Titano.Core.Archiving; using Titano.Core.IO; @@ -35,6 +36,36 @@ public sealed class ArchiveIngestTests(TestContext context) return created; } + /// + /// Makes unreadable until disposed: without read permission where that stops the test process + /// (UnauthorizedAccessException), otherwise (Windows, root) held open without sharing (IOException). + /// + private static IDisposable Unreadable(string path) + { + if (!OperatingSystem.IsWindows()) + { + var mode = File.GetUnixFileMode(path); + File.SetUnixFileMode(path, UnixFileMode.None); + try + { + File.OpenRead(path).Dispose(); + File.SetUnixFileMode(path, mode); + } + catch (UnauthorizedAccessException) + { + return new UnixPermissions(path, mode); + } + } + return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None); + } + + /// Gives a file its permissions back. + [UnsupportedOSPlatform("windows")] + private sealed class UnixPermissions(string path, UnixFileMode mode) : IDisposable + { + public void Dispose() => File.SetUnixFileMode(path, mode); + } + [Test] public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates() { @@ -334,6 +365,92 @@ public sealed class ArchiveIngestTests(TestContext context) Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder); } + [Test] + public async Task ADamagedRootSidecarIsRebuiltUnderTheRootSequenceNotOneWaitingForItsFirstFrame() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var root = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault()); + // Created later, so its id sorts after the root sequence's; then another tool tears the root sidecar. + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + var sidecar = fx.Paths.Sequence(string.Empty).ManifestPath; + File.WriteAllText(sidecar, "{"); + + var summary = await fx.ScanAsync(); + + Assert.Empty(summary.Failed); + Assert.Equal(root.Id, ManifestStore.ReadSequence(sidecar).Id); + Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(sidecar)!, $"{Path.GetFileName(sidecar)}.damaged-*")); + Assert.Equal(2, (await fx.Db.Sequences.GetAsync(root.Id))!.FrameCount); + var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal((string.Empty, 0, "Harbour sunset"), (row.Folder, row.FrameCount, row.Title)); + Assert.Empty(await fx.Db.Frames.ListBySequenceAsync(waiting.Id)); + } + + [Test] + public async Task ARootSidecarThatCannotBeReadFailsTheRootFolderAloneAndDropsNoSequence() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var root = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault()); + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start.AddDays(1), 2, firstSeed: 50); + var sidecar = fx.Paths.Sequence(string.Empty).ManifestPath; + + ScanSummary rebuilt; + using (Unreadable(sidecar)) rebuilt = await fx.ScanAsync(ScanMode.Rebuild); + + Assert.SequenceEqual([string.Empty], rebuilt.Failed.Select(f => f.Folder)); + Assert.Equal(0, rebuilt.Removed); + var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct); + Assert.SequenceEqual(new[] { root.Id, waiting.Id }.Order(), locations.Where(l => l.Folder.Length == 0).Select(l => l.Id).Order()); + Assert.SequenceEqual(["Trips/Comet"], locations.Where(l => l.Folder.Length > 0).Select(l => l.Folder)); + + // Readable again, the root is indexed under its own sequence. + Assert.Empty((await fx.ScanAsync()).Failed); + Assert.Equal(root.Id, ManifestStore.ReadSequence(sidecar).Id); + var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal((string.Empty, 0), (row.Folder, row.FrameCount)); + } + + [Test] + public async Task MaintenanceCompactsTheRootPackUnderTheRootSequencesLeaseOnly() + { + await using var fx = await CreateAsync(context); + // Left over from before the root was adopted, so its id sorts before the root sequence's. + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var root = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(l => l.Id != waiting.Id); + var packPath = fx.Paths.Sequence(string.Empty).ThumbsPackPath; + var hash = new byte[ThumbPack.HashLength]; + using (var pack = ThumbPackWriter.Open(packPath)) + { + // The same entry written again and again: everything but the last copy is garbage. + for (var i = 0; i < 4; i++) pack.Append(0, 48, 32, hash, Photos.Jpeg(Start, 70 + i)); + pack.Commit(); + } + var length = new FileInfo(packPath).Length; + + // A thumbnail pass of the root sequence appends under its own lease: the pack waits for it. + Task maintenance; + using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{root.Id:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) + { + maintenance = fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); + await Task.Delay(500, fx.Ct); + Assert.False(maintenance.IsCompleted); + Assert.Equal(length, new FileInfo(packPath).Length); + } + + var report = await maintenance; + Assert.SequenceEqual([string.Empty], report.CompactedPacks); + using var compacted = ThumbPackReader.Open(packPath); + Assert.Equal(0L, compacted.GarbageBytes); + Assert.NotNull(compacted.Read(0)); + } + [Test] public async Task FramesInsertedBeforeOthersTakeTheProjectPositionsAlong() { From bb7546b2ea82941bc2eba290fcec7e1279747ba7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 16:29:32 +0000 Subject: [PATCH 5/6] fix(archive): server scans queue one image pass per sequence A batched Thumbs or Proxies pass stopped at the first sequence whose folder was renamed, deleted or damaged before the pass ran, and the others kept no images until an administrator retried it. Each indexed sequence now gets its own background pass, labelled with its title and de-duplicated against a queued pass of its own, as after an upload. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../Archiving/Jobs/ScanJobHandler.cs | 32 +++---- .../Archiving/ArchiveJobTests.cs | 83 +++++++++++++++---- 2 files changed, 87 insertions(+), 28 deletions(-) diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs index 2b77e95a..fa1580d7 100644 --- a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs +++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs @@ -22,8 +22,8 @@ internal static class JobArchive /// indexed, the sequences without a current light analysis (adopted folders, new sidecars, photos added to a folder) are /// queued for it, and for a background deep analysis too when asks. On the /// server (, ) the sequences it indexed -/// without thumbnails or proxies are queued for them too, as after an upload. Sequences waiting for their first frame -/// () have nothing to analyse or draw yet and are left out. +/// without thumbnails or proxies are queued for them too, one pass per sequence as after an upload. Sequences waiting for +/// their first frame () have nothing to analyse or draw yet and are left out. /// internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind, bool imagesAfterScan = false) : IJobHandler { @@ -65,29 +65,33 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, } /// - /// Queues one background (thumbnails or proxies) of the sequences this run indexed whose + /// Queues a background (thumbnails or proxies) of each sequence this run indexed whose /// flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the - /// flag then). After a resumed run, which may have indexed sequences before the interruption, every sequence of the - /// archive without the flag but those in . Sequences already waiting in a queued pass of that - /// kind are left out. + /// flag then). After a resumed run, which may have indexed sequences before the interruption, of every sequence of the + /// archive without the flag but those in . One pass per sequence, as after an upload: a folder + /// renamed, deleted or damaged before its pass runs fails that pass alone, and the other sequences still get their + /// images. A sequence whose own pass of that kind is already queued is left out. /// private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList indexed, bool resumed, IReadOnlySet awaiting) { - var missing = new HashSet(); + var missing = new Dictionary(); var query = new SequenceQuery { ArchiveId = archive.Id, WithoutStatus = built, Sort = SequenceSort.Oldest, Limit = 1000 }; do { var page = await db.Sequences.ListAsync(query, context.Cancellation).ConfigureAwait(false); - missing.UnionWith(page.Items.Select(s => s.Id)); + foreach (var sequence in page.Items) missing[sequence.Id] = sequence; query = query with { Cursor = page.NextCursor }; } while (query.Cursor is not null); - missing.ExceptWith(awaiting); - var waiting = (await db.Jobs.ListQueuedAsync([pass], 1000, context.Cancellation).ConfigureAwait(false)).SelectMany(j => j.TargetIds).ToHashSet(); - var targets = (resumed ? missing : indexed.Where(missing.Contains)).Where(id => !waiting.Contains(id)).Distinct().ToList(); - if (targets.Count == 0) return; - var request = pass == JobKind.Thumbs ? ArchiveJobs.Thumbnails(targets, context.Job.TargetLabel) : ArchiveJobs.Proxies(targets, context.Job.TargetLabel); - await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false); + var targets = resumed ? missing.Values.ToList() : indexed.Distinct().Where(missing.ContainsKey).Select(id => missing[id]).ToList(); + foreach (var sequence in targets) + { + if (awaiting.Contains(sequence.Id) || await db.Jobs.IsQueuedAsync(pass, sequence.Id, context.Cancellation).ConfigureAwait(false)) continue; + var request = pass == JobKind.Thumbs + ? ArchiveJobs.Thumbnails([sequence.Id], sequence.DisplayTitle) + : ArchiveJobs.Proxies([sequence.Id], sequence.DisplayTitle); + await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false); + } } } diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs index f6d5e090..d8a6d4d5 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs @@ -23,6 +23,23 @@ public sealed class ArchiveJobTests(TestContext context) private static async Task> LocationsAsync(ArchiveFixture fx) => await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id); + /// The server's archive engine (ArchiveModule), not started; the desktop's scans queue no image passes. + private static JobEngine ServerEngine(ArchiveFixture fx, params IJobHandler[] others) + { + var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true }); + return new JobEngine(fx.Db, [.. service.CreateJobHandlers(), .. others], + new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + } + + private static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt); + + private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler + { + public JobKind Kind => kind; + + public Task RunAsync(JobContext context) => run(context); + } + [Test] public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint() { @@ -97,31 +114,34 @@ public sealed class ArchiveJobTests(TestContext context) { await using var fx = await CreateAsync(context); Populate(fx); - // The server's archive engine (ArchiveModule); the desktop's scans queue no image passes. - var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true }); - await using var engine = new JobEngine(fx.Db, service.CreateJobHandlers(), - new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await using var engine = ServerEngine(fx); await engine.StartAsync(fx.Ct); var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); await fx.WaitForAsync(scan.Id, JobState.Done); var locations = await LocationsAsync(fx); - var first = new Dictionary(); + var first = new HashSet(); foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) { - var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(); - Assert.SequenceEqual(locations.Select(l => l.Id), pass.TargetIds); - Assert.Equal(JobPriority.Background, pass.Priority); - await fx.WaitForAsync(pass.Id, JobState.Done); - first[kind] = pass.Id; + // One pass per sequence, named after it, as after an upload. + var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items; + Assert.SequenceEqual(locations.Select(l => l.Id).Order(), passes.SelectMany(j => j.TargetIds).Order()); + foreach (var pass in passes) + { + Assert.Count(1, pass.TargetIds); + var target = Assert.NotNull(await fx.Db.Sequences.GetAsync(pass.TargetIds[0])); + Assert.Equal((JobPriority.Background, target.DisplayTitle), (pass.Priority, pass.TargetLabel)); + await fx.WaitForAsync(pass.Id, JobState.Done); + first.Add(pass.Id); + } } foreach (var location in locations) Assert.True(Built((await fx.Db.Sequences.GetAsync(location.Id))!.Status), location.Folder); // Every folder indexed again, but none without its images: nothing is queued. var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct); await fx.WaitForAsync(full.Id, JobState.Done); - Assert.Count(1, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items); - Assert.Count(1, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items); + Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items); + Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items); // A photo added to a folder: its sequence lacks the images of that frame until the passes queued for it alone ran. Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(5), 1, firstSeed: 90, firstNumber: 7); @@ -130,7 +150,7 @@ public sealed class ArchiveJobTests(TestContext context) var a = locations.Single(l => l.Folder == "a"); foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) { - var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first[kind]); + var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id)); Assert.SequenceEqual([a.Id], pass.TargetIds); await fx.WaitForAsync(pass.Id, JobState.Done); } @@ -138,8 +158,43 @@ public sealed class ArchiveJobTests(TestContext context) using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath); Assert.Count(7, pack.Entries); Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies)); + } - static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt); + [Test] + public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages() + { + await using var fx = await CreateAsync(context); + Populate(fx); + // The only worker runs the scan, then this job ahead of the background passes the scan queued: the folder goes + // meanwhile, as when someone renames or deletes it on the NAS between a poll and its passes. + var holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var engine = ServerEngine(fx, new CallbackHandler(JobKind.Render, async c => + { + holding.SetResult(); + await release.Task.WaitAsync(c.Cancellation); + })); + var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + var hold = await engine.EnqueueAsync(JobKind.Render, [fx.Archive.Id], priority: JobPriority.Normal, cancellationToken: fx.Ct); + await engine.StartAsync(fx.Ct); + await holding.Task.WaitAsync(TimeSpan.FromSeconds(30), fx.Ct); + Assert.Equal(JobState.Done, (await fx.Db.Jobs.GetAsync(scan.Id, fx.Ct))!.State); + var locations = await LocationsAsync(fx); + var (a, b) = (locations.Single(l => l.Folder == "a"), locations.Single(l => l.Folder == "b")); + Directory.Delete(Path.Combine(fx.Paths.Root, "a"), recursive: true); + release.SetResult(); + await fx.WaitForAsync(hold.Id, JobState.Done); + + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items; + await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([a.Id])).Id, JobState.Failed); + await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([b.Id])).Id, JobState.Done); + } + Assert.True(Built((await fx.Db.Sequences.GetAsync(b.Id))!.Status), "b built"); + using var pack = ThumbPackReader.Open(fx.Paths.Sequence("b").ThumbsPackPath); + Assert.Count(6, pack.Entries); + Assert.Count(6, Directory.GetFiles(fx.Paths.Sequence("b").Proxies)); } [Test] From 3a74527f46b18c1f45e6d6c392c31e2b3186b213 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 16:29:33 +0000 Subject: [PATCH 6/6] fix(server): the chunk route raises its body limit before authentication Authentication answers 401 and 403 before the endpoint runs, with the chunk unread; over Kestrel's 1 MiB limit the connection was closed and the Uploader saw a network error instead of being asked to sign in. The chunk route now carries IRequestSizeLimitMetadata for the largest chunk size a server allows, which routing applies when it selects the endpoint. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Server/Api/UploadEndpoints.cs | 22 +++++++++++++++---- .../Storage/UploadTests.cs | 10 +++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/Titano.Server/Api/UploadEndpoints.cs b/src/Titano.Server/Api/UploadEndpoints.cs index 74bd66f0..3e62ee29 100644 --- a/src/Titano.Server/Api/UploadEndpoints.cs +++ b/src/Titano.Server/Api/UploadEndpoints.cs @@ -1,3 +1,4 @@ +using Microsoft.AspNetCore.Http.Metadata; using Titano.Core.Model; using Titano.Protocol; using Titano.Server.Security; @@ -14,11 +15,14 @@ public static class UploadEndpoints { private const int MaxCardName = 200; + /// The largest chunk size a server can be set to (). + private const long MaxChunkBytes = 64L * 1024 * 1024; + public static void Map(RouteGroupBuilder api) { var uploads = api.MapGroup("/uploads").Requires(Permissions.Upload | Permissions.RenderNode); uploads.MapPost("", StartAsync).Requires(Permissions.Upload); - uploads.MapPut("/{id:guid}/chunks/{index:int}", ChunkAsync); + uploads.MapPut("/{id:guid}/chunks/{index:int}", ChunkAsync).WithMetadata(new ChunkBodyLimit()); uploads.MapGet("/{id:guid}", async (HttpContext context, Caller caller, Guid id, UploadService service) => (await service.GetStateAsync(caller, id, context.RequestAborted)).ToResult(context, s => Results.Json(s, ProtocolJson.Default.UploadState))); uploads.MapDelete("/{id:guid}", async (HttpContext context, Caller caller, Guid id, UploadService service) => @@ -53,13 +57,23 @@ public static class UploadEndpoints } /// - /// Chunks are the only large request bodies; every other endpoint keeps Kestrel's small default limit. The service - /// raises it before any answer to at least the chunk size of this upload (fixed when it started) and of this server, - /// so a body it leaves unread can still be drained. + /// Chunks are the only large request bodies; every other endpoint keeps Kestrel's small default limit. Routing raises + /// it for this route (), and the service keeps it at least at the chunk size of this upload + /// (fixed when it started) and of this server, so a body left unread by any answer can still be drained. /// private static async Task ChunkAsync(HttpContext context, Caller caller, Guid id, int index, UploadService service) { var outcome = await service.ReceiveChunkAsync(caller, id, index, context.Request, context.RequestAborted); return outcome.ToResult(context, r => Results.Json(r, ProtocolJson.Default.ChunkReceived)); } + + /// + /// The chunk route's body limit, applied by routing as it selects the endpoint, before authentication can answer 401 + /// or 403 with the chunk unread. Kestrel drains an unread body only within the limit; above it, it closes the connection + /// and the client sees a network error instead of the answer (an Uploader would retry rather than ask to sign in). + /// + private sealed class ChunkBodyLimit : IRequestSizeLimitMetadata + { + public long? MaxRequestBodySize => MaxChunkBytes; + } } diff --git a/tests/Titano.Server.Tests/Storage/UploadTests.cs b/tests/Titano.Server.Tests/Storage/UploadTests.cs index b449139f..eccfb06a 100644 --- a/tests/Titano.Server.Tests/Storage/UploadTests.cs +++ b/tests/Titano.Server.Tests/Storage/UploadTests.cs @@ -189,8 +189,14 @@ public sealed class UploadTests(TestContext context) } using (var unknown = await Uploads.PutChunkAsync(uploader, Guid.CreateVersion7(), 0, data)) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); using (var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null)) await complete.ExpectAsync(HttpStatusCode.OK); - using var late = await Uploads.PutChunkAsync(uploader, id, 0, data); - await late.ExpectErrorAsync(HttpStatusCode.Conflict, "upload_state"); + using (var late = await Uploads.PutChunkAsync(uploader, id, 0, data)) await late.ExpectErrorAsync(HttpStatusCode.Conflict, "upload_state"); + + // Authentication answers before the endpoint runs: a user who may not upload, and a session signed out meanwhile. + var (viewer, _, _) = await server.UserAsync("marco", "viewer"); + using (var forbidden = await Uploads.PutChunkAsync(viewer, id, 0, data)) await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); + using (var logout = await uploader.PostAsync("/api/v1/auth/logout", null)) await logout.ExpectAsync(HttpStatusCode.NoContent); + using var revoked = await Uploads.PutChunkAsync(uploader, id, 0, data); + await revoked.ExpectErrorAsync(HttpStatusCode.Unauthorized, "unauthorized"); } [Test]