diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 84cf4086..3166661a 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -29,6 +29,7 @@ Another program is changing the archive's settings; try again shortly.long The archive's archive.json is damaged, from a newer Titano or of another archive; check the archive folder.long The archive is offline or read-only; try again when it is back.long + The archive's archive.json has a thumbnail or proxy setting Titano cannot use; correct it in the archive folder.long Its photos are the archive's root folder, which never moves to the trash.long Wrong user name or password.long The request is not valid.long diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index b0aa9af3..fc9c78d1 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -27,6 +27,7 @@ Un altro programma sta modificando le impostazioni dell'archivio; riprova tra poco. Il file archive.json dell'archivio è danneggiato, di un Titano più recente o di un altro archivio; controlla la cartella dell'archivio. L'archivio non è raggiungibile o è di sola lettura; riprova quando torna disponibile. + Il file archive.json dell'archivio ha un'impostazione delle miniature o dei proxy che Titano non può usare; correggila nella cartella dell'archivio. Le sue foto sono la cartella principale dell'archivio, che non va mai nel cestino. Nome utente o password errati. La richiesta non è valida. diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index a85265da..f8cfe10e 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -20,7 +20,9 @@ public enum ScanMode public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed); /// Folders that could not be indexed, with the reason; they are retried by the next scan. -public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed); +/// The sequences this run indexed ( of them), in folder order. +public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed, + IReadOnlyList Sequences); /// /// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts @@ -49,11 +51,15 @@ public sealed class ArchiveIndexer if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false); var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken); var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false); - var locations = new Locations(await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)); + var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + // A sequence waiting for its first frame has no folder yet: the archive root, adopted or re-adopted, never takes its id. + var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false); + var locations = new Locations(rows.Where(r => !awaiting.Contains(r.Id))); var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null; using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null; - int indexed = 0, adopted = 0; + int adopted = 0; + var indexed = new List(); var failed = new List<(string, string)>(); for (var i = 0; i < folders.Count; i++) { @@ -67,14 +73,14 @@ public sealed class ArchiveIndexer try { var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos); - var bytes = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false); + var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false); if (snapshotWriter is not null) { var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath; snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value, bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false)); } - indexed++; + indexed.Add(id); if (folder.Sidecar is null) adopted++; } catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) @@ -88,7 +94,7 @@ public sealed class ArchiveIndexer var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0; if (failed.Count == 0) snapshotWriter?.Commit(); - return new ScanSummary(folders.Count, indexed, adopted, removed, failed); + return new ScanSummary(folders.Count, indexed.Count, adopted, removed, failed, indexed); } /// @@ -156,8 +162,8 @@ public sealed class ArchiveIndexer } } - /// Indexes one folder; returns the sidecar bytes when they were read and not rewritten since (for the snapshot). - private async Task IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot, + /// Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot). + private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot, Locations locations, CancellationToken cancellationToken) { var sequence = paths.Sequence(folder.RelativeFolder); @@ -189,10 +195,11 @@ public sealed class ArchiveIndexer } if (reconcile || File.Exists(sequence.IngestJournalPath)) { - // Photos added, gone or back: the light analysis no longer describes the frames, so the scan's follow-up runs it again. + // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, so the scan's + // follow-ups make them again (the images on a server, ArchiveServiceOptions.ImagesAfterScan). manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled - ? reconciled with { Status = reconciled.Status & ~SequenceStatus.AnalyzedLight } + ? reconciled with { Status = reconciled.Status & ~(SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt) } : null, cancellationToken).ConfigureAwait(false); bytes = null; @@ -208,7 +215,45 @@ public sealed class ArchiveIndexer await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); } locations.Set(manifest.Id, folder.RelativeFolder); - return bytes; + return (manifest.Id, bytes); + } + + /// + /// The sequences among created without photos that still wait for their first frame: no + /// folder, no frames, and not the sequence the archive root's sidecar names. Their empty folder is not the archive + /// root: scans do not adopt or rebuild the root under their id, a rebuild keeps them although they have no sidecar yet, + /// orphan checks do not count them missing, and maintenance leaves the root's files to the root's own sequence. Unlike + /// , whose commit reports it, a root sidecar that cannot be read names nobody + /// here: a scan rebuilds a damaged one under the id of the sequence that holds the root's frames, and one it may not + /// read fails the root folder alone rather than the whole scan or rebuild. + /// + internal async Task> AwaitingFolderAsync(ArchivePaths paths, IEnumerable locations, CancellationToken cancellationToken) + { + var awaiting = new HashSet(); + (Guid? Id, bool Read) root = (null, false); + foreach (var location in locations.Where(l => l.Folder.Length == 0)) + { + if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is not { Folder.Length: 0, FrameCount: 0 } sequence) continue; + // Read once, and only when a row may be waiting: a root sidecar can list a hundred thousand frames. + if (!root.Read) root = (RootSidecarId(paths), true); + if (root.Id != sequence.Id) awaiting.Add(sequence.Id); + } + return awaiting; + } + + /// The id the archive root's sidecar names; null when there is none or it cannot be read. + private static Guid? RootSidecarId(ArchivePaths paths) + { + var sidecar = paths.Sequence(string.Empty).ManifestPath; + try + { + return File.Exists(sidecar) ? ManifestStore.ReadSequence(sidecar).Id : null; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + // Indexing the root reports it (a damaged sidecar is rebuilt, an unreadable one fails that folder alone). + return null; + } } /// The index holds a project for this sequence and other frames than the sidecar: its positions may move. @@ -261,16 +306,21 @@ public sealed class ArchiveIndexer UpdatedAt = DateTimeOffset.UtcNow, }; - /// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). + /// + /// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence + /// waiting for its first frame has no sidecar yet and stays. + /// private async Task RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList folders, Locations locations, CancellationToken cancellationToken) { ArchiveVolume.EnsureWritable(paths); var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal); + var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false); var removed = 0; - foreach (var location in await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)) + foreach (var location in rows) { - if (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id) continue; + if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue; if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++; await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false); } diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs index c3cc7dc7..e9c921c3 100644 --- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -71,7 +71,10 @@ public sealed class ArchiveMaintenance { var paths = new ArchivePaths(archive.RootPath); if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline); - var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + // A sequence waiting for its first frame has no folder yet: none of it is missing, and the archive root is not its folder. + var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false); + var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList(); var sequences = new List(); var missing = new List(); foreach (var location in locations) @@ -91,7 +94,10 @@ public sealed class ArchiveMaintenance return new OrphanReport(sequences, missing, unindexed); } - /// Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept. + /// + /// Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept, and + /// so are sequences waiting for their first frame (), which have none yet. + /// public async Task RemoveMissingSequencesAsync(Archive archive, IReadOnlyList sequenceIds, CancellationToken cancellationToken = default) { var paths = new ArchivePaths(archive.RootPath); @@ -100,7 +106,7 @@ public sealed class ArchiveMaintenance foreach (var id in sequenceIds) { if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue; - if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath)) continue; + if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath) || ArchiveIngest.AwaitsFolder(sequence, paths)) continue; if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++; await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false); } @@ -117,7 +123,11 @@ public sealed class ArchiveMaintenance { var paths = new ArchivePaths(archive.RootPath); if (!dryRun) ArchiveVolume.EnsureWritable(paths); - var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + // A sequence waiting for its first frame has no folder yet: the archive root's pack is compacted under the leases of + // the root's own sequence, which its thumbnail passes hold, never under another id's. + var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false); + var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList(); IReadOnlyList leases = [], temps = [], packs = []; int groups = 0, flagChanges = 0; long? reclaimable = null; diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs index 562d9e42..0d43f48e 100644 --- a/src/Titano.Data/Archiving/ArchiveService.cs +++ b/src/Titano.Data/Archiving/ArchiveService.cs @@ -54,8 +54,8 @@ public sealed class ArchiveService : IJobHandlerProvider /// Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the . public IReadOnlyList CreateJobHandlers() => [ - new ScanJobHandler(_db, Indexer, JobKind.Scan), - new ScanJobHandler(_db, Indexer, JobKind.Rebuild), + new ScanJobHandler(_db, Indexer, JobKind.Scan, _writer.Options.ImagesAfterScan), + new ScanJobHandler(_db, Indexer, JobKind.Rebuild, _writer.Options.ImagesAfterScan), new VerifyJobHandler(_db, Verifier), new ImportJobHandler(_db, Imports), new ImageJobHandler(_db, Images, JobKind.Thumbs), diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs index db6bcd8c..3cdf58a7 100644 --- a/src/Titano.Data/Archiving/ArchiveVerifier.cs +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -44,7 +44,7 @@ public sealed class ArchiveVerifier { var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); - var targets = await TargetsAsync(archive, sequenceIds, cancellationToken).ConfigureAwait(false); + var targets = await TargetsAsync(archive, paths, sequenceIds, cancellationToken).ConfigureAwait(false); var bytesTotal = targets.Sum(t => t.TotalBytes); var bytesDone = targets.Take(resume.Sequence).Sum(t => t.TotalBytes); long checkedFrames = 0, ok = 0, hashed = 0; @@ -82,13 +82,15 @@ public sealed class ArchiveVerifier return new VerifyReport(targets.Count, checkedFrames, ok, hashed, problems); } - private async Task> TargetsAsync(Archive archive, IReadOnlyList? sequenceIds, CancellationToken cancellationToken) + /// The sequences to check; one waiting for its first frame () has no sidecar to check yet. + private async Task> TargetsAsync(Archive archive, ArchivePaths paths, IReadOnlyList? sequenceIds, CancellationToken cancellationToken) { var ids = sequenceIds ?? (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id).ToList(); var targets = new List(ids.Count); foreach (var id in ids) { - if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id) targets.Add(s); + if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id && !ArchiveIngest.AwaitsFolder(s, paths)) + targets.Add(s); } return targets; } diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs index e0812277..fa1580d7 100644 --- a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs +++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs @@ -1,7 +1,9 @@ +using Titano.Core.Archiving; using Titano.Core.Localization; using Titano.Core.Model; using Titano.Data.Jobs; using Titano.Data.Processing; +using Titano.Data.Repositories; namespace Titano.Data.Archiving.Jobs; @@ -18,9 +20,12 @@ internal static class JobArchive /// (incremental, or full with ) and /// . Checkpoint: after:<folder>, the last folder indexed. When anything was /// indexed, the sequences without a current light analysis (adopted folders, new sidecars, photos added to a folder) are -/// queued for it, and for a background deep analysis too when asks. +/// queued for it, and for a background deep analysis too when asks. On the +/// server (, ) the sequences it indexed +/// without thumbnails or proxies are queued for them too, one pass per sequence as after an upload. Sequences waiting for +/// their first frame () have nothing to analyse or draw yet and are left out. /// -internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler +internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind, bool imagesAfterScan = false) : IJobHandler { private const string Prefix = "after:"; @@ -42,9 +47,51 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, // A resumed scan may have indexed its folders before the interruption. if (summary.Indexed > 0 || resume is not null) { - var unanalysed = await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false); + var awaiting = await indexer.AwaitingFolderAsync(new ArchivePaths(archive.RootPath), + await db.Sequences.ListLocationsAsync(archive.Id, context.Cancellation).ConfigureAwait(false), context.Cancellation).ConfigureAwait(false); + var unanalysed = (await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false)) + .Where(id => !awaiting.Contains(id)).ToList(); await AnalysisFollowUp.QueueAsync(db, context, unanalysed).ConfigureAwait(false); if (kind == JobKind.Scan && parameters.DeepAnalysis) await AnalysisFollowUp.QueueDeepAsync(db, context, unanalysed).ConfigureAwait(false); + if (imagesAfterScan) + { + var resumed = resume is not null; + await QueueImagesAsync(context, archive, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, summary.Sequences, resumed, awaiting).ConfigureAwait(false); + // An archive whose policy makes no proxies would get a pass that does nothing. + if (archive.ProxyLongEdge != ProxyPolicy.Disabled) + await QueueImagesAsync(context, archive, JobKind.Proxies, SequenceStatus.ProxiesBuilt, summary.Sequences, resumed, awaiting).ConfigureAwait(false); + } + } + } + + /// + /// Queues a background (thumbnails or proxies) of each sequence this run indexed whose + /// flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the + /// flag then). After a resumed run, which may have indexed sequences before the interruption, of every sequence of the + /// archive without the flag but those in . One pass per sequence, as after an upload: a folder + /// renamed, deleted or damaged before its pass runs fails that pass alone, and the other sequences still get their + /// images. A sequence whose own pass of that kind is already queued is left out. + /// + private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList indexed, + bool resumed, IReadOnlySet awaiting) + { + var missing = new Dictionary(); + var query = new SequenceQuery { ArchiveId = archive.Id, WithoutStatus = built, Sort = SequenceSort.Oldest, Limit = 1000 }; + do + { + var page = await db.Sequences.ListAsync(query, context.Cancellation).ConfigureAwait(false); + foreach (var sequence in page.Items) missing[sequence.Id] = sequence; + query = query with { Cursor = page.NextCursor }; + } + while (query.Cursor is not null); + var targets = resumed ? missing.Values.ToList() : indexed.Distinct().Where(missing.ContainsKey).Select(id => missing[id]).ToList(); + foreach (var sequence in targets) + { + if (awaiting.Contains(sequence.Id) || await db.Jobs.IsQueuedAsync(pass, sequence.Id, context.Cancellation).ConfigureAwait(false)) continue; + var request = pass == JobKind.Thumbs + ? ArchiveJobs.Thumbnails([sequence.Id], sequence.DisplayTitle) + : ArchiveJobs.Proxies([sequence.Id], sequence.DisplayTitle); + await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false); } } } diff --git a/src/Titano.Data/Archiving/MassRenamer.cs b/src/Titano.Data/Archiving/MassRenamer.cs index 6846e70d..e6b1c11e 100644 --- a/src/Titano.Data/Archiving/MassRenamer.cs +++ b/src/Titano.Data/Archiving/MassRenamer.cs @@ -2,10 +2,14 @@ using Titano.Core.Archiving; using Titano.Core.Diagnostics; using Titano.Core.Localization; using Titano.Core.Model; +using Titano.Data.Repositories; namespace Titano.Data.Archiving; -/// The plan no longer matches the archive (a sequence moved or vanished since it was made). +/// +/// The plan no longer matches the archive: a sequence moved or vanished since it was made, or one was indexed inside a +/// folder the plan moves. +/// public sealed class StaleRenamePlanException(Guid sequenceId) : InvalidOperationException(Loc.L("Archive.Error.StalePlan")) { @@ -62,7 +66,8 @@ public sealed class MassRenamer private static RenameSubject Subject(Sequence s) => new(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens); /// Applies a plan; each completed step is journaled before the next starts. - /// A sequence is no longer where the plan expects it; nothing of that step was changed. + /// A sequence is no longer where the plan expects it, or a folder that moves holds + /// a sequence the plan does not take along; nothing of that step was changed. public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress? progress = null, CancellationToken cancellationToken = default) => RunAsync(archive, new RenameJournal(plan, []), resuming: false, progress, cancellationToken); @@ -85,12 +90,17 @@ public sealed class MassRenamer var completed = journal.Completed.ToHashSet(); if (!resuming) { - // A plan made before another change is refused before anything moves. + // A plan made before another change is refused before anything moves: a sequence moved, or one was indexed + // inside a folder that moves (a scan adopted a subfolder copied in since the dry run). foreach (var step in journal.Plan.Steps) { if ((await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false))?.Folder != step.FromFolder) throw new StaleRenamePlanException(step.SequenceId); } + var planned = journal.Plan.Steps.Select(s => s.SequenceId).ToHashSet(); + if (await StrangerInsideAsync(archive, journal.Plan.Steps.Where(s => s.MovesFolder && !s.Carried), planned, cancellationToken) + .ConfigureAwait(false) is { } stranger) + throw new StaleRenamePlanException(stranger.Id); } RenameJournal.Write(paths, journal); var steps = journal.Plan.Steps; @@ -147,6 +157,7 @@ public sealed class MassRenamer { if (await StandsAsync(c, resuming, cancellationToken).ConfigureAwait(false)) follow.Add(c); } + if (!await CarriesAllInsideAsync(archive, paths, step, carried, resuming, cancellationToken).ConfigureAwait(false)) return; RenamePlanner.MoveFolder(paths, step); await RecordAsync(archive, paths, step, cancellationToken).ConfigureAwait(false); // Moved with their carrier: only their sidecars and index rows follow. @@ -170,6 +181,35 @@ public sealed class MassRenamer return false; } + /// + /// Whether every sequence the index has inside the folder of a moving step moves with it as planned, checked again under + /// the step's leases just before the folder moves. One indexed there since the dry run would move without its index + /// row following, and was never shown in the plan: the step is refused, or dropped when resuming a folder that has not + /// moved yet (a folder moved before the interruption goes on; the next scan indexes the newcomer where it ended up). + /// + /// Not resuming, and such a sequence is inside the folder. + private async Task CarriesAllInsideAsync(Archive archive, ArchivePaths paths, RenameStep step, IReadOnlyList carried, bool resuming, + CancellationToken cancellationToken) + { + if (!step.MovesFolder) return true; + var taken = carried.Select(c => c.SequenceId).ToHashSet(); + if (await StrangerInsideAsync(archive, [step], taken, cancellationToken).ConfigureAwait(false) is not { } stranger) return true; + if (!resuming) throw new StaleRenamePlanException(stranger.Id); + if (!Directory.Exists(paths.Sequence(step.FromFolder).Folder)) return true; + Log.Warning("archive", $"Rename of {step.FromFolder} dropped: {stranger.Folder} was indexed inside it after the plan was made"); + return false; + } + + /// A sequence the index has inside the folder of one of that is not in ; null when there is none. + private async Task StrangerInsideAsync(Archive archive, IEnumerable moving, IReadOnlySet taken, + CancellationToken cancellationToken) + { + var folders = moving.Select(s => s.FromFolder).ToList(); + if (folders.Count == 0) return null; + return (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)) + .FirstOrDefault(l => !taken.Contains(l.Id) && folders.Any(f => RenamePlanner.IsInside(f, l.Folder))); + } + /// Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it. private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken) { diff --git a/src/Titano.Data/Archiving/SequenceWriter.cs b/src/Titano.Data/Archiving/SequenceWriter.cs index fe636cf4..d91283ef 100644 --- a/src/Titano.Data/Archiving/SequenceWriter.cs +++ b/src/Titano.Data/Archiving/SequenceWriter.cs @@ -17,6 +17,13 @@ public sealed record ArchiveServiceOptions public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2); public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration; + + /// + /// Scans queue thumbnails and proxies of the sequences they index without them, as an upload does (the server, whose + /// clients load both from it: BRIEF §4.3, §14.4). The desktop decodes its thumbnails on demand and makes proxies when + /// asked. + /// + public bool ImagesAfterScan { get; init; } } /// diff --git a/src/Titano.Server/Api/ArchiveEndpoints.cs b/src/Titano.Server/Api/ArchiveEndpoints.cs index 953df725..c71e1df6 100644 --- a/src/Titano.Server/Api/ArchiveEndpoints.cs +++ b/src/Titano.Server/Api/ArchiveEndpoints.cs @@ -67,7 +67,9 @@ public static class ArchiveEndpoints /// Writes the name, templates and proxy policy into the archive's archive.json through the archive engine (so a /// restart and the proxy jobs see them), then into the index. 503 archive_offline while the archive cannot be /// written; 409 archive_busy while another process holds its settings; 409 archive_manifest when its - /// archive.json is damaged, written by a newer Titano or of another archive (nothing is written). + /// archive.json is damaged, written by a newer Titano or of another archive; 409 archive_policy when it holds a + /// thumbnail or proxy setting this API cannot set and Titano cannot use (details: thumbnailPolicy or + /// proxyPolicy, the property to correct in the file). Nothing is written. /// private static async Task UpdateAsync(HttpContext context, Caller caller, ArchiveSettingsDto request, TitanoDatabase db, ServerArchive archive, [FromServices] ArchiveService? archives, AuditLog audit) @@ -99,6 +101,17 @@ public static class ArchiveEndpoints { return ApiErrors.Conflict(context, "archive_manifest"); } + catch (ArgumentException ex) + { + // The request's fields were checked above: what fails is a value read from archive.json (a thumbnail size, or a + // proxy quality, written by hand or by another tool), which the admin corrects in the file, not in this form. + return ApiErrors.Conflict(context, "archive_policy", ex.ParamName switch + { + "thumbnailLongEdge" => "thumbnailPolicy", + "proxyLongEdge" => "proxyPolicy", + var other => other, + }); + } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { // ArchiveOfflineException included: nothing was written. diff --git a/src/Titano.Server/Api/UploadEndpoints.cs b/src/Titano.Server/Api/UploadEndpoints.cs index 56755b08..3e62ee29 100644 --- a/src/Titano.Server/Api/UploadEndpoints.cs +++ b/src/Titano.Server/Api/UploadEndpoints.cs @@ -1,3 +1,4 @@ +using Microsoft.AspNetCore.Http.Metadata; using Titano.Core.Model; using Titano.Protocol; using Titano.Server.Security; @@ -14,11 +15,14 @@ public static class UploadEndpoints { private const int MaxCardName = 200; + /// The largest chunk size a server can be set to (). + private const long MaxChunkBytes = 64L * 1024 * 1024; + public static void Map(RouteGroupBuilder api) { var uploads = api.MapGroup("/uploads").Requires(Permissions.Upload | Permissions.RenderNode); uploads.MapPost("", StartAsync).Requires(Permissions.Upload); - uploads.MapPut("/{id:guid}/chunks/{index:int}", ChunkAsync); + uploads.MapPut("/{id:guid}/chunks/{index:int}", ChunkAsync).WithMetadata(new ChunkBodyLimit()); uploads.MapGet("/{id:guid}", async (HttpContext context, Caller caller, Guid id, UploadService service) => (await service.GetStateAsync(caller, id, context.RequestAborted)).ToResult(context, s => Results.Json(s, ProtocolJson.Default.UploadState))); uploads.MapDelete("/{id:guid}", async (HttpContext context, Caller caller, Guid id, UploadService service) => @@ -53,12 +57,23 @@ public static class UploadEndpoints } /// - /// Chunks are the only large request bodies; every other endpoint keeps Kestrel's small default limit. The service - /// raises it to the chunk size of this upload (fixed when it started) before it reads the body. + /// Chunks are the only large request bodies; every other endpoint keeps Kestrel's small default limit. Routing raises + /// it for this route (), and the service keeps it at least at the chunk size of this upload + /// (fixed when it started) and of this server, so a body left unread by any answer can still be drained. /// private static async Task ChunkAsync(HttpContext context, Caller caller, Guid id, int index, UploadService service) { var outcome = await service.ReceiveChunkAsync(caller, id, index, context.Request, context.RequestAborted); return outcome.ToResult(context, r => Results.Json(r, ProtocolJson.Default.ChunkReceived)); } + + /// + /// The chunk route's body limit, applied by routing as it selects the endpoint, before authentication can answer 401 + /// or 403 with the chunk unread. Kestrel drains an unread body only within the limit; above it, it closes the connection + /// and the client sees a network error instead of the answer (an Uploader would retry rather than ask to sign in). + /// + private sealed class ChunkBodyLimit : IRequestSizeLimitMetadata + { + public long? MaxRequestBodySize => MaxChunkBytes; + } } diff --git a/src/Titano.Server/Storage/ArchiveModule.cs b/src/Titano.Server/Storage/ArchiveModule.cs index ad5bebc2..ee318bf0 100644 --- a/src/Titano.Server/Storage/ArchiveModule.cs +++ b/src/Titano.Server/Storage/ArchiveModule.cs @@ -8,8 +8,9 @@ namespace Titano.Server.Storage; /// /// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and /// thumbnails, its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance) and the periodic -/// incremental scan (). Decoding is managed only (BRIEF §4.2). The daily backup of an -/// SQLite index runs with it (). +/// incremental scan (); a scan queues thumbnails and proxies of what it indexes, as an upload +/// does. Decoding is managed only (BRIEF §4.2). The daily backup of an SQLite index runs with it +/// (). /// public static class ArchiveModule { @@ -19,7 +20,7 @@ public static class ArchiveModule public static void Register(IServiceCollection services) { services.AddSingleton(sp => new ArchiveService(sp.GetRequiredService(), - new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = Owner })); + new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = Owner, ImagesAfterScan = true })); services.AddSingleton(sp => sp.GetRequiredService().Ingest); services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); diff --git a/src/Titano.Server/Storage/UploadService.cs b/src/Titano.Server/Storage/UploadService.cs index 22f799bc..a468bad2 100644 --- a/src/Titano.Server/Storage/UploadService.cs +++ b/src/Titano.Server/Storage/UploadService.cs @@ -101,14 +101,17 @@ public sealed class UploadService( public async Task> ReceiveChunkAsync(Caller caller, Guid id, int index, HttpRequest request, CancellationToken cancellationToken) { var upload = await db.Uploads.GetAsync(id, cancellationToken); + // A body left unread by an answer below is drained after it: one of a chunk's size must not count as too large, or + // Kestrel would close the connection while the client still sends it and the client would never see the answer. The + // upload keeps the chunk size it started with: a smaller setting since a restart must not refuse its chunks (413). + if (request.HttpContext.Features.Get() is { IsReadOnly: false, MaxRequestBodySize: { } current } limit) + limit.MaxRequestBodySize = Math.Max(current, Math.Max(upload?.ChunkSize ?? 0, options.ChunkSizeBytes)); if (upload is null || upload.UserId != caller.Id) return Fail(404, "not_found", "upload"); if (upload.State != UploadState.Receiving) return Fail(409, "upload_state"); if (index < 0 || index >= upload.ChunkCount) return Fail(400, "chunk_out_of_range"); var expected = Math.Min(upload.ChunkSize, upload.Size - (long)index * upload.ChunkSize); if (request.ContentLength is { } announced && announced != expected) return Fail(400, "chunk_size", expected.ToString(CultureInfo.InvariantCulture)); if (upload.HasChunk(index)) return new ChunkReceived(index, UploadRules.ReceivedChunks(upload)); - // The upload keeps the chunk size it started with: a smaller setting since a restart must not refuse its chunks (413). - if (request.HttpContext.Features.Get() is { IsReadOnly: false } limit) limit.MaxRequestBodySize = upload.ChunkSize; if (_chunksInFlight.AddOrUpdate(caller.Id, 1, (_, n) => n + 1) > options.ConcurrentChunksPerUser) { diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs index 548f799c..76033c7a 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -1,3 +1,4 @@ +using System.Runtime.Versioning; using System.Text.Json.Nodes; using Titano.Core.Archiving; using Titano.Core.IO; @@ -17,6 +18,54 @@ public sealed class ArchiveIngestTests(TestContext context) Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") => new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null); + /// A sequence as POST /api/v1/sequences creates it: a row without a folder or frames. + private static async Task CreateWithoutPhotosAsync(ArchiveFixture fx, string title) + { + var created = new Sequence + { + Id = Guid.CreateVersion7(), + ArchiveId = fx.Archive.Id, + Title = title, + Folder = string.Empty, + Rating = 4, + Origin = SequenceOrigin.Server, + OwnerUserId = Guid.CreateVersion7(), + UpdatedAt = DateTimeOffset.UtcNow, + }; + await fx.Db.Sequences.UpsertAsync(created, fx.Ct); + return created; + } + + /// + /// Makes unreadable until disposed: without read permission where that stops the test process + /// (UnauthorizedAccessException), otherwise (Windows, root) held open without sharing (IOException). + /// + private static IDisposable Unreadable(string path) + { + if (!OperatingSystem.IsWindows()) + { + var mode = File.GetUnixFileMode(path); + File.SetUnixFileMode(path, UnixFileMode.None); + try + { + File.OpenRead(path).Dispose(); + File.SetUnixFileMode(path, mode); + } + catch (UnauthorizedAccessException) + { + return new UnixPermissions(path, mode); + } + } + return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None); + } + + /// Gives a file its permissions back. + [UnsupportedOSPlatform("windows")] + private sealed class UnixPermissions(string path, UnixFileMode mode) : IDisposable + { + public void Dispose() => File.SetUnixFileMode(path, mode); + } + [Test] public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates() { @@ -242,6 +291,166 @@ public sealed class ArchiveIngestTests(TestContext context) Assert.FileExists(staged.Path); } + [Test] + public async Task ASequenceWaitingForItsFirstFrameNeverLendsItsIdToTheArchiveRoot() + { + await using var fx = await CreateAsync(context); + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + // Loose photos at the root, which a scan adopts in place. + Photos.WriteSeries(fx.Archive.RootPath, Start.AddDays(-1), 2); + + await fx.ScanAsync(); + + var root = ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath); + Assert.NotEqual(waiting.Id, root.Id); + var rootRow = (await fx.Db.Sequences.GetAsync(root.Id))!; + Assert.Equal((string.Empty, 2), (rootRow.Folder, rootRow.FrameCount)); + var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal((string.Empty, 0, "Harbour sunset"), (row.Folder, row.FrameCount, row.Title)); + + // Its first frame still gets a folder of its own, and the root keeps its photos. + var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), sequenceId: waiting.Id), fx.Ct); + Assert.Equal(waiting.Id, first.SequenceId); + var placed = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal(("2026/2026-09-28 Canon EOS R5", 1), (placed.Folder, placed.FrameCount)); + Assert.Equal(root.Id, ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath).Id); + Assert.Equal(2, (await fx.Db.Sequences.GetAsync(root.Id))!.FrameCount); + } + + [Test] + public async Task ARootSequenceWhoseSidecarIsGoneIsAdoptedAgainUnderItsOwnId() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var adopted = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault()); + // Created later, so its id sorts after the root sequence's. + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + File.Delete(fx.Paths.Sequence(string.Empty).ManifestPath); + + await fx.ScanAsync(); + + Assert.Equal(adopted.Id, ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath).Id); + Assert.Equal(2, (await fx.Db.Sequences.GetAsync(adopted.Id))!.FrameCount); + var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal((string.Empty, 0), (row.Folder, row.FrameCount)); + } + + [Test] + public async Task RebuildOrphansAndVerifyLeaveASequenceWaitingForItsFirstFrame() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2); + await fx.ScanAsync(); + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + + // It has no sidecar yet, which is no reason to drop it. + var rebuilt = await fx.ScanAsync(ScanMode.Rebuild); + Assert.Equal(0, rebuilt.Removed); + var orphans = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct); + Assert.Empty(orphans.MissingSequences); + Assert.Empty(orphans.UnindexedFolders); + Assert.Equal(0, await fx.Service.RemoveMissingSequencesAsync(fx.Archive.Id, [waiting.Id], fx.Ct)); + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + Assert.Equal((1, 2L), (report.Sequences, report.Checked)); + var kept = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id)); + Assert.Equal(("Harbour sunset", 4), (kept.Title, kept.Rating)); + + // The archive root is not its folder: loose photos there are a folder the index does not know yet. + Photos.WriteSeries(fx.Archive.RootPath, Start.AddDays(-1), 1, firstSeed: 50); + Assert.SequenceEqual([string.Empty], (await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).UnindexedFolders); + + var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct); + Assert.Equal(waiting.Id, first.SequenceId); + Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder); + } + + [Test] + public async Task ADamagedRootSidecarIsRebuiltUnderTheRootSequenceNotOneWaitingForItsFirstFrame() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var root = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault()); + // Created later, so its id sorts after the root sequence's; then another tool tears the root sidecar. + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + var sidecar = fx.Paths.Sequence(string.Empty).ManifestPath; + File.WriteAllText(sidecar, "{"); + + var summary = await fx.ScanAsync(); + + Assert.Empty(summary.Failed); + Assert.Equal(root.Id, ManifestStore.ReadSequence(sidecar).Id); + Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(sidecar)!, $"{Path.GetFileName(sidecar)}.damaged-*")); + Assert.Equal(2, (await fx.Db.Sequences.GetAsync(root.Id))!.FrameCount); + var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal((string.Empty, 0, "Harbour sunset"), (row.Folder, row.FrameCount, row.Title)); + Assert.Empty(await fx.Db.Frames.ListBySequenceAsync(waiting.Id)); + } + + [Test] + public async Task ARootSidecarThatCannotBeReadFailsTheRootFolderAloneAndDropsNoSequence() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var root = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault()); + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start.AddDays(1), 2, firstSeed: 50); + var sidecar = fx.Paths.Sequence(string.Empty).ManifestPath; + + ScanSummary rebuilt; + using (Unreadable(sidecar)) rebuilt = await fx.ScanAsync(ScanMode.Rebuild); + + Assert.SequenceEqual([string.Empty], rebuilt.Failed.Select(f => f.Folder)); + Assert.Equal(0, rebuilt.Removed); + var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct); + Assert.SequenceEqual(new[] { root.Id, waiting.Id }.Order(), locations.Where(l => l.Folder.Length == 0).Select(l => l.Id).Order()); + Assert.SequenceEqual(["Trips/Comet"], locations.Where(l => l.Folder.Length > 0).Select(l => l.Folder)); + + // Readable again, the root is indexed under its own sequence. + Assert.Empty((await fx.ScanAsync()).Failed); + Assert.Equal(root.Id, ManifestStore.ReadSequence(sidecar).Id); + var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal((string.Empty, 0), (row.Folder, row.FrameCount)); + } + + [Test] + public async Task MaintenanceCompactsTheRootPackUnderTheRootSequencesLeaseOnly() + { + await using var fx = await CreateAsync(context); + // Left over from before the root was adopted, so its id sorts before the root sequence's. + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var root = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(l => l.Id != waiting.Id); + var packPath = fx.Paths.Sequence(string.Empty).ThumbsPackPath; + var hash = new byte[ThumbPack.HashLength]; + using (var pack = ThumbPackWriter.Open(packPath)) + { + // The same entry written again and again: everything but the last copy is garbage. + for (var i = 0; i < 4; i++) pack.Append(0, 48, 32, hash, Photos.Jpeg(Start, 70 + i)); + pack.Commit(); + } + var length = new FileInfo(packPath).Length; + + // A thumbnail pass of the root sequence appends under its own lease: the pack waits for it. + Task maintenance; + using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{root.Id:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) + { + maintenance = fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); + await Task.Delay(500, fx.Ct); + Assert.False(maintenance.IsCompleted); + Assert.Equal(length, new FileInfo(packPath).Length); + } + + var report = await maintenance; + Assert.SequenceEqual([string.Empty], report.CompactedPacks); + using var compacted = ThumbPackReader.Open(packPath); + Assert.Equal(0L, compacted.GarbageBytes); + Assert.NotNull(compacted.Read(0)); + } + [Test] public async Task FramesInsertedBeforeOthersTakeTheProjectPositionsAlong() { diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs index 1a50f5d3..d8a6d4d5 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs @@ -3,6 +3,7 @@ using Titano.Core.Import; using Titano.Core.Model; using Titano.Data.Archiving; using Titano.Data.Archiving.Jobs; +using Titano.Data.Jobs; using Titano.Data.Repositories; using Titano.Testing; using static Titano.Data.Tests.Archiving.ArchiveFixture; @@ -22,6 +23,23 @@ public sealed class ArchiveJobTests(TestContext context) private static async Task> LocationsAsync(ArchiveFixture fx) => await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id); + /// The server's archive engine (ArchiveModule), not started; the desktop's scans queue no image passes. + private static JobEngine ServerEngine(ArchiveFixture fx, params IJobHandler[] others) + { + var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true }); + return new JobEngine(fx.Db, [.. service.CreateJobHandlers(), .. others], + new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + } + + private static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt); + + private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler + { + public JobKind Kind => kind; + + public Task RunAsync(JobContext context) => run(context); + } + [Test] public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint() { @@ -40,6 +58,9 @@ public sealed class ArchiveJobTests(TestContext context) var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct); await fx.WaitForAsync(rebuild.Id, JobState.Done); Assert.Equal(2, (await LocationsAsync(fx)).Count); + // The desktop decodes its thumbnails on demand and makes proxies when asked. + Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items); + Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items); } [Test] @@ -88,6 +109,94 @@ public sealed class ArchiveJobTests(TestContext context) Assert.Contains("made 3, reused 3", done.LogTail); } + [Test] + public async Task AServerScanQueuesThumbnailsAndProxiesOfTheSequencesItIndexesWithoutThem() + { + await using var fx = await CreateAsync(context); + Populate(fx); + await using var engine = ServerEngine(fx); + await engine.StartAsync(fx.Ct); + + var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(scan.Id, JobState.Done); + var locations = await LocationsAsync(fx); + var first = new HashSet(); + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + // One pass per sequence, named after it, as after an upload. + var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items; + Assert.SequenceEqual(locations.Select(l => l.Id).Order(), passes.SelectMany(j => j.TargetIds).Order()); + foreach (var pass in passes) + { + Assert.Count(1, pass.TargetIds); + var target = Assert.NotNull(await fx.Db.Sequences.GetAsync(pass.TargetIds[0])); + Assert.Equal((JobPriority.Background, target.DisplayTitle), (pass.Priority, pass.TargetLabel)); + await fx.WaitForAsync(pass.Id, JobState.Done); + first.Add(pass.Id); + } + } + foreach (var location in locations) Assert.True(Built((await fx.Db.Sequences.GetAsync(location.Id))!.Status), location.Folder); + + // Every folder indexed again, but none without its images: nothing is queued. + var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct); + await fx.WaitForAsync(full.Id, JobState.Done); + Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items); + Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items); + + // A photo added to a folder: its sequence lacks the images of that frame until the passes queued for it alone ran. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(5), 1, firstSeed: 90, firstNumber: 7); + var added = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(added.Id, JobState.Done); + var a = locations.Single(l => l.Folder == "a"); + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id)); + Assert.SequenceEqual([a.Id], pass.TargetIds); + await fx.WaitForAsync(pass.Id, JobState.Done); + } + Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built again"); + using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath); + Assert.Count(7, pack.Entries); + Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies)); + } + + [Test] + public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages() + { + await using var fx = await CreateAsync(context); + Populate(fx); + // The only worker runs the scan, then this job ahead of the background passes the scan queued: the folder goes + // meanwhile, as when someone renames or deletes it on the NAS between a poll and its passes. + var holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var engine = ServerEngine(fx, new CallbackHandler(JobKind.Render, async c => + { + holding.SetResult(); + await release.Task.WaitAsync(c.Cancellation); + })); + var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + var hold = await engine.EnqueueAsync(JobKind.Render, [fx.Archive.Id], priority: JobPriority.Normal, cancellationToken: fx.Ct); + await engine.StartAsync(fx.Ct); + await holding.Task.WaitAsync(TimeSpan.FromSeconds(30), fx.Ct); + Assert.Equal(JobState.Done, (await fx.Db.Jobs.GetAsync(scan.Id, fx.Ct))!.State); + var locations = await LocationsAsync(fx); + var (a, b) = (locations.Single(l => l.Folder == "a"), locations.Single(l => l.Folder == "b")); + Directory.Delete(Path.Combine(fx.Paths.Root, "a"), recursive: true); + release.SetResult(); + await fx.WaitForAsync(hold.Id, JobState.Done); + + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items; + await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([a.Id])).Id, JobState.Failed); + await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([b.Id])).Id, JobState.Done); + } + Assert.True(Built((await fx.Db.Sequences.GetAsync(b.Id))!.Status), "b built"); + using var pack = ThumbPackReader.Open(fx.Paths.Sequence("b").ThumbsPackPath); + Assert.Count(6, pack.Entries); + Assert.Count(6, Directory.GetFiles(fx.Paths.Sequence("b").Proxies)); + } + [Test] public async Task AThumbnailPassFlagsFramesNothingDecodesAndClearsThemOnceTheyDecode() { diff --git a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs index d15b5036..cdcee194 100644 --- a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs +++ b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs @@ -105,6 +105,57 @@ public sealed class MassRenameTests(TestContext context) Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder))); } + [Test] + public async Task ASequenceIndexedInsideAMovingFolderAfterTheDryRunMakesThePlanStale() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips"), Start, 2); + await fx.ScanAsync(); + var trips = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [trips.Id], "moved/{date}", null, fx.Ct); + Assert.Count(1, plan.Steps); + + // Photos copied into a subfolder and indexed by the next scan, after the dry run was shown. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips", "day2"), Start.AddDays(1), 2, firstSeed: 10); + await fx.ScanAsync(); + var day2 = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "trips/day2"); + + var stale = await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct)); + Assert.Equal(day2.Id, stale.SequenceId); + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + Assert.SequenceEqual(["trips|trips|trips/IMG_0001.JPG,trips/IMG_0002.JPG", "trips/day2|day2|trips/day2/IMG_0001.JPG,trips/day2/IMG_0002.JPG"], + await StateAsync(fx, [trips.Id, day2.Id])); + } + + [Test] + public async Task AFolderThatGainedASequenceWhileTheRenameRanStaysWhereItIs() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0], ids[1]], "renamed/{date}", null, fx.Ct); + Assert.SequenceEqual(["inbox/a", "inbox/b"], plan.Steps.Select(s => s.FromFolder)); + + // Once the first folder moved, a scan indexes photos copied into the second one. + var late = Path.Combine(fx.Paths.Root, "inbox", "b", "late"); + var progress = new SyncProgress(done => + { + if (done != 1) return; + Photos.WriteSeries(late, Start.AddDays(5), 2, firstSeed: 40); + fx.ScanAsync().GetAwaiter().GetResult(); + }); + await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, progress, fx.Ct)); + var newcomer = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "inbox/b/late"); + Assert.True(Directory.Exists(late), "the folder did not move"); + Assert.Count(1, RenameJournal.ListPending(fx.Paths).Single().Completed); + + // Resumed, the rest of the plan runs but that step is dropped: every index row still names its files. + Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct)); + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + var state = await StateAsync(fx, [ids[0], ids[1], newcomer.Id]); + Assert.SequenceEqual(["renamed/2026-09-28", "inbox/b", "inbox/b/late"], state.Select(s => s.Split('|')[0])); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } + /// "trips" with photos of its own, "trips/day2" inside it and "trips/day2/dawn" inside that: three nested sequences. private static async Task> NestedAsync(ArchiveFixture fx) { diff --git a/tests/Titano.Server.Tests/Api/AdminTests.cs b/tests/Titano.Server.Tests/Api/AdminTests.cs index ea8aebd3..744d20a2 100644 --- a/tests/Titano.Server.Tests/Api/AdminTests.cs +++ b/tests/Titano.Server.Tests/Api/AdminTests.cs @@ -177,5 +177,21 @@ public sealed class AdminTests(TestContext context) { await File.WriteAllBytesAsync(path, saved, context.Cancellation); } + + // A policy this form cannot set, written out of range by another tool, is named so the admin can correct the file. + var current = Titano.Core.Archiving.ManifestStore.ReadArchive(path); + foreach (var (odd, property) in new[] + { + (current with { ThumbnailPolicy = new Titano.Core.Archiving.ThumbnailPolicy { LongEdge = 2048 } }, "thumbnailPolicy"), + (current with { ProxyPolicy = current.ProxyPolicy with { Quality = 0 } }, "proxyPolicy"), + }) + { + Titano.Core.Archiving.ManifestStore.WriteArchive(path, odd); + var before = await File.ReadAllBytesAsync(path, context.Cancellation); + using var refused = await admin.PutAsJsonAsync("/api/v1/archive/settings", changed with { ProxyLongEdge = 1280 }, ProtocolJson.Default.ArchiveSettingsDto); + Assert.Equal(property, (await refused.ExpectErrorAsync(HttpStatusCode.Conflict, "archive_policy")).Details); + Assert.SequenceEqual(before, await File.ReadAllBytesAsync(path, context.Cancellation)); + } + await File.WriteAllBytesAsync(path, saved, context.Cancellation); } } diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs index 9112850f..9ca8dbc8 100644 --- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs @@ -68,7 +68,7 @@ public sealed class ArchiveModuleTests(TestContext context) } [Test] - public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPolling() + public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPollingWithThumbnailsAndProxies() { var photo = Photo(); // An adopted folder that already holds photos. @@ -78,8 +78,11 @@ public sealed class ArchiveModuleTests(TestContext context) File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo); }, ArchiveModule.Register); var archive = server.Services.GetRequiredService(); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); await UntilIndexedAsync(server, archive.Id, "roof"); await UntilScannedAsync(server); + // The scan queues the thumbnails and proxies clients load from the server, as an upload does. + await UntilImagesAsync(server, editor, archive.Id, "roof"); // Another writer adds a folder: the next poll queues a scan, but not while one is pending. Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden")); @@ -100,6 +103,7 @@ public sealed class ArchiveModuleTests(TestContext context) var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation)); Assert.Equal(JobPriority.Background, scan.Priority); await UntilIndexedAsync(server, archive.Id, "garden"); + await UntilImagesAsync(server, editor, archive.Id, "garden"); } private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder) @@ -112,6 +116,30 @@ public sealed class ArchiveModuleTests(TestContext context) } } + /// Waits until the first frame of the sequence in has its thumbnail and proxy served. + private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder) + { + var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder); + var frame = (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).First(); + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + foreach (var asset in new[] { "thumb", "proxy" }) + { + while (true) + { + using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}"); + if (response.StatusCode == HttpStatusCode.OK) + { + Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {folder}"); + break; + } + Assert.True(DateTime.UtcNow < until, $"no {asset} of {folder}: {(int)response.StatusCode}"); + await Task.Delay(100); + } + } + var built = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status; + Assert.True(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt), $"{folder}: {built}"); + } + /// Waits for the scan queued at start to finish: its sequences are indexed before it ends. private static async Task UntilScannedAsync(TestServer server) { diff --git a/tests/Titano.Server.Tests/Storage/UploadTests.cs b/tests/Titano.Server.Tests/Storage/UploadTests.cs index aea2324e..eccfb06a 100644 --- a/tests/Titano.Server.Tests/Storage/UploadTests.cs +++ b/tests/Titano.Server.Tests/Storage/UploadTests.cs @@ -170,6 +170,35 @@ public sealed class UploadTests(TestContext context) await oversized.ExpectErrorAsync(HttpStatusCode.BadRequest, "chunk_size"); } + [Test] + public async Task AnswersThatLeaveALargeChunkUnreadReachTheClient() + { + // Chunks far above the 1 MiB other requests may send: more than the connection buffers while the server reads nothing. + const int large = 32 * 1024 * 1024; + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = large); + var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader"); + var data = Uploads.Photo(24 * 1024 * 1024, 16); + var id = (await Uploads.StartAsync(uploader, data)).UploadId!.Value; + using (var first = await Uploads.PutChunkAsync(uploader, id, 0, data)) await first.ExpectAsync(HttpStatusCode.OK); + + // The client lost the answer and sends the chunk again: the same answer, not a closed connection. + using (var again = await Uploads.PutChunkAsync(uploader, id, 0, data)) + { + await again.ExpectAsync(HttpStatusCode.OK); + Assert.SequenceEqual([0], (await again.ReadAsync(ProtocolJson.Default.ChunkReceived)).Received); + } + using (var unknown = await Uploads.PutChunkAsync(uploader, Guid.CreateVersion7(), 0, data)) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); + using (var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null)) await complete.ExpectAsync(HttpStatusCode.OK); + using (var late = await Uploads.PutChunkAsync(uploader, id, 0, data)) await late.ExpectErrorAsync(HttpStatusCode.Conflict, "upload_state"); + + // Authentication answers before the endpoint runs: a user who may not upload, and a session signed out meanwhile. + var (viewer, _, _) = await server.UserAsync("marco", "viewer"); + using (var forbidden = await Uploads.PutChunkAsync(viewer, id, 0, data)) await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); + using (var logout = await uploader.PostAsync("/api/v1/auth/logout", null)) await logout.ExpectAsync(HttpStatusCode.NoContent); + using var revoked = await Uploads.PutChunkAsync(uploader, id, 0, data); + await revoked.ExpectErrorAsync(HttpStatusCode.Unauthorized, "unauthorized"); + } + [Test] public async Task RenderUploadsNeedTheEditPermission() {