From b0e3baa47cffe792f74954dfefff271b58a9564b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 09:56:30 +0000 Subject: [PATCH 1/8] fix(archive): hold the thumbnail lease while compacting packs CompactPacksAsync rewrote thumbs.pack under the sequence lease only, while thumbnail jobs append to the pack under the thumbs- lease, so a compaction racing a thumbnail pass could drop the entries just written. Compaction now takes the thumbnail lease too (CD-5). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Archiving/ThumbPack.cs | 2 +- .../Archiving/ArchiveMaintenance.cs | 4 +++ .../Archiving/ArchiveIntegrityTests.cs | 32 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/Titano.Core/Archiving/ThumbPack.cs b/src/Titano.Core/Archiving/ThumbPack.cs index 48bf29df..c2bf8eaa 100644 --- a/src/Titano.Core/Archiving/ThumbPack.cs +++ b/src/Titano.Core/Archiving/ThumbPack.cs @@ -279,7 +279,7 @@ public sealed class ThumbPackReader : IDisposable public void Dispose() => _stream?.Dispose(); } -/// Appends thumbnails to a pack; one writer per pack at a time (the sequence lease guarantees it). +/// Appends thumbnails to a pack; one writer per pack at a time (the thumbs-<sequence id> lease guarantees it). public sealed class ThumbPackWriter : IDisposable { private readonly FileStream _stream; diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs index 7e598f2d..26578022 100644 --- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -188,7 +188,11 @@ public sealed class ArchiveMaintenance if (length == 0 || garbage < length * CompactionRatio) continue; compacted.Add(location.Folder); if (dryRun) continue; + // The thumbnail builder appends under the thumbs lease only: holding it too keeps the compacted pack from + // replacing the file under a writer, whose later thumbnails would land in the orphaned one. await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) + using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{location.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait, + _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) ThumbPackWriter.Compact(sequence.ThumbsPackPath); } return compacted; diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs index 87a0215a..e73e5444 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs @@ -150,6 +150,38 @@ public sealed class ArchiveIntegrityTests(TestContext context) Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease")); Assert.True(folder.Frames().All(File.Exists)); } + + [Test] + public async Task PackCompactionWaitsForTheThumbnailWriter() + { + await using var fx = await CreateAsync(context); + var id = await IngestSeriesAsync(fx, "a", 2, 1); + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); + var hash = new byte[ThumbPack.HashLength]; + using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath)) + { + // The same entry written again and again: everything but the last copy is garbage. + for (var i = 0; i < 4; i++) pack.Append(0, 48, 32, hash, Photos.Jpeg(Start, 70 + i)); + pack.Commit(); + } + var length = new FileInfo(folder.ThumbsPackPath).Length; + + // A thumbnail pass holds the pack's own lease (not the sequence's) while it appends. + Task maintenance; + using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{id:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) + { + maintenance = fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); + await Task.Delay(500, fx.Ct); + Assert.False(maintenance.IsCompleted); + Assert.Equal(length, new FileInfo(folder.ThumbsPackPath).Length); + } + + var report = await maintenance; + Assert.SequenceEqual([folder.RelativeFolder], report.CompactedPacks); + using var compacted = ThumbPackReader.Open(folder.ThumbsPackPath); + Assert.Equal(0L, compacted.GarbageBytes); + Assert.NotNull(compacted.Read(0)); + } } internal static class SequencePathsExtensions From 5d01d6ab3450943004aa12a9aca12ae413246e5c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 09:56:54 +0000 Subject: [PATCH 2/8] fix(archive): move sequences nested in a renamed folder with it A mass rename or move of a sequence whose folder holds other sequences moved their files without updating their index rows, and a target inside another sequence's folder could nest one sequence in another. The plan now carries the nested sequences (listed in the dry run, moved with the folder that holds them, inner folders leaving first) and refuses a target inside a sequence folder with a localized NestedRenameException (CD-4). The Archive screen counts only the selected sequences in its summary. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- i18n/en/Archive.resx | 1 + i18n/it/Archive.it.resx | 1 + src/Titano.App/Archive/ArchiveFailures.cs | 3 +- src/Titano.App/Archive/MassOperations.cs | 4 +- src/Titano.Core/Archiving/MassRename.cs | 57 ++++++++++++++--- src/Titano.Data/Archiving/MassRenamer.cs | 17 ++++- .../Archiving/MassRenameTests.cs | 63 +++++++++++++++++++ 7 files changed, 134 insertions(+), 12 deletions(-) diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index 79ccbb34..5c11be26 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -81,6 +81,7 @@ The copy of {0} does not match the original.long: error sentence {0} could not be added to the sequence.long: error sentence {0} is already in the library.long: error sentence + {0} would be inside the sequence folder {1}.long: error sentence {0} was written by a newer Titano (format {1}).long: error sentence The archive {0} is offline.long: error sentence The archive {0} is read-only.long: error sentence diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 889e7ad4..06606110 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -76,6 +76,7 @@ La copia di {0} non corrisponde all'originale. Impossibile aggiungere {0} alla sequenza. {0} è già nella libreria. + {0} finirebbe dentro la cartella della sequenza {1}. {0} è stato scritto da un Titano più recente (formato {1}). L'archivio {0} non è raggiungibile. L'archivio {0} è di sola lettura. diff --git a/src/Titano.App/Archive/ArchiveFailures.cs b/src/Titano.App/Archive/ArchiveFailures.cs index b7013175..94565389 100644 --- a/src/Titano.App/Archive/ArchiveFailures.cs +++ b/src/Titano.App/Archive/ArchiveFailures.cs @@ -17,7 +17,8 @@ public static class ArchiveFailures ArchiveOfflineException offline => Loc.L(offline.Availability == ArchiveAvailability.ReadOnly ? "Archive.Error.ReadOnly" : "Archive.Error.Offline", Formatting.LeftToRight(offline.Root)), // Already one localized sentence each. - LeaseUnavailableException or NewerFormatException or StaleRenamePlanException or StaleDeletePlanException or SequenceInLibraryException => ex.Message, + LeaseUnavailableException or NewerFormatException or StaleRenamePlanException or NestedRenameException or StaleDeletePlanException + or SequenceInLibraryException => ex.Message, UnauthorizedAccessException => Loc.L("Archive.Failed.Access"), DatabaseException => Loc.L("Archive.Failed.Index"), ArgumentException => Loc.L("Archive.Failed.Gone"), diff --git a/src/Titano.App/Archive/MassOperations.cs b/src/Titano.App/Archive/MassOperations.cs index 4bcb2d3b..7be16553 100644 --- a/src/Titano.App/Archive/MassOperations.cs +++ b/src/Titano.App/Archive/MassOperations.cs @@ -41,8 +41,8 @@ public static class MassRename : new RenameRow(s.SequenceId, s.FromTitle ?? string.Empty, s.ToTitle ?? string.Empty)), ]; - /// "12 of 36 sequences change". - public static string Summary(RenamePlan plan, int setSize) => Loc.Plural("Archive.Rename.Summary", plan.Steps.Count, setSize); + /// "12 of 36 sequences change"; sequences that only move with a folder they are in are listed, not counted. + public static string Summary(RenamePlan plan, int setSize) => Loc.Plural("Archive.Rename.Summary", plan.Steps.Count(s => !s.Carried), setSize); } /// Adding and removing tags on a set of sequences (BRIEF §12.5 Retag). diff --git a/src/Titano.Core/Archiving/MassRename.cs b/src/Titano.Core/Archiving/MassRename.cs index caf423c9..c653b694 100644 --- a/src/Titano.Core/Archiving/MassRename.cs +++ b/src/Titano.Core/Archiving/MassRename.cs @@ -1,6 +1,7 @@ using System.Globalization; using System.Text.Json; using Titano.Core.IO; +using Titano.Core.Localization; using Titano.Core.Naming; namespace Titano.Core.Archiving; @@ -9,11 +10,20 @@ namespace Titano.Core.Archiving; public sealed record RenameSubject(Guid SequenceId, string Folder, string? Title, DateTime? FirstCapture, string? CameraMake, string? CameraModel, string? Lens); /// One sequence's change: its folder, its title, or both. -public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle) +/// The folder moves only because a folder it is inside moves (a sequence nested in another). +public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle, bool Carried = false) { public bool MovesFolder => !string.Equals(FromFolder, ToFolder, StringComparison.Ordinal); } +/// A planned folder would lie inside a sequence's folder (nesting one sequence in another); nothing is planned. +public sealed class NestedRenameException(string target, string folder) + : ArgumentException(Loc.L("Archive.Error.NestedTarget", target, folder)) +{ + public string Target { get; } = target; + public string Folder { get; } = folder; +} + /// The exact changes a mass rename will make (its dry run); applying it makes exactly these. public sealed record RenamePlan(Guid Id, Guid ArchiveId, IReadOnlyList Steps); @@ -50,16 +60,21 @@ public static class RenamePlanner /// Expands (where {title} is the current title) and then /// (where {title} is the new title) for every subject. A target folder /// that is any current sequence folder, exists on disk or was given to an earlier subject gets " (2)", " (3)"…. - /// Null templates leave that part unchanged; subjects without a change produce no step. + /// Null templates leave that part unchanged; subjects without a change produce no step. A folder that moves takes + /// the sequences inside it along: each gets a step after every move, so the index + /// follows its files; a selected sequence inside a moving folder that moves elsewhere leaves first. /// /// Every sequence folder of the archive (renamed or not). + /// The archive's other sequences inside a subject's folder, with their titles. + /// A target lies inside a sequence's folder, or inside another target. public static RenamePlan Plan(Guid archiveId, IReadOnlyList subjects, string? folderTemplate, string? titleTemplate, - IEnumerable occupied, Func existsOnDisk) + IEnumerable occupied, Func existsOnDisk, IReadOnlyList? nested = null) { var folder = folderTemplate is null ? null : PathTemplate.Parse(folderTemplate); var title = titleTemplate is null ? null : PathTemplate.Parse(titleTemplate); - var claimed = new HashSet(occupied, StringComparer.OrdinalIgnoreCase); - var steps = new List(); + var taken = occupied.ToList(); + var claimed = new HashSet(taken, StringComparer.OrdinalIgnoreCase); + var own = new List(subjects.Count); foreach (var s in subjects) { var context = new NamingContext @@ -80,11 +95,39 @@ public static class RenamePlanner newFolder = expanded.Equals(s.Folder, StringComparison.OrdinalIgnoreCase) ? expanded : Unique(expanded, claimed, existsOnDisk); claimed.Add(newFolder); } - if (newFolder != s.Folder || newTitle != s.Title) steps.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle)); + own.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle)); } - return new RenamePlan(Guid.CreateVersion7(), archiveId, steps); + + var moving = own.Where(s => s.MovesFolder).ToList(); + foreach (var step in moving) + { + // Into a sequence's folder (its own included) or another target: one sequence would end up inside another. + var host = taken.Concat(moving.Where(m => m != step).Select(m => m.ToFolder)).FirstOrDefault(f => IsInside(f, step.ToFolder)); + if (host is not null) throw new NestedRenameException(step.ToFolder, host); + } + var steps = new List<(RenameStep Step, int Order)>(); + foreach (var step in own.Concat((nested ?? []).Select(n => new RenameStep(n.SequenceId, n.Folder, n.Folder, n.Title, n.Title)))) + { + // The nearest folder around it that moves carries it; a sequence that moves elsewhere leaves before its folder does. + var carrier = step.MovesFolder ? null : moving.Where(m => IsInside(m.FromFolder, step.FromFolder)).MaxBy(m => m.FromFolder.Length); + if (carrier is not null) + steps.Add((step with { ToFolder = carrier.ToFolder + step.FromFolder[carrier.FromFolder.Length..], Carried = true }, 1)); + else if (step.MovesFolder) + steps.Add((step, -moving.Count(m => m != step && IsInside(m.FromFolder, step.FromFolder)))); + else if (step.FromTitle != step.ToTitle) + steps.Add((step, 0)); + } + // Stable: subjects keep their order, sequences inside moving folders move out first, carried ones come last. + return new RenamePlan(Guid.CreateVersion7(), archiveId, [.. steps.OrderBy(s => s.Order).Select(s => s.Step)]); } + /// + /// Whether lies inside (archive-relative, forward slashes); the + /// archive root, a sequence only when a folder was adopted in place, counts as containing nothing. + /// + public static bool IsInside(string folder, string path) => + folder.Length > 0 && path.Length > folder.Length + 1 && path[folder.Length] == '/' && path.StartsWith(folder, StringComparison.OrdinalIgnoreCase); + /// /// Moves the folder of a step; idempotent so a resumed plan can repeat it: an already moved folder is accepted, /// a case-only change goes through a temporary name (case-insensitive file systems). diff --git a/src/Titano.Data/Archiving/MassRenamer.cs b/src/Titano.Data/Archiving/MassRenamer.cs index 9cc0af2e..ad46a81b 100644 --- a/src/Titano.Data/Archiving/MassRenamer.cs +++ b/src/Titano.Data/Archiving/MassRenamer.cs @@ -32,6 +32,8 @@ public sealed class MassRenamer /// New folder template ({title} = the new title), or null to keep folders. /// New title template ({title} = the current title), or null to keep titles. + /// Sequences nested in a folder that moves are part of the plan (they move with it); see . + /// A new folder would lie inside a sequence's folder. public async Task PlanAsync(Archive archive, IReadOnlyList sequenceIds, string? folderTemplate, string? titleTemplate, CancellationToken cancellationToken = default) { @@ -42,12 +44,23 @@ public sealed class MassRenamer { if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } s || s.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(sequenceIds)); - subjects.Add(new RenameSubject(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens)); + subjects.Add(Subject(s)); + } + var nested = new List(); + if (folderTemplate is not null) + { + var selected = sequenceIds.ToHashSet(); + foreach (var location in locations.Where(l => !selected.Contains(l.Id) && subjects.Any(s => RenamePlanner.IsInside(s.Folder, l.Folder)))) + { + if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is { } inner) nested.Add(Subject(inner)); + } } return RenamePlanner.Plan(archive.Id, subjects, folderTemplate, titleTemplate, locations.Select(l => l.Folder), - folder => Directory.Exists(paths.Sequence(folder).Folder)); + folder => Directory.Exists(paths.Sequence(folder).Folder), nested); } + private static RenameSubject Subject(Sequence s) => new(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens); + /// Applies a plan; each completed step is journaled before the next starts. /// A sequence is no longer where the plan expects it; nothing of that step was changed. public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress? progress = null, CancellationToken cancellationToken = default) => diff --git a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs index 5cf3130a..c54ca153 100644 --- a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs +++ b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs @@ -105,6 +105,69 @@ public sealed class MassRenameTests(TestContext context) Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder))); } + /// "trips" with photos of its own, "trips/day2" inside it and "trips/day2/dawn" inside that: three nested sequences. + private static async Task> NestedAsync(ArchiveFixture fx) + { + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips"), Start, 2); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips", "day2"), Start.AddDays(1), 2, firstSeed: 10); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips", "day2", "dawn"), Start.AddDays(2), 2, firstSeed: 20); + await fx.ScanAsync(); + var locations = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).OrderBy(l => l.Folder, StringComparer.Ordinal).ToList(); + Assert.SequenceEqual(["trips", "trips/day2", "trips/day2/dawn"], locations.Select(l => l.Folder)); + return [.. locations.Select(l => l.Id)]; + } + + [Test] + public async Task SequencesInsideAMovedFolderAreInTheDryRunAndMoveWithIt() + { + await using var fx = await CreateAsync(context); + var ids = await NestedAsync(fx); + var before = await StateAsync(fx, ids); + + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "moved/{date}", null, fx.Ct); + Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-28/day2", "moved/2026-09-28/day2/dawn"], plan.Steps.Select(s => s.ToFolder)); + Assert.SequenceEqual([false, true, true], plan.Steps.Select(s => s.Carried)); + Assert.SequenceEqual(before, await StateAsync(fx, ids)); + + await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).MissingSequences); + } + + [Test] + public async Task NestedSequencesMovedTogetherLeaveTheirFolderFirst() + { + await using var fx = await CreateAsync(context); + var ids = await NestedAsync(fx); + var before = await StateAsync(fx, ids); + + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "moved/{date}", null, fx.Ct); + Assert.SequenceEqual([ids[2], ids[1], ids[0]], plan.Steps.Select(s => s.SequenceId)); + Assert.True(plan.Steps.All(s => !s.Carried)); + + await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); + Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-29", "moved/2026-09-30"], (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)) + .Select(l => l.Folder).Order(StringComparer.Ordinal)); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } + + [Test] + public async Task AFolderInsideASequenceFolderIsRefused() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var before = await StateAsync(fx, ids); + + var into = await Assert.ThrowsAsync(() => fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[1]], "inbox/a/{date}", null, fx.Ct)); + Assert.Equal("inbox/a", into.Folder); + var itself = await Assert.ThrowsAsync(() => fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "inbox/a/{date}", null, fx.Ct)); + Assert.Equal("inbox/a/2026-09-28", itself.Target); + Assert.SequenceEqual(before, await StateAsync(fx, ids)); + } + /// Reports synchronously (Progress<T> would post to the thread pool). private sealed class SyncProgress(Action report) : IProgress { From c87357b02c10d1c140cba400e5fba30f2734d279 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 09:58:36 +0000 Subject: [PATCH 3/8] fix(sync): finish stuck downloads and keep edits saved during a comparison A download whose .part already held the whole original (the job stopped while hashing it) or more asked the server to resume past the end, got 416 and failed on every retry; such a part is now hashed again, and one the server cannot resume starts over (CD-1). Auto-sync pulled the server's settings with the copy of the sequence read when the comparison began, so an edit saved on this PC meanwhile was overwritten unseen. The payload is now applied in one step under the sequence lease and only while the local settings version is the one read; otherwise nothing is written and the change is settled as a conflict (CD-3). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Sync/SyncService.cs | 95 +++++++++++++++---- src/Titano.Data/Sync/SyncTransfers.cs | 21 +++- .../Sync/SyncServiceTests.cs | 41 +++++++- .../Sync/SyncServerTests.cs | 7 +- 4 files changed, 137 insertions(+), 27 deletions(-) diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index 947e706c..7f8763e3 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -58,7 +58,7 @@ public sealed class SyncService private readonly TitanoDatabase _db; private readonly SyncLinkStore _links; - private readonly ProjectStore _projects; + private readonly SequenceWriter _writer; private readonly SemaphoreSlim _gate = new(1, 1); /// Unlinked pairs whose manifests were compared at these frame versions without a match (not fetched again). @@ -70,7 +70,7 @@ public sealed class SyncService Archives = archives; Options = options; _links = new SyncLinkStore(db); - _projects = new ProjectStore(db, new SequenceWriter(options.Archive)); + _writer = new SequenceWriter(options.Archive); Proxies = new ProxyCache(options.ProxyCacheDirectory); } @@ -211,8 +211,17 @@ public sealed class SyncService else { if (resolution == SyncResolution.SaveBoth) saved = await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false); - var applied = await ApplyLocallyAsync(local, theirs, cancellationToken).ConfigureAwait(false); - link = Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, seenRemote); + if (await ApplyLocallyAsync(local, theirs, cancellationToken).ConfigureAwait(false) is { } applied) + { + link = Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, seenRemote); + } + else + { + // Edited on this PC since it was read: the copy of a version the user did not choose to keep goes. + changed = true; + if (saved is not null) File.Delete(saved); + saved = null; + } } } if (!changed) @@ -359,7 +368,8 @@ public sealed class SyncService var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false); var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false); - var applied = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false); + // Edited on this PC meanwhile: nothing is agreed yet, so the next comparison settles both sides' content. + if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false) is not { } applied) return; await _links.UpsertAsync(Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, manifest.SettingsVersion), cancellationToken) .ConfigureAwait(false); } @@ -547,8 +557,13 @@ public sealed class SyncService return (policy.CanEdit ? await PushAsync(remote, link, local, link.BaseRemoteSettings, cancellationToken).ConfigureAwait(false) : link, local); manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); if (manifest.SettingsVersion != dto.SettingsVersion) return (link, local); - var pulled = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false); - return (Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion), pulled); + if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false) is { } pulled) + return (Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion), pulled); + // The local copy was read when the comparison began and an edit was saved on this PC since: both sides changed, + // so the server's version must not replace it unseen (BRIEF §14.5). + var fresh = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false); + if (fresh is null || fresh.SettingsVersion == local.SettingsVersion) return (link, local); + return await SettleSettingsAsync(remote, link, dto, fresh, manifest, policy, cancellationToken).ConfigureAwait(false); } /// Sends this PC's payload unless the server moved past , which shows up as a conflict next. @@ -621,20 +636,64 @@ public sealed class SyncService private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified); - /// Writes a payload into the local sidecar, project and index (each write bumps the local settings version). - private async Task ApplyLocallyAsync(Sequence local, SyncPayload payload, CancellationToken cancellationToken) + /// + /// Writes a payload into the local index, project and sidecar in one step (settings version + 1) under the sequence's + /// lease, which the editor's saves take too, and only while the local copy is still at the settings version + /// was read at. Null when an edit was saved on this PC since: nothing is written then. + /// + private async Task ApplyLocallyAsync(Sequence local, SyncPayload payload, CancellationToken cancellationToken) { - var current = local; - if (!payload.SameMetadata(SyncPayload.Of(local))) - current = await Archives.SetSequenceInfoAsync(local.Id, payload.Title, payload.Tags, payload.Rating, payload.Notes, cancellationToken).ConfigureAwait(false); - if (!SyncRules.SameSettings(payload.SettingsJson, current.SettingsJson)) + var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed."); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var folder = paths.Sequence(local.Folder); + await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false)) { - var archive = await _db.Archives.GetAsync(current.ArchiveId, cancellationToken).ConfigureAwait(false) - ?? throw new InvalidDataException($"Archive {current.ArchiveId} is not indexed."); - var project = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson)); - await _projects.UpdateAsync(new ArchivePaths(archive.RootPath), current, _ => project, cancellationToken).ConfigureAwait(false); + var row = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidDataException($"Sequence {local.Id} is not indexed."); + if (row.SettingsVersion != local.SettingsVersion) return null; + var metadata = !payload.SameMetadata(SyncPayload.Of(row)); + var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson); + if (!metadata && !settings) return row; + + string? project = null; + Guid? hero = null; + if (settings) + { + var edited = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson)); + project = edited.ToJson(); + hero = await HeroFrame.EditAsync(_db, row.Id, ProjectStore.Read(row, folder).HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false); + } + var edit = new SequenceEdit + { + Title = metadata ? payload.Title?.Trim() ?? string.Empty : null, + Tags = metadata ? payload.Tags : null, + Rating = metadata ? payload.Rating : null, + Notes = metadata ? payload.Notes ?? string.Empty : null, + SettingsJson = project, + HeroFrameId = hero, + }; + // The index first, at the version read under the lease: a writer that skips the lease still cannot be overwritten. + var result = await _db.Sequences.EditAsync(row.Id, edit, row.SettingsVersion, cancellationToken).ConfigureAwait(false); + if (result is not { Outcome: SequenceEditOutcome.Updated, Sequence: { } updated }) return null; + if (project is not null) AtomicFile.WriteAllText(folder.ProjectPath, project); + if (metadata) + { + var manifest = SequenceWriter.ReadLocked(folder, row.Id); + ManifestStore.WriteSequence(folder.ManifestPath, manifest with + { + Title = updated.Title, + Tags = updated.Tags, + Rating = updated.Rating, + Notes = updated.Notes, + Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) }, + UpdatedAt = DateTimeOffset.UtcNow, + }); + await Archives.Indexer.RecordStampsAsync(archive, folder, row.Id, cancellationToken).ConfigureAwait(false); + } + return updated; } - return await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) ?? current; } /// diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs index 5f0f356d..d4f2d7a7 100644 --- a/src/Titano.Data/Sync/SyncTransfers.cs +++ b/src/Titano.Data/Sync/SyncTransfers.cs @@ -397,7 +397,11 @@ internal sealed class SyncTransfers(SyncService sync) return result; } - /// A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two attempts). + /// + /// A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two + /// attempts). A part as long as the file (a job stopped while hashing it) is only hashed again; a part the server can + /// no longer resume (416) starts over. + /// private static async Task FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging, CancellationToken cancellationToken) { var sha = frame.Sha256.ToLowerInvariant(); @@ -407,8 +411,19 @@ internal sealed class SyncTransfers(SyncService sync) var part = target + ".part"; for (var attempt = 1; ; attempt++) { - await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) - await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false); + if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) + { + try + { + await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false); + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1) + { + File.Delete(part); + continue; + } + } if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha) { File.Move(part, target, overwrite: true); diff --git a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs index 0e28a1e7..f3ed537a 100644 --- a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs @@ -94,6 +94,31 @@ public sealed class SyncServiceTests(TestContext context) Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } + [Test] + public async Task AnEditSavedHereDuringTheComparisonIsNotReplacedByTheServers() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(1); + var local = await f.LocalAsync("Harbour", photos); + var remote = f.Remote.Add("Harbour", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + + // The server's copy changes; the editor saves this PC's edit after the comparison read the local copy, before it + // takes the server's version. + f.Remote.EditSettings(remote.Id, SyncFixture.Window33); + var racing = new RacingRemote(f.Remote, () => f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41).GetAwaiter().GetResult(), + beforeManifest: true); + await f.Sync.RefreshAsync(racing, new SyncPolicy(), f.Ct); + + var entry = await f.EntryAsync("Harbour"); + Assert.Equal(SyncStatus.Conflict, entry.Status); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local!.SettingsJson), "this PC's edit is kept"); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(entry.Local.Folder).ProjectPath, f.Ct))); + Assert.True(SyncRules.SameSettings(SyncFixture.Window33, entry.Link!.ConflictRemote!.SettingsJson), "the user picks"); + Assert.True(SyncRules.SameSettings(SyncFixture.Window33, remote.SettingsJson)); + } + [Test] public async Task WithoutAutomaticSyncAnEditWaitsForTheUser() { @@ -432,8 +457,11 @@ public sealed class SyncServiceTests(TestContext context) return (local, remote); } - /// A remote where someone else edits the settings right after the first read of a sequence, or right before the first edit. - private sealed class RacingRemote(FakeSyncRemote inner, Action race, bool beforeUpdate = false) : ISyncRemote + /// + /// A remote where someone edits right after the first read of a sequence, right before the first edit, or right + /// before the first manifest is read. + /// + private sealed class RacingRemote(FakeSyncRemote inner, Action race, bool beforeUpdate = false, bool beforeManifest = false) : ISyncRemote { private Action? _race = race; @@ -442,12 +470,17 @@ public sealed class SyncServiceTests(TestContext context) public async Task GetSequenceAsync(Guid id, CancellationToken cancellationToken) { var dto = await inner.GetSequenceAsync(id, cancellationToken); - if (!beforeUpdate) Interlocked.Exchange(ref _race, null)?.Invoke(); + if (!beforeUpdate && !beforeManifest) Interlocked.Exchange(ref _race, null)?.Invoke(); return dto; } public Task> ListSequencesAsync(CancellationToken cancellationToken) => inner.ListSequencesAsync(cancellationToken); - public Task GetManifestAsync(Guid id, CancellationToken cancellationToken) => inner.GetManifestAsync(id, cancellationToken); + public Task GetManifestAsync(Guid id, CancellationToken cancellationToken) + { + if (beforeManifest) Interlocked.Exchange(ref _race, null)?.Invoke(); + return inner.GetManifestAsync(id, cancellationToken); + } + public Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken) => inner.ListFramesAsync(sequenceId, cancellationToken); public Task UpdateSequenceAsync(Guid id, UpdateSequenceRequest request, CancellationToken cancellationToken) { diff --git a/tests/Titano.Server.Tests/Sync/SyncServerTests.cs b/tests/Titano.Server.Tests/Sync/SyncServerTests.cs index 7381c478..effc6a7c 100644 --- a/tests/Titano.Server.Tests/Sync/SyncServerTests.cs +++ b/tests/Titano.Server.Tests/Sync/SyncServerTests.cs @@ -112,16 +112,19 @@ public sealed class SyncServerTests(TestContext context) var (_, _, token) = await server.UserAsync("alberto", "editor"); await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop"); - var photos = Enumerable.Range(20, 3).Select(SyncClient.Photo).ToList(); + var photos = Enumerable.Range(20, 4).Select(SyncClient.Photo).ToList(); var local = await studio.LocalAsync("Aurora", photos); await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); var remoteId = (await studio.EntryAsync("Aurora")).Link!.RemoteId; - // An earlier attempt left half of one original, and a damaged half of another. + // An earlier attempt left half of one original, a damaged half of another, a third complete (stopped while it was + // being hashed) and a fourth longer than the file: the server cannot resume either of the last two (416). var staging = Path.Combine(laptop.Archive.RootPath, ".titano", "staging", "sync-" + remoteId.ToString("N")); Directory.CreateDirectory(staging); await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[0]) + ".jpg.part"), photos[0][..(photos[0].Length / 2)]); await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[1]) + ".jpg.part"), new byte[photos[1].Length / 2]); + await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[2]) + ".jpg.part"), photos[2]); + await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[3]) + ".jpg.part"), [.. photos[3], .. new byte[64]]); await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Aurora", SyncDownloadMode.Originals, laptop.Archive.Id); From 078cac75e48f3417e278f24bed0ff2117d15ba9a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 09:58:45 +0000 Subject: [PATCH 4/8] fix(server): scan the archive at start and every ten minutes The server indexed only what arrived through its own uploads, so photos already in an adopted folder and sequences another writer added stayed unknown until an administrator asked for a rescan. ArchivePolling, registered by the archive module, queues an incremental scan of the online archive at start and every ten minutes at background priority, unless a scan or rebuild of it is waiting, running or paused (ENG-13). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Server/Storage/ArchiveModule.cs | 6 +- src/Titano.Server/Storage/ArchivePolling.cs | 68 +++++++++++++++++++ .../Storage/ArchiveModuleTests.cs | 66 +++++++++++++++++- 3 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 src/Titano.Server/Storage/ArchivePolling.cs diff --git a/src/Titano.Server/Storage/ArchiveModule.cs b/src/Titano.Server/Storage/ArchiveModule.cs index 3002c9d4..29df251d 100644 --- a/src/Titano.Server/Storage/ArchiveModule.cs +++ b/src/Titano.Server/Storage/ArchiveModule.cs @@ -7,8 +7,8 @@ namespace Titano.Server.Storage; /// /// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and -/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance). Decoding is -/// managed only (BRIEF §4.2). +/// thumbnails, its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance) and the periodic +/// incremental scan (). Decoding is managed only (BRIEF §4.2). /// public static class ArchiveModule { @@ -22,5 +22,7 @@ public static class ArchiveModule services.AddSingleton(sp => sp.GetRequiredService().Ingest); services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); } } diff --git a/src/Titano.Server/Storage/ArchivePolling.cs b/src/Titano.Server/Storage/ArchivePolling.cs new file mode 100644 index 00000000..fe5bb2e4 --- /dev/null +++ b/src/Titano.Server/Storage/ArchivePolling.cs @@ -0,0 +1,68 @@ +using Titano.Core.Archiving; +using Titano.Core.Model; +using Titano.Data; +using Titano.Data.Archiving; +using Titano.Server.Jobs; + +namespace Titano.Server.Storage; + +/// +/// Polls the archive (BRIEF §7: by modification time and size, no file-system watchers): an incremental scan at start +/// and every while the archive is online, so photos already in an adopted folder and sequences +/// another writer adds (a desktop sharing the archive, files copied in by hand) are indexed without an administrator's +/// rescan. A scan or rebuild of the archive that is waiting, running or paused is not queued again. +/// +public sealed class ArchivePolling(TitanoDatabase db, ServerArchive archive, JobEngineHost jobs, TimeProvider time, ILogger logger) + : BackgroundService +{ + /// The desktop's rhythm (it polls its archive the same way). + public static readonly TimeSpan Interval = TimeSpan.FromMinutes(10); + + private static readonly JobState[] Pending = [JobState.Queued, JobState.Running, JobState.Paused]; + + /// Queues an incremental scan of an online archive unless one is pending; returns it, or null when none was queued. + public async Task PollAsync(CancellationToken cancellationToken) + { + if (!jobs.CanRun(JobKind.Scan)) return null; + var current = archive.Current; + if (ArchiveService.GetAvailability(current) != ArchiveAvailability.Online) return null; + foreach (var kind in new[] { JobKind.Scan, JobKind.Rebuild }) + { + foreach (var state in Pending) + { + if ((await db.Jobs.ListAsync(state, kind, 50, null, cancellationToken)).Items.Any(j => j.TargetIds.Contains(current.Id))) return null; + } + } + var scan = ArchiveJobs.Scan(current); + // Behind what users asked for (BRIEF §14.6). + return await jobs.Engine.EnqueueAsync(scan.Kind, scan.Targets, scan.Label, scan.ParametersJson, JobPriority.Background, + cancellationToken: cancellationToken); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (true) + { + try + { + await PollAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + logger.LogWarning(ex, "The archive could not be polled"); + } + try + { + await Task.Delay(Interval, time, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + } + } +} diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs index 172540f8..c2ba6b44 100644 --- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs @@ -1,6 +1,9 @@ using System.Net; +using Microsoft.Extensions.DependencyInjection; using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; +using Titano.Core.Model; +using Titano.Server.Jobs; using Titano.Server.Storage; using Titano.Testing; @@ -42,11 +45,70 @@ public sealed class ArchiveModuleTests(TestContext context) } } - private static byte[] Photo() + [Test] + public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPolling() + { + var photo = Photo(); + // An adopted folder that already holds photos. + await using var server = await TestServer.StartAsync(context, options => + { + Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof")); + File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo); + }, ArchiveModule.Register); + var archive = server.Services.GetRequiredService(); + await UntilIndexedAsync(server, archive.Id, "roof"); + await UntilScannedAsync(server); + + // Another writer adds a folder: the next poll queues a scan, but not while one is pending. + Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden")); + File.WriteAllBytes(Path.Combine(server.Options.ArchivePath, "garden", "IMG_0002.JPG"), Photo(seed: 3)); + var polling = server.Services.GetRequiredService(); + var paused = new Job + { + Id = Guid.CreateVersion7(), + Kind = JobKind.Scan, + TargetIds = [archive.Id], + State = JobState.Paused, + Created = DateTimeOffset.UtcNow, + }; + await server.Database.Jobs.EnqueueAsync(paused, context.Cancellation); + Assert.Null(await polling.PollAsync(context.Cancellation)); + await server.Services.GetRequiredService().Engine.CancelAsync(paused.Id, context.Cancellation); + + var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation)); + Assert.Equal(JobPriority.Background, scan.Priority); + await UntilIndexedAsync(server, archive.Id, "garden"); + } + + private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder) + { + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!(await server.Database.Sequences.ListLocationsAsync(archiveId)).Any(l => l.Folder == folder)) + { + Assert.True(DateTime.UtcNow < until, $"{folder} was not indexed"); + await Task.Delay(50); + } + } + + /// Waits for the scan queued at start to finish: its sequences are indexed before it ends. + private static async Task UntilScannedAsync(TestServer server) + { + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + foreach (var state in new[] { JobState.Queued, JobState.Running }) + { + while ((await server.Database.Jobs.ListAsync(state, JobKind.Scan)).Items.Count > 0) + { + Assert.True(DateTime.UtcNow < until, "the scan at start did not finish"); + await Task.Delay(50); + } + } + } + + private static byte[] Photo(int seed = 0) { const int width = 96, height = 64; var pixels = new byte[width * height * 3]; - for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)(i * 7 % 251); + for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed) % 251); return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 }); } } From 9588ed7f4bf9ad1dec1ba7cc69eea6d2b3351654 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 09:58:58 +0000 Subject: [PATCH 5/8] fix(proxies): get proxies of undecodable originals from clients and nodes The server decodes with managed decoders only, so HEIF originals had no proxies, thumbnails from their small previews and no pixels for analysis or managed renders, and nothing could send it proxies (ENG-05). Manifests now carry the archive's proxy spec and, per frame, its flags and whether the server wants its proxy. PUT /api/v1/frames/{id}/proxy (Edit or RenderNode) stores a JPEG of at most 64 MiB that decodes and whose Exif capture time is the frame's, then queues thumbnails and the proxy pass. An upload from the desktop makes and sends the wanted proxies after the originals and counts them as local changes; a render node sends those it can make from the originals it fetched once the job is done. The server never makes such a proxy from a preview, drops one left by a renumbered frame, takes thumbnails from a proxy, and analyses and renders those frames from their proxies. Render nodes fetched every frame of the manifest: frames missing, corrupt or excluded on the server failed the download, and originals not hashed yet failed the SHA-256 check (ENG-07). Nodes now skip those frames, fetch unhashed originals into the job folder with a size check, judge decodability with their own decoders, and use the proxy of a frame they cannot decode. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- i18n/en/Archive.resx | 1 + i18n/en/Server.resx | 3 + i18n/it/Archive.it.resx | 1 + i18n/it/Server.it.resx | 3 + src/Titano.Data/Archiving/ArchiveImages.cs | 185 +++++++++++++++++- .../Processing/Jobs/RenderJobHandler.cs | 2 +- .../Processing/RenderNodeClient.cs | 139 ++++++++++--- .../Processing/SequenceAnalyser.cs | 2 +- src/Titano.Data/Processing/SequenceFrames.cs | 37 +++- src/Titano.Data/Sync/SyncModel.cs | 5 +- src/Titano.Data/Sync/SyncRemote.cs | 9 + src/Titano.Data/Sync/SyncRules.cs | 14 ++ src/Titano.Data/Sync/SyncService.cs | 11 +- src/Titano.Data/Sync/SyncTransfers.cs | 81 +++++++- src/Titano.Protocol/Client/TitanoClient.cs | 19 ++ src/Titano.Protocol/Jobs.cs | 5 +- src/Titano.Protocol/Sequences.cs | 17 +- src/Titano.Server/Api/FileEndpoints.cs | 52 ++++- src/Titano.Server/Api/SequenceEndpoints.cs | 25 +-- src/Titano.Server/Jobs/RenderNodeService.cs | 24 +-- src/Titano.Server/Jobs/SequenceJobs.cs | 14 +- src/Titano.Server/Storage/ServerProxies.cs | 66 +++++++ .../Titano.Data.Tests/Archiving/ProxyTests.cs | 129 ++++++++++++ .../Processing/RenderJobTests.cs | 27 +++ .../Titano.Data.Tests/Sync/FakeSyncRemote.cs | 28 ++- .../Sync/SyncTransferTests.cs | 30 +++ .../Jobs/RenderNodeClientTests.cs | 71 ++++++- .../Storage/ProxyUploadTests.cs | 99 ++++++++++ 28 files changed, 1005 insertions(+), 94 deletions(-) create mode 100644 src/Titano.Server/Storage/ServerProxies.cs create mode 100644 tests/Titano.Data.Tests/Archiving/ProxyTests.cs create mode 100644 tests/Titano.Server.Tests/Storage/ProxyUploadTests.cs diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index 5c11be26..5486ffbe 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -132,6 +132,7 @@ Not copied: {0} ({1})long: job log line {0} stale locks, {1} temporary files, {2} packs compacted, {3} duplicate flags changedlong: job log line {0} unindexed photos, {1} missing photoslong: job log line + Proxies for the server: {0} sent, {1} not sentlong: job log line; {0} proxies sent; {1} proxies this PC could not make or the server refused {0} folders, {1} indexed, {2} adopted, {3} removedlong: job log line Sequence not found: {0}long: job log line {0} photos checked, {1} intact, {2} hashed, {3} with problemslong: job log line diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 543c7626..2f5002b7 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -54,6 +54,7 @@ This job kind cannot be started here.long The job parameters are not valid.long The priority must be background, normal or user.long + The proxy is not a JPEG the server can read.long The role must be admin, editor, uploader or viewer.long The setting {0} is not valid.long The settings must be valid JSON.long @@ -67,6 +68,8 @@ Not found.long That address is not a Titano Server.long Change your password first.long + This archive makes no proxies.long + The proxy was made from another photo.long This upload would exceed your quota.long Too many attempts; wait a minute and try again.long Choose a password different from the current one.long diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 06606110..27a8eca6 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -127,6 +127,7 @@ Non copiata: {0} ({1}) {0} blocchi scaduti, {1} file temporanei, {2} pacchetti compattati, {3} segni di duplicato cambiati {0} foto non indicizzate, {1} foto mancanti + Proxy per il server: {0} inviati, {1} non inviati {0} cartelle, {1} indicizzate, {2} adottate, {3} rimosse Sequenza non trovata: {0} {0} foto controllate, {1} integre, {2} con impronta nuova, {3} con problemi diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index 8baccf97..4ed52416 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -52,6 +52,7 @@ Questo tipo di job non si può avviare qui. I parametri del job non sono validi. La priorità deve essere background, normal o user. + Il proxy non è un JPEG che il server sa leggere. Il ruolo deve essere admin, editor, uploader o viewer. L'impostazione {0} non è valida. Le impostazioni devono essere JSON valido. @@ -65,6 +66,8 @@ Non trovato. Questo indirizzo non è un Titano Server. Cambia prima la password. + Questo archivio non crea proxy. + Il proxy è stato creato da un'altra foto. Questo caricamento supererebbe la tua quota. Troppi tentativi; attendi un minuto e riprova. Scegli una password diversa da quella attuale. diff --git a/src/Titano.Data/Archiving/ArchiveImages.cs b/src/Titano.Data/Archiving/ArchiveImages.cs index e8ae63be..f7fb6dc4 100644 --- a/src/Titano.Data/Archiving/ArchiveImages.cs +++ b/src/Titano.Data/Archiving/ArchiveImages.cs @@ -1,5 +1,7 @@ using System.Diagnostics; using Titano.Core.Archiving; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; using Titano.Core.IO; using Titano.Core.Metadata; using Titano.Core.Model; @@ -9,14 +11,35 @@ namespace Titano.Data.Archiving; /// Frames processed by one thumbnail or proxy pass over a sequence. public sealed record ImagePassResult(int Built, int Reused, int Undecodable); +/// What became of a proxy sent by a client or render node (). +public enum ProxyStoreResult +{ + Stored, + + /// The archive's proxy policy makes no proxies. + Disabled, + + /// Not a JPEG that decodes, or larger than . + Invalid, + + /// Its Exif capture time is not the frame's: it was made from another photo. + WrongFrame, +} + /// /// Thumbnails (thumbs.pack) and proxies (proxies/0001.jpg) of sequences (BRIEF §8.5). Both are /// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after -/// frames were renumbered the stale ones are rebuilt and the rest reused. +/// frames were renumbered the stale ones are rebuilt and the rest reused. A frame whose original this host cannot +/// decode () gets its proxy from a client or render node that can +/// (, BRIEF §2.3) and its thumbnail from that proxy. /// public sealed class ArchiveImages { + /// Largest proxy a client or render node may send: a full-size proxy of a large sensor at q92 fits. + public const long MaxProxyBytes = 64L << 20; + private const int CommitFrames = 200; + private const int ProbeLongEdge = 64; private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10); private readonly TitanoDatabase _db; @@ -32,6 +55,81 @@ public sealed class ArchiveImages _renderer = new FrameImageRenderer(writer.Options.Decoder); } + /// + /// Whether a host with takes the pixels of a original from its + /// proxy: no decoder reads the format, or only previews of a HEIF, which embeds thumbnails at best (BRIEF §2.3, + /// §8.1: the server keeps HEIF originals and relies on proxies made by a Windows client or render node). Never for a + /// JPEG (proxies are JPEGs: a host that reads them reads it) or a file no format was recognised in. + /// + public static bool NeedsProxy(ImageDecoder decoder, ImageFileFormat format) + { + ArgumentNullException.ThrowIfNull(decoder); + if (format is ImageFileFormat.Unknown or ImageFileFormat.Jpeg) return false; + var tier = decoder.TierFor(format); + return tier == DecodeTier.Unavailable || (tier != DecodeTier.Native && ImageFormatInfo.Of(format).Family == ImageFamily.Heif); + } + + /// with this archive engine's decoders. + public bool NeedsProxy(ImageFileFormat format) => NeedsProxy(_writer.Options.Decoder, format); + + /// + /// A proxy of the original at for a server that cannot decode it (): + /// at (0 = full size) and , with 's capture + /// data (the server's record, which it checks) and in its Exif. Null when + /// cannot decode the original either. + /// + /// The original cannot be read. + public static byte[]? MakeProxy(ImageDecoder decoder, string path, Frame frame, SidecarCamera? camera, int longEdge, int quality, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(frame); + if (NeedsProxy(decoder, frame.Format)) return null; + // The original's folder stands in for a sequence folder: the renderer reads frames relative to one. + var folder = new ArchivePaths(Path.GetDirectoryName(Path.GetFullPath(path))!).Sequence(string.Empty); + var source = new SidecarFrame + { + Id = frame.Id, + I = frame.Index, + File = Path.GetFileName(path), + Size = frame.Size, + CapturedAt = frame.CapturedAt, + UtcOffset = frame.UtcOffsetMinutes, + Exposure = frame.ExposureTime, + FNumber = frame.FNumber, + Iso = frame.Iso, + Focal = frame.FocalLength, + Bias = frame.ExposureBias, + Orientation = frame.Orientation, + Format = frame.Format, + }; + var policy = new ProxyPolicy { LongEdge = longEdge, Quality = quality }; + return new FrameImageRenderer(decoder).Proxy(folder, source, camera, policy, cancellationToken)?.Jpeg; + } + + /// + /// Files a proxy made elsewhere (a client or render node whose decoders read the original) as the frame's proxy, + /// replacing the one there: a JPEG of at most that decodes, whose Exif capture time is the + /// frame's (proxies carry it, so the proxy counts as current and cannot belong to another photo), written atomically. + /// + /// The archive is offline or read-only. + public ProxyStoreResult StoreProxy(Archive archive, Sequence sequence, Frame frame, byte[] jpeg) + { + ArgumentNullException.ThrowIfNull(jpeg); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + if (ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy.LongEdge == ProxyPolicy.Disabled) return ProxyStoreResult.Disabled; + if (jpeg.Length is 0 or > (int)MaxProxyBytes || !Decodes(jpeg)) return ProxyStoreResult.Invalid; + if (frame.CapturedAt is { } captured) + { + using var stream = new MemoryStream(jpeg, writable: false); + if (MetadataReader.Read(stream, "proxy.jpg").CapturedAt != captured) return ProxyStoreResult.WrongFrame; + } + var folder = paths.Sequence(sequence.Folder); + Directory.CreateDirectory(folder.Proxies); + AtomicFile.WriteAllBytes(folder.Proxy(frame.Index), jpeg); + return ProxyStoreResult.Stored; + } + /// Resume point inside the sequence (frames before it were done by an earlier run). /// Called with the next frame index whenever the pack has been committed. public async Task BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame, @@ -53,12 +151,22 @@ public sealed class ArchiveImages cancellationToken.ThrowIfCancellationRequested(); var frame = manifest.Frames[i]; var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size); - if (!force && pack.Entries.TryGetValue(frame.I, out var entry) && entry.SourceHash.AsSpan().SequenceEqual(fingerprint)) + var entry = pack.Entries.GetValueOrDefault(frame.I); + var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height); + var current = !force && entry is not null && entry.SourceHash.AsSpan().SequenceEqual(fingerprint); + // A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded + // preview, whose smaller thumbnail the proxy's replaces. + var fromProxy = false; + if (NeedsProxy(frame.Format) && !(current && edge >= FrameImageRenderer.ThumbnailLongEdge) + && ReadProxy(folder.Proxy(frame.I), frame) is { Check: ProxyCheck.Current or ProxyCheck.Unverifiable } proxy) + fromProxy = !current || edge < System.Math.Min(FrameImageRenderer.ThumbnailLongEdge, proxy.LongEdge); + if (current && !fromProxy) { reused++; continue; } - if (Render(() => _renderer.Thumbnail(folder, frame, cancellationToken)) is { } image) + var source = fromProxy ? ProxySource(frame) : frame; + if (Render(() => _renderer.Thumbnail(folder, source, cancellationToken)) is { } image) { pack.Append(frame.I, image.Width, image.Height, fingerprint, image.Jpeg); built++; @@ -97,12 +205,27 @@ public sealed class ArchiveImages cancellationToken.ThrowIfCancellationRequested(); var frame = manifest.Frames[i]; var path = folder.Proxy(frame.I); - if (!force && IsCurrent(path, frame)) + if (NeedsProxy(frame.Format)) + { + // Only a client or render node that decodes the original makes this proxy: a preview would make one the + // size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again. + var check = ReadProxy(path, frame).Check; + if (check is ProxyCheck.Current or ProxyCheck.Unverifiable) + { + reused++; + } + else + { + if (check == ProxyCheck.Stale) File.Delete(path); + undecodable++; + } + } + else if (!force && ReadProxy(path, frame).Check == ProxyCheck.Current) { reused++; continue; } - if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image) + else if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image) { AtomicFile.WriteAllBytes(path, image.Jpeg); built++; @@ -117,15 +240,59 @@ public sealed class ArchiveImages return new ImagePassResult(built, reused, undecodable); } - /// A proxy is current when its Exif capture time is the frame's (frames renumbered since make it stale). - private static bool IsCurrent(string proxy, SidecarFrame frame) + /// Where a frame's proxy stands; only a capture time tells whether it is the frame's. + private enum ProxyCheck { - if (!File.Exists(proxy)) return false; + Missing, + + /// Its Exif capture time is the frame's. + Current, + + /// The frame has no capture time to compare with. + Unverifiable, + + /// Made from another frame (frames were renumbered since). + Stale, + } + + /// Where the proxy at stands for , and its long edge. + private static (ProxyCheck Check, int LongEdge) ReadProxy(string proxy, SidecarFrame frame) + { + if (!File.Exists(proxy)) return (ProxyCheck.Missing, 0); try { - return MetadataReader.Read(proxy).CapturedAt == frame.CapturedAt && frame.CapturedAt is not null; + var metadata = MetadataReader.Read(proxy); + var (width, height) = metadata.OrientedSize; + var check = frame.CapturedAt is null ? ProxyCheck.Unverifiable : metadata.CapturedAt == frame.CapturedAt ? ProxyCheck.Current : ProxyCheck.Stale; + return (check, System.Math.Max(width, height)); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return (ProxyCheck.Missing, 0); + } + } + + /// The frame's proxy as the frame to draw: a JPEG already oriented, under proxies/. + private static SidecarFrame ProxySource(SidecarFrame frame) => frame with + { + File = ArchiveLayout.ProxiesFolder + "/" + ArchiveLayout.ProxyFileName(frame.I), + Format = ImageFileFormat.Jpeg, + Orientation = 1, + PreviewOffset = null, + PreviewLength = null, + }; + + /// A colour or grey JPEG whose headers parse and whose scans decode (at a small DCT-scaled size). + private static bool Decodes(byte[] jpeg) + { + try + { + var info = JpegDecoder.ReadInfo(jpeg); + if (info.Process == JpegProcess.Lossless || info.Components is not (1 or 3) || info.Width < 1 || info.Height < 1) return false; + using var probe = JpegDecoder.DecodeToLinear(jpeg, new JpegDecodeOptions { MaxLongEdge = ProbeLongEdge }); + return true; + } + catch (Exception ex) when (ex is InvalidDataException or NotSupportedException) { return false; } diff --git a/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs b/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs index 359361d5..8c1f1145 100644 --- a/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs +++ b/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs @@ -62,7 +62,7 @@ internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions opti var targets = all.Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).Select(t => (t.i, adapt(t.t))).ToList(); if (targets.Count == 0) return; - var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, cancellation).ConfigureAwait(false); + var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, options.Archive.Decoder, cancellation).ConfigureAwait(false); IReadOnlyList outcomes; await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames.Count), options)) { diff --git a/src/Titano.Data/Processing/RenderNodeClient.cs b/src/Titano.Data/Processing/RenderNodeClient.cs index c8938ad7..602548a4 100644 --- a/src/Titano.Data/Processing/RenderNodeClient.cs +++ b/src/Titano.Data/Processing/RenderNodeClient.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Net; using System.Text.Json; +using Titano.Core.Archiving; using Titano.Core.Diagnostics; using Titano.Core.Export; using Titano.Core.Imaging; @@ -8,6 +9,7 @@ using Titano.Core.Localization; using Titano.Core.Model; using Titano.Core.Render; using Titano.Core.Video; +using Titano.Data.Archiving; using Titano.Protocol; using Titano.Protocol.Client; @@ -47,8 +49,10 @@ public sealed record RenderNodeOptions /// /// "Offer this PC as render node" (BRIEF §14.7): registers this machine's single-file encoders with the server, /// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals -/// hash-verified into a content-addressed cache, or proxies when the job asks; a dropped download resumes), renders the -/// job's preset with the host's encoders and uploads the output, which the server files as a render of the sequence. +/// hash-verified into a content-addressed cache, or proxies when the job asks or this PC cannot decode the original; a +/// dropped download resumes; frames missing, corrupt or excluded on the server are left out), renders the job's preset +/// with the host's encoders and uploads the output, which the server files as a render of the sequence. Then it sends +/// the proxies the server asks for, made from the originals it fetched (BRIEF §2.3: the server cannot decode HEIF). /// Progress and render remarks go to the server; a job cancelled or reassigned there stops here; a failure is reported /// with its reason. Stopping abandons the job in flight without reporting it: the server requeues it when the node /// registers again or its heartbeats stop. Image sequences are not offered and batches are refused: the protocol uploads @@ -58,6 +62,12 @@ public sealed class RenderNodeClient : IAsyncDisposable { private const string LogCategory = "render-node"; + /// + /// Frames no node renders: absent on the server, not the archived file, or excluded. Undecodable is the server's + /// verdict on its own decoders, so this PC's decoders try those frames. + /// + internal const FrameFlags NotFetched = FrameFlags.Missing | FrameFlags.Corrupt | FrameFlags.Excluded; + private readonly TitanoClient _client; private readonly ProcessingOptions _processing; private readonly RenderNodeOptions _options; @@ -127,7 +137,9 @@ public sealed class RenderNodeClient : IAsyncDisposable try { Directory.CreateDirectory(work); - await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false); + var input = await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false); + // The job is done on the server; the proxies it asked for follow, on the node's own time. + await SendProxiesAsync(claimed, input, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (job.IsCancellationRequested) { @@ -221,7 +233,8 @@ public sealed class RenderNodeClient : IAsyncDisposable } } - private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken) + /// Renders the job and uploads its output; returns the frames it read. + private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken) { var export = ProcessingJobs.ReadExport(claimed.PresetJson); var resolved = ExportResolver.Resolve(export); @@ -232,9 +245,14 @@ public sealed class RenderNodeClient : IAsyncDisposable var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export }; var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false)); - var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false); - var files = await DownloadAsync(claimed, work, reporter, cancellationToken).ConfigureAwait(false); - var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id))]); + // This PC's decoders judge what they can read: the server's Undecodable does not keep a frame out of the render here. + var frames = (await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false)) + .ConvertAll(f => f with { Flags = f.Flags & ~FrameFlags.Undecodable }); + var files = await DownloadAsync(claimed, frames, work, reporter, cancellationToken).ConfigureAwait(false); + var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id).Path)]) + { + Proxied = [.. frames.Select(f => files.GetValueOrDefault(f.Id).Proxy)], + }; IReadOnlyList outcomes; await using (var render = new SequenceRender(input, project, null, _processing)) @@ -253,6 +271,56 @@ public sealed class RenderNodeClient : IAsyncDisposable cancellationToken).ConfigureAwait(false); } if (outcomes[0].Error is { } error) throw new InvalidOperationException(error); + return input; + } + + /// + /// Sends the proxies the server asks for () that this PC can make from the + /// originals it fetched, as the server's proxy policy says; a failure is logged and never fails the finished job. + /// + private async Task SendProxiesAsync(ClaimedJob claimed, SequenceFrames input, CancellationToken cancellationToken) + { + if (claimed.Proxies is not { } spec) return; + var wanted = claimed.Frames.Where(f => f.WantsProxy).Select(f => f.Id).ToHashSet(); + if (wanted.Count == 0) return; + var camera = new SidecarCamera { Make = input.Sequence.CameraMake, Model = input.Sequence.CameraModel }; + int sent = 0, unsent = 0; + for (var i = 0; i < input.Count; i++) + { + var frame = input.Frames[i]; + if (!wanted.Remove(frame.Id)) continue; + if (input.Proxied?[i] == true || input.Files[i] is not { } original) + { + unsent++; + continue; + } + try + { + var jpeg = await Task.Run(() => ArchiveImages.MakeProxy(_processing.Archive.Decoder, original, frame, camera, spec.LongEdge, spec.Quality, + cancellationToken), cancellationToken).ConfigureAwait(false); + if (jpeg is null) + { + unsent++; + continue; + } + await _client.UploadProxyAsync(frame.Id, jpeg, cancellationToken).ConfigureAwait(false); + sent++; + } + catch (TitanoApiException ex) when (ex.Status is HttpStatusCode.Conflict or HttpStatusCode.Forbidden) + { + // Proxies turned off on the server, or this account may no longer send them: none of the rest would go either. + Log.Warning(LogCategory, $"The server refused the proxies of job {claimed.Job.Id}", ex); + return; + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + // Best effort (a timed-out upload included): the job is done, and a proxy the server still lacks is asked for + // again with the next job. + Log.Warning(LogCategory, $"The proxy of {frame.FileName} could not be sent", ex); + unsent++; + } + } + Log.Info(LogCategory, $"Proxies of job {claimed.Job.Id}: {sent} sent, {unsent + wanted.Count} not"); } private async Task UploadAsync(Guid nodeId, Guid jobId, ExportTarget target, RenderResult result, CancellationToken cancellationToken) @@ -280,27 +348,33 @@ public sealed class RenderNodeClient : IAsyncDisposable } /// - /// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused - /// across jobs, fetched once per content), or proxies into the job folder when the job asks for them and the server - /// has them. + /// The files of the job's frames (none for frames missing, corrupt or excluded on the server): originals verified + /// against their SHA-256 into the content-addressed cache (reused across jobs, fetched once per content), originals the + /// server has not hashed yet into the job folder, or proxies into the job folder when the job asks for them or this PC + /// cannot decode the original, and the server has them. /// - private async Task> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken) + private async Task> DownloadAsync(ClaimedJob claimed, IReadOnlyList rows, string work, + NodeReporter reporter, CancellationToken cancellationToken) { - var files = new Dictionary(); + var formats = rows.ToDictionary(f => f.Id, f => f.Format); + bool Proxy(ManifestFrame f) => + f.HasProxy && (claimed.UseProxies || (formats.TryGetValue(f.Id, out var format) && ArchiveImages.NeedsProxy(_processing.Archive.Decoder, format))); + var files = new Dictionary(); + var fetched = claimed.Frames.Where(f => ((FrameFlags)f.Flags & NotFetched) == 0).ToList(); var done = 0; - var total = claimed.Frames.Count; + var total = fetched.Count; await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false); - var downloads = claimed.Frames.GroupBy(f => claimed.UseProxies && f.HasProxy ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList(); + var downloads = fetched.GroupBy(f => Proxy(f) || f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList(); await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken }, async (frames, ct) => { var frame = frames.First(); - var path = claimed.UseProxies && frame.HasProxy + var file = Proxy(frame) ? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false) - : await DownloadOriginalAsync(frame, ct).ConfigureAwait(false); + : (await DownloadOriginalAsync(frame, work, ct).ConfigureAwait(false), false); lock (files) { - foreach (var f in frames) files[f.Id] = path; + foreach (var f in frames) files[f.Id] = file; } await reporter.ReportAsync(Interlocked.Add(ref done, frames.Count()), total, null).ConfigureAwait(false); }).ConfigureAwait(false); @@ -312,25 +386,29 @@ public sealed class RenderNodeClient : IAsyncDisposable /// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server /// has none for the frame. /// - private async Task DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) + private async Task<(string Path, bool Proxy)> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) { var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg"); Directory.CreateDirectory(Path.GetDirectoryName(path)!); try { await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false); - return path; + return (path, true); } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) { File.Delete(path); - return await DownloadOriginalAsync(frame, cancellationToken).ConfigureAwait(false); + return (await DownloadOriginalAsync(frame, work, cancellationToken).ConfigureAwait(false), false); } } - /// A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256. - private async Task DownloadOriginalAsync(ManifestFrame frame, CancellationToken cancellationToken) + /// + /// A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256; an original the + /// server has not hashed yet goes into the job folder and must have the manifest's size. + /// + private async Task DownloadOriginalAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) { + if (frame.Sha256.Length == 0) return await DownloadUnhashedAsync(frame, work, cancellationToken).ConfigureAwait(false); var sha = frame.Sha256.ToLowerInvariant(); if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName)); var path = Path.Combine(_options.CacheDirectory, "originals", sha[..2], sha + Path.GetExtension(frame.FileName).ToLowerInvariant()); @@ -355,6 +433,23 @@ public sealed class RenderNodeClient : IAsyncDisposable } } + /// + /// An original without a SHA-256 on the server (adopted, not verified yet): nothing to check it against but its size, + /// so it is neither cached nor shared with other frames; a copy of another size is fetched again once. + /// + private async Task DownloadUnhashedAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) + { + var path = Path.Combine(work, "originals", frame.Id.ToString("N") + Path.GetExtension(frame.FileName).ToLowerInvariant()); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + for (var attempt = 1; ; attempt++) + { + if (!File.Exists(path) || new FileInfo(path).Length < frame.Size) await FetchAsync(frame, "original", path, resume: true, cancellationToken).ConfigureAwait(false); + if (new FileInfo(path).Length == frame.Size) return path; + File.Delete(path); + if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName)); + } + } + /// /// Fetches a frame resource into . A dropped connection or a busy server is retried (up to /// times, the wait doubling from ); diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index c99cd8c7..cfd48a58 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -36,7 +36,7 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); - var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, cancellationToken).ConfigureAwait(false); + var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false); var metadata = frames.LightFrames(); var light = LightAnalyzer.Analyze(metadata); diff --git a/src/Titano.Data/Processing/SequenceFrames.cs b/src/Titano.Data/Processing/SequenceFrames.cs index 6d9f7cc0..98ea6134 100644 --- a/src/Titano.Data/Processing/SequenceFrames.cs +++ b/src/Titano.Data/Processing/SequenceFrames.cs @@ -1,8 +1,10 @@ using Titano.Core.Analysis; using Titano.Core.Archiving; using Titano.Core.Diagnostics; +using Titano.Core.Imaging; using Titano.Core.Metadata; using Titano.Core.Model; +using Titano.Data.Archiving; namespace Titano.Data.Processing; @@ -20,22 +22,39 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList Fr public int Count => Frames.Count; - /// Every frame row of with its archive file (the proxy when asked and built). - public static async Task LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies, + /// + /// Per frame, whether its file is its proxy. is this host's verdict on the + /// original, so it does not keep a frame read from its proxy out of a render. Null: none is. + /// + public IReadOnlyList? Proxied { get; init; } + + /// The originals, for metadata (proxies carry no GPS); null when are the originals. + public IReadOnlyList? Originals { get; init; } + + /// + /// Every frame row of with its archive file: the proxy when asked, and whenever the + /// original is one cannot decode (: + /// HEIF on the server, BRIEF §2.3, §8.5), as long as the proxy exists; else the original. + /// + public static async Task LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies, ImageDecoder decoder, CancellationToken cancellationToken) { var frames = await ListAsync(db, sequence.Id, cancellationToken).ConfigureAwait(false); var paths = new ArchivePaths(archive.RootPath); var folder = paths.Sequence(sequence.Folder); var files = new string?[frames.Count]; + var originals = new string?[frames.Count]; + var proxied = new bool[frames.Count]; for (var i = 0; i < files.Length; i++) { var frame = frames[i]; if ((frame.Flags & FrameFlags.Missing) != 0) continue; - var proxy = useProxies ? folder.Proxy(frame.Index) : null; - files[i] = proxy is not null && File.Exists(proxy) ? proxy : paths.FromRelative(frame.RelativePath); + originals[i] = paths.FromRelative(frame.RelativePath); + var proxy = useProxies || ArchiveImages.NeedsProxy(decoder, frame.Format) ? folder.Proxy(frame.Index) : null; + proxied[i] = proxy is not null && File.Exists(proxy); + files[i] = proxied[i] ? proxy : originals[i]; } - return new SequenceFrames(sequence, frames, files); + return new SequenceFrames(sequence, frames, files) { Proxied = proxied, Originals = originals }; } public static async Task> ListAsync(TitanoDatabase db, Guid sequenceId, CancellationToken cancellationToken) @@ -80,15 +99,17 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList Fr { for (var i = 0; i < Frames.Count; i++) { - if (Files[i] is null || (Frames[i].Flags & NotRendered) != 0) yield return i; + var skip = Proxied is { } proxied && proxied[i] ? NotRendered & ~FrameFlags.Undecodable : NotRendered; + if (Files[i] is null || (Frames[i].Flags & skip) != 0) yield return i; } } private (int Index, GpsPosition Gps)? FirstGpsFix() { - for (var i = 0; i < Files.Count; i++) + var files = Originals ?? Files; + for (var i = 0; i < files.Count; i++) { - if (Files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue; + if (files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue; try { return MetadataReader.Read(file).Gps is { } gps ? (i, gps) : null; diff --git a/src/Titano.Data/Sync/SyncModel.cs b/src/Titano.Data/Sync/SyncModel.cs index ca465d6c..eb842ff5 100644 --- a/src/Titano.Data/Sync/SyncModel.cs +++ b/src/Titano.Data/Sync/SyncModel.cs @@ -91,7 +91,10 @@ public sealed record SyncLink public int? DiffLocalRenders { get; init; } public int? DiffRemoteRenders { get; init; } - /// Local originals the server does not have. + /// + /// Local originals the server does not have, and proxies it asks for that this PC can make (it cannot decode those + /// originals): an upload sends both. + /// public int MissingRemote { get; init; } /// Server originals this PC does not have. diff --git a/src/Titano.Data/Sync/SyncRemote.cs b/src/Titano.Data/Sync/SyncRemote.cs index 76268b4f..08537a27 100644 --- a/src/Titano.Data/Sync/SyncRemote.cs +++ b/src/Titano.Data/Sync/SyncRemote.cs @@ -42,6 +42,13 @@ public interface ISyncRemote : IDisposable /// Streams a frame's original, proxy or thumb, resuming with Range from the destination's length. Task DownloadFrameAsync(Guid frameId, string kind, Stream destination, CancellationToken cancellationToken); + /// + /// Sends the proxy of a frame the server cannot decode (); a 422 + /// means it was refused (not a JPEG of that frame), 409 that the server makes no + /// proxies. A remote without the call (the screenshots' sample server) throws . + /// + Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken cancellationToken) => throw new NotSupportedException(); + /// Streams a render's file, resuming with Range from the destination's length; returns the file name the server gives it. Task DownloadRenderAsync(Guid renderId, Stream destination, CancellationToken cancellationToken); @@ -116,6 +123,8 @@ public sealed class TitanoSyncRemote(TitanoClient client, bool ownsClient) : ISy public Task DownloadFrameAsync(Guid frameId, string kind, Stream destination, CancellationToken cancellationToken) => client.DownloadFrameAsync(frameId, kind, destination, null, cancellationToken); + public Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken cancellationToken) => client.UploadProxyAsync(frameId, jpeg, cancellationToken); + public Task DownloadRenderAsync(Guid renderId, Stream destination, CancellationToken cancellationToken) => client.DownloadRenderAsync(renderId, destination, cancellationToken); diff --git a/src/Titano.Data/Sync/SyncRules.cs b/src/Titano.Data/Sync/SyncRules.cs index a8aa0e12..ccda9f46 100644 --- a/src/Titano.Data/Sync/SyncRules.cs +++ b/src/Titano.Data/Sync/SyncRules.cs @@ -2,8 +2,10 @@ using System.Globalization; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; +using Titano.Core.Imaging; using Titano.Core.Model; using Titano.Core.Project; +using Titano.Data.Archiving; using Titano.Protocol; namespace Titano.Data.Sync; @@ -130,6 +132,18 @@ public static class SyncRules return (hashed.Count - matched + Math.Max(0, unhashed - unmatchedRemote), Math.Max(0, unmatchedRemote - unhashed)); } + /// + /// Frames whose proxy the server asks for (it cannot decode their originals, BRIEF §2.3) that this PC holds and its + /// reads: an upload makes and sends them. + /// + public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection local, ImageDecoder decoder) + { + ArgumentNullException.ThrowIfNull(manifest); + var readable = local.Where(f => !string.IsNullOrEmpty(f.Sha256) && !ArchiveImages.NeedsProxy(decoder, f.Format)) + .Select(f => f.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + return manifest.Frames.Count(f => f.WantsProxy && readable.Contains(f.Sha256.ToLowerInvariant())); + } + internal static string Blank(string? text) => text?.Trim() ?? string.Empty; private static JsonNode? Parse(string json) diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index 7f8763e3..a1e655f7 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -43,8 +43,9 @@ public sealed record SyncOptions /// conflict the user settles with , against the versions the user was shown. Uploads and /// downloads run as jobs (): uploads use the chunked, hash-verified, resumable protocol; /// downloaded originals are verified and committed through the archive ingest, so sidecars stay right; proxies go to a -/// cache and open as a ; renders travel on request. Proxies and thumbnails never travel -/// upward. A local copy and a server sequence holding the same originals are linked by content when no link joins +/// cache and open as a ; renders travel on request. Thumbnails never travel upward, and +/// proxies only for originals the server cannot decode (BRIEF §2.3: HEIF), made here and counted with the originals the +/// server lacks. A local copy and a server sequence holding the same originals are linked by content when no link joins /// them. The state of each local sequence is written to its sync_state for the Library. A failure while /// comparing one sequence is recorded on its link and does not stop the others. /// @@ -494,8 +495,10 @@ public sealed class SyncService manifest = await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false); // A frame flagged missing here counts as absent: downloading the originals restores it from the server. - var (missingRemote, missingLocal) = SyncRules.CompareOriginals( - [.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]); + var present = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList(); + var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]); + // Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3). + if (policy.CanEdit && manifest.Proxies is not null) missingRemote += SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder); link = await CompareExportsAsync(link, local, manifest.Renders, cancellationToken).ConfigureAwait(false) with { DiffLocalFrames = local.FramesVersion, diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs index d4f2d7a7..95e42f10 100644 --- a/src/Titano.Data/Sync/SyncTransfers.cs +++ b/src/Titano.Data/Sync/SyncTransfers.cs @@ -16,7 +16,8 @@ namespace Titano.Data.Sync; /// /// The work of upload and download jobs. Uploads send every local original the server lacks through the chunked, /// resumable protocol after hashing it again (an original that no longer matches its archived hash is never sent), -/// creating the server sequence first and sending its settings when the upload made it. Downloads fetch originals the +/// creating the server sequence first and sending its settings when the upload made it, then the proxies the server asks +/// for (originals it cannot decode, BRIEF §2.3), made here from this PC's originals. Downloads fetch originals the /// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 and commit /// them through the archive ingest, which names them and writes the sidecar; the first original creates the local /// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the @@ -73,12 +74,90 @@ internal sealed class SyncTransfers(SyncService sync) Interlocked.Increment(ref skipped); await progress.StepAsync().ConfigureAwait(false); }).ConfigureAwait(false); + await SendProxiesAsync(context, remote, target, local, paths, frames).ConfigureAwait(false); if (created) await sync.PushAllAsync(remote, target, ct).ConfigureAwait(false); await sync.RecompareAsync(remote, target, ct).ConfigureAwait(false); if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped)); } + /// + /// The proxies the server asks for () that this PC's decoders can make from its + /// originals, made as the server's proxy policy says with the server's capture data, and sent. One the server refuses + /// is counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a + /// server busy or out of reach fails the job like an original would. + /// + private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList frames) + { + var ct = context.Cancellation; + var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); + var wanted = manifest.Frames.Where(f => f.WantsProxy).ToList(); + if (manifest.Proxies is not { } spec || wanted.Count == 0) return; + var originals = frames.Where(f => f.Sha256 is not null).GroupBy(f => f.Sha256!.ToLowerInvariant()).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id); + var camera = new SidecarCamera { Make = local.CameraMake, Model = local.CameraModel, Serial = local.CameraSerial }; + var decoder = sync.Options.Archive.Decoder; + var progress = new TransferProgress(context, wanted.Count, 0); + await progress.ReportAsync().ConfigureAwait(false); + int sent = 0, unsent = 0; + using var refused = CancellationTokenSource.CreateLinkedTokenSource(ct); + try + { + await Parallel.ForEachAsync(wanted, Workers(sync.Options.Uploads, refused.Token), async (frame, token) => + { + var jpeg = originals.TryGetValue(frame.Sha256.ToLowerInvariant(), out var original) && records.TryGetValue(frame.Id, out var record) + ? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName, + token), token).ConfigureAwait(false) + : null; + try + { + if (jpeg is not null) + { + await remote.UploadProxyAsync(frame.Id, jpeg, token).ConfigureAwait(false); + Interlocked.Increment(ref sent); + } + else + { + Interlocked.Increment(ref unsent); + } + } + catch (Exception ex) when (ex is NotSupportedException or TitanoApiException { Status: HttpStatusCode.Conflict or HttpStatusCode.Forbidden }) + { + // The server takes no proxies (or none from this account): nothing more to send. + Core.Diagnostics.Log.Info("sync", $"Proxies of {local.DisplayTitle} not sent: {ex.Message}"); + await refused.CancelAsync().ConfigureAwait(false); + } + catch (TitanoApiException ex) when (!ex.IsTransient) + { + // This proxy only (not a JPEG of that frame, a frame gone meanwhile): the others still go. + Core.Diagnostics.Log.Warning("sync", $"The proxy of {frame.FileName} was refused", ex); + Interlocked.Increment(ref unsent); + } + await progress.StepAsync().ConfigureAwait(false); + }).ConfigureAwait(false); + } + catch (OperationCanceledException) when (refused.IsCancellationRequested && !ct.IsCancellationRequested) + { + unsent = wanted.Count - sent; + } + context.Log(Loc.L("Archive.Log.ProxiesSent", sent, unsent)); + } + + /// A proxy made here for the server; null (logged) when this PC cannot decode or read the original either. + private static byte[]? MakeProxy(Core.Imaging.ImageDecoder decoder, string path, Frame frame, SidecarCamera camera, ProxySpec spec, string name, + CancellationToken cancellationToken) + { + try + { + return ArchiveImages.MakeProxy(decoder, path, frame, camera, spec.LongEdge, spec.Quality, cancellationToken); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or NotSupportedException) + { + Core.Diagnostics.Log.Warning("sync", $"No proxy of {name} could be made", ex); + return null; + } + } + public async Task DownloadOriginalsAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid? targetArchive) { var ct = context.Cancellation; diff --git a/src/Titano.Protocol/Client/TitanoClient.cs b/src/Titano.Protocol/Client/TitanoClient.cs index 704ab62a..2d8c3dfa 100644 --- a/src/Titano.Protocol/Client/TitanoClient.cs +++ b/src/Titano.Protocol/Client/TitanoClient.cs @@ -142,6 +142,25 @@ public sealed class TitanoClient : IDisposable } } + /// + /// Sends the proxy of a frame whose original the server cannot decode (): a JPEG + /// with the frame's capture time in its Exif, replacing the server's. A transient failure is retried. + /// + public Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(jpeg); + return RetryAsync(async () => + { + await Bandwidth.AcquireAsync(jpeg.Length, ct).ConfigureAwait(false); + using var content = new ByteArrayContent(jpeg); + content.Headers.ContentType = new MediaTypeHeaderValue("image/jpeg"); + using var response = await SendRawAsync(new HttpRequestMessage(HttpMethod.Put, $"api/v1/frames/{frameId}/proxy") { Content = content }, ct) + .ConfigureAwait(false); + await EnsureSuccessAsync(response, ct).ConfigureAwait(false); + return true; + }, ct); + } + // ---------------- Uploads ---------------- /// diff --git a/src/Titano.Protocol/Jobs.cs b/src/Titano.Protocol/Jobs.cs index 38ba16cf..398b4d73 100644 --- a/src/Titano.Protocol/Jobs.cs +++ b/src/Titano.Protocol/Jobs.cs @@ -37,7 +37,10 @@ public sealed record NodeEncoder(string Codec, string Name, bool Hardware, bool public sealed record RenderNodeDto(Guid Id, string ClientName, string Username, IReadOnlyList Encoders, DateTimeOffset LastHeartbeat, string State); /// POST /api/v1/nodes/{id}/claim: the job and everything needed to run it, or 204 when none matches. -public sealed record ClaimedJob(JobDto Job, Guid SequenceId, string SettingsJson, string PresetJson, bool UseProxies, IReadOnlyList Frames); +/// Every frame of the sequence; a node renders none flagged missing, corrupt or excluded. +/// How to make the proxies the server asks for (); null when it makes none. +public sealed record ClaimedJob(JobDto Job, Guid SequenceId, string SettingsJson, string PresetJson, bool UseProxies, IReadOnlyList Frames, + ProxySpec? Proxies = null); /// POST /api/v1/nodes/{id}/jobs/{jobId}/progress. public sealed record NodeProgress(long Done, long Total, double? ItemsPerSecond, string? LogLine); diff --git a/src/Titano.Protocol/Sequences.cs b/src/Titano.Protocol/Sequences.cs index 8ad09422..45a91f17 100644 --- a/src/Titano.Protocol/Sequences.cs +++ b/src/Titano.Protocol/Sequences.cs @@ -65,6 +65,7 @@ public sealed record FrameDto( long Flags); /// GET /api/v1/sequences/{id}/manifest: what a client needs to diff a sequence (BRIEF §14.5). +/// How to make the proxies the server asks for (); null when it makes none. public sealed record SequenceManifest( Guid SequenceId, int FramesVersion, @@ -72,9 +73,21 @@ public sealed record SequenceManifest( string? SettingsJson, IReadOnlyList Tags, IReadOnlyList Frames, - IReadOnlyList Renders); + IReadOnlyList Renders, + ProxySpec? Proxies = null); -public sealed record ManifestFrame(Guid Id, int Index, string FileName, long Size, string Sha256, bool HasProxy, bool HasThumb); +/// One frame of a manifest: its content, whether the server has a proxy and thumbnail of it, and its flags. +/// Lower-case hex SHA-256; empty while the server has not hashed it (a folder adopted and not verified yet). +/// The frame's FrameFlags bits (missing, corrupt, excluded…). +/// +/// The server cannot decode this original and has no proxy of it (BRIEF §2.3, §8.1: HEIF): a client or render node that +/// decodes it sends one with PUT /api/v1/frames/{id}/proxy, made as says. +/// +public sealed record ManifestFrame(Guid Id, int Index, string FileName, long Size, string Sha256, bool HasProxy, bool HasThumb, long Flags = 0, + bool WantsProxy = false); + +/// How the server's proxies are made (BRIEF §8.5): long edge in pixels (0 = full size) and JPEG quality. +public sealed record ProxySpec(int LongEdge, int Quality); public sealed record RenderDto( Guid Id, diff --git a/src/Titano.Server/Api/FileEndpoints.cs b/src/Titano.Server/Api/FileEndpoints.cs index 76cf182f..42f273f7 100644 --- a/src/Titano.Server/Api/FileEndpoints.cs +++ b/src/Titano.Server/Api/FileEndpoints.cs @@ -1,6 +1,12 @@ using System.Globalization; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.Mvc; +using Titano.Core.Archiving; +using Titano.Core.Imaging.Decoders; using Titano.Core.Model; using Titano.Data; +using Titano.Data.Archiving; +using Titano.Server.Jobs; using Titano.Server.Security; using Titano.Server.Storage; @@ -8,7 +14,8 @@ namespace Titano.Server.Api; /// /// Streamed archive files with Range support (BRIEF §14.3): originals, proxies, thumbnails and exports. Paths come -/// from the database by id and are resolved under the archive root only. +/// from the database by id and are resolved under the archive root only. Proxies of frames the server cannot decode +/// arrive from clients and render nodes (PUT /frames/{id}/proxy, BRIEF §2.3). /// public static class FileEndpoints { @@ -21,6 +28,8 @@ public static class FileEndpoints api.MapGet("/frames/{id:guid}/original", OriginalAsync).Requires(Permissions.Edit | Permissions.RenderNode); api.MapGet("/frames/{id:guid}/proxy", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Proxy, db, a, l)) .Requires(Permissions.Read | Permissions.RenderNode); + // Made by clients that edit and by render nodes, which decode what the server cannot. + api.MapPut("/frames/{id:guid}/proxy", StoreProxyAsync).Requires(Permissions.Edit | Permissions.RenderNode); api.MapGet("/frames/{id:guid}/thumb", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Thumbnail, db, a, l)) .Requires(Permissions.Read); api.MapGet("/renders/{id:guid}/file", RenderAsync).Requires(Permissions.Read); @@ -48,6 +57,47 @@ public static class FileEndpoints ?? ApiErrors.NotFound(context, "asset_missing"); } + /// + /// Files the body (a JPEG made from the original, its Exif capture time the frame's, at most + /// ) as the frame's proxy; the sequence's thumbnails and proxy state follow as + /// background jobs. 204; 413 too_large, 422 invalid_proxy or proxy_mismatch, 409 + /// proxies_disabled, 503 archive_offline. + /// + private static async Task StoreProxyAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive, + [FromServices] ArchiveService? archives, SequenceJobs follow) + { + var ct = context.RequestAborted; + if (await db.Frames.GetAsync(id, ct) is not { } frame || await db.Sequences.GetAsync(frame.SequenceId, ct) is not { } sequence) + return ApiErrors.NotFound(context, "frame"); + if (context.Request.ContentLength is > ArchiveImages.MaxProxyBytes) return ApiErrors.Result(context, StatusCodes.Status413PayloadTooLarge, "too_large"); + // Proxies are larger than the JSON bodies Kestrel is limited to; reading past this answers 413. + if (context.Features.Get() is { IsReadOnly: false } limit) limit.MaxRequestBodySize = ArchiveImages.MaxProxyBytes; + using var body = new MemoryStream(); + await context.Request.Body.CopyToAsync(body, ct); + + var images = (archives ?? new ArchiveService(db, new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = ArchiveModule.Owner })).Images; + ProxyStoreResult result; + try + { + result = images.StoreProxy(archive.Current, sequence, frame, body.ToArray()); + } + catch (ArchiveOfflineException) + { + return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline"); + } + switch (result) + { + case ProxyStoreResult.Disabled: + return ApiErrors.Conflict(context, "proxies_disabled"); + case ProxyStoreResult.Invalid: + return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "invalid_proxy"); + case ProxyStoreResult.WrongFrame: + return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_mismatch"); + } + await follow.ProxyArrivedAsync(sequence, ct); + return Results.NoContent(); + } + private static async Task RenderAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive) { if (await db.Renders.GetAsync(id, context.RequestAborted) is not { } render) return ApiErrors.NotFound(context, "render"); diff --git a/src/Titano.Server/Api/SequenceEndpoints.cs b/src/Titano.Server/Api/SequenceEndpoints.cs index c84b82ad..f3ec9993 100644 --- a/src/Titano.Server/Api/SequenceEndpoints.cs +++ b/src/Titano.Server/Api/SequenceEndpoints.cs @@ -213,29 +213,24 @@ public static class SequenceEndpoints _ => ApiErrors.Conflict(context, "folder_missing"), }; - /// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5); ETag over the version counters. - private static async Task ManifestAsync(HttpContext context, Guid id, TitanoDatabase db) + /// + /// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5), with the proxies the server asks + /// for (); ETag over the version counters and the number of proxies asked for. + /// + private static async Task ManifestAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive) { var ct = context.RequestAborted; if (await db.Sequences.GetAsync(id, ct) is not { } sequence) return ApiErrors.NotFound(context, "sequence"); - var etag = string.Create(CultureInfo.InvariantCulture, $"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}\""); + var spec = ServerProxies.Spec(archive.Current); + var frames = await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, ct); + var etag = string.Create(CultureInfo.InvariantCulture, + $"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}-{frames.Count(f => f.WantsProxy)}\""); if (context.Request.Headers.IfNoneMatch == etag) return Results.StatusCode(StatusCodes.Status304NotModified); - var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt); - var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt); - var frames = new List(sequence.FrameCount); - var after = -1; - while (true) - { - var batch = await db.Frames.ListBySequenceAsync(id, after, 1000, ct); - frames.AddRange(batch.Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb))); - if (batch.Count < 1000) break; - after = batch[^1].Index; - } var renders = await db.Renders.ListBySequenceAsync(id, ct); context.Response.Headers.ETag = etag; return Results.Json(new SequenceManifest(id, sequence.FramesVersion, sequence.SettingsVersion, sequence.SettingsJson, sequence.Tags, frames, - [.. renders.Select(Dto.Render)]), ProtocolJson.Default.SequenceManifest); + [.. renders.Select(Dto.Render)], spec), ProtocolJson.Default.SequenceManifest); } private static async Task FramesAsync(HttpContext context, Guid id, string? cursor, int? limit, TitanoDatabase db) diff --git a/src/Titano.Server/Jobs/RenderNodeService.cs b/src/Titano.Server/Jobs/RenderNodeService.cs index b9f22d6f..4bb8a0d8 100644 --- a/src/Titano.Server/Jobs/RenderNodeService.cs +++ b/src/Titano.Server/Jobs/RenderNodeService.cs @@ -16,11 +16,14 @@ namespace Titano.Server.Jobs; /// /// Render nodes (BRIEF §14.7): clients with the render-node right register their encoders, heartbeat, claim queued /// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of -/// the sequence. A node that stops heart-beating loses its jobs to the queue (). +/// the sequence. A claimed job lists every frame with its flags (a node renders none missing, corrupt or excluded) and +/// the proxies the server asks for (), which the node sends from the originals it fetched. +/// A node that stops heart-beating loses its jobs to the queue (). /// public sealed class RenderNodeService( TitanoDatabase db, ServerOptions options, + ServerArchive archive, UploadService uploads, UploadStore store, RenderStore renders, @@ -84,8 +87,9 @@ public sealed class RenderNodeService( continue; } await TouchAsync(node, cancellationToken); + var spec = ServerProxies.Spec(archive.Current); return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson, - parameters.UseProxies, await ManifestFramesAsync(sequence, cancellationToken)); + parameters.UseProxies, await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, cancellationToken), spec); } await TouchAsync(node, cancellationToken); return (ClaimedJob?)null; @@ -234,22 +238,6 @@ public sealed class RenderNodeService( if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state)); } - private async Task> ManifestFramesAsync(Sequence sequence, CancellationToken cancellationToken) - { - var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt); - var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt); - var frames = new List(sequence.FrameCount); - var after = -1; - while (true) - { - var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, 1000, cancellationToken); - frames.AddRange(batch.Where(f => !f.Flags.HasFlag(FrameFlags.Excluded)) - .Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb))); - if (batch.Count < 1000) return frames; - after = batch[^1].Index; - } - } - /// /// The node has an encoder for the job's codec that handles its resolution: the encoder's own limit, else the node's /// (0 = no announced limit). diff --git a/src/Titano.Server/Jobs/SequenceJobs.cs b/src/Titano.Server/Jobs/SequenceJobs.cs index f53c962f..d1743b04 100644 --- a/src/Titano.Server/Jobs/SequenceJobs.cs +++ b/src/Titano.Server/Jobs/SequenceJobs.cs @@ -4,21 +4,27 @@ using Titano.Data; namespace Titano.Server.Jobs; /// -/// Follow-up work after frames arrive (BRIEF §14.4): thumbnails, proxies and light analysis of the sequence, queued at -/// background priority once per sequence while one is still waiting, and only for kinds this server can run. +/// Follow-up work after frames arrive (BRIEF §14.4): thumbnails, proxies and light analysis of the sequence; after a +/// proxy sent by a client or render node arrives, the thumbnails and the sequence's proxy state. Queued at background +/// priority once per sequence while one is still waiting, and only for kinds this server can run. /// public sealed class SequenceJobs(TitanoDatabase db, JobEngineHost jobs) { private static readonly JobKind[] AfterNewFrames = [JobKind.Thumbs, JobKind.Proxies, JobKind.LightAnalyse]; + private static readonly JobKind[] AfterProxy = [JobKind.Thumbs, JobKind.Proxies]; private readonly SemaphoreSlim _gate = new(1, 1); - public async Task FramesAddedAsync(Sequence sequence, CancellationToken cancellationToken) + public Task FramesAddedAsync(Sequence sequence, CancellationToken cancellationToken) => QueueAsync(sequence, AfterNewFrames, cancellationToken); + + public Task ProxyArrivedAsync(Sequence sequence, CancellationToken cancellationToken) => QueueAsync(sequence, AfterProxy, cancellationToken); + + private async Task QueueAsync(Sequence sequence, JobKind[] kinds, CancellationToken cancellationToken) { await _gate.WaitAsync(cancellationToken); try { - foreach (var kind in AfterNewFrames) + foreach (var kind in kinds) { if (!jobs.CanRun(kind) || await db.Jobs.IsQueuedAsync(kind, sequence.Id, cancellationToken)) continue; await jobs.Engine.EnqueueAsync(kind, [sequence.Id], sequence.DisplayTitle, priority: JobPriority.Background, cancellationToken: cancellationToken); diff --git a/src/Titano.Server/Storage/ServerProxies.cs b/src/Titano.Server/Storage/ServerProxies.cs new file mode 100644 index 00000000..fd70e4e5 --- /dev/null +++ b/src/Titano.Server/Storage/ServerProxies.cs @@ -0,0 +1,66 @@ +using Titano.Core.Archiving; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Decoders; +using Titano.Core.Model; +using Titano.Data; +using Titano.Data.Archiving; +using Titano.Protocol; + +namespace Titano.Server.Storage; + +/// +/// Proxies the server asks for (BRIEF §2.3, §8.5): it decodes with managed decoders only, so the frames whose originals +/// it cannot decode (HEIF) get their proxies from a Windows client or render node, made as the archive's proxy policy +/// says. Manifests (sync and render nodes) list every frame with its flags and whether its proxy is wanted. +/// +public static class ServerProxies +{ + private const int Page = 1000; + + /// The server's decoders (BRIEF §4.2: managed only). + private static readonly ImageDecoder Decoder = DecoderFactory.CreateServer(); + + /// The server takes this format's pixels from proxies made elsewhere. + public static bool CannotDecode(ImageFileFormat format) => ArchiveImages.NeedsProxy(Decoder, format); + + /// How proxies of the archive are made; null when it makes none (or its manifest cannot be read now). + public static ProxySpec? Spec(Archive archive) + { + try + { + var policy = ManifestStore.ReadArchive(new ArchivePaths(archive.RootPath).ManifestPath).ProxyPolicy; + return policy.LongEdge == ProxyPolicy.Disabled ? null : new ProxySpec(policy.LongEdge, policy.Quality); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) + { + return null; + } + } + + /// + /// Every frame of in index order, as a manifest lists it: a proxy is there when the server + /// built them all or one was sent for the frame; it is wanted for a frame the server cannot decode that has none + /// (and is not missing), unless is null. + /// + public static async Task> FramesAsync(TitanoDatabase db, Archive archive, Sequence sequence, ProxySpec? spec, + CancellationToken cancellationToken) + { + var proxies = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt); + var thumbs = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt); + var folder = new ArchivePaths(archive.RootPath).Sequence(sequence.Folder); + var frames = new List(sequence.FrameCount); + var after = -1; + while (true) + { + var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, Page, cancellationToken); + foreach (var f in batch) + { + var sent = !proxies && CannotDecode(f.Format) && File.Exists(folder.Proxy(f.Index)); + var wanted = spec is not null && !proxies && !sent && CannotDecode(f.Format) && (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0; + frames.Add(new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, proxies || sent, thumbs, (long)f.Flags, wanted)); + } + if (batch.Count < Page) return frames; + after = batch[^1].Index; + } + } +} diff --git a/tests/Titano.Data.Tests/Archiving/ProxyTests.cs b/tests/Titano.Data.Tests/Archiving/ProxyTests.cs new file mode 100644 index 00000000..f7b86b17 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ProxyTests.cs @@ -0,0 +1,129 @@ +using Titano.Core.Archiving; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Decoders; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +/// Proxies of originals a host cannot decode (BRIEF §2.3, §8.5): sent by a client or render node, checked, kept, thumbnailed. +[Category("archive")] +public sealed class ProxyTests(TestContext context) +{ + [Test] + public void OnlyFormatsAHostCannotDecodeTakeTheirPixelsFromProxies() + { + var server = DecoderFactory.CreateServer(); + Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Heif), "HEIF previews are thumbnails at best"); + Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Avif)); + Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.WebP), "no decoder at all"); + Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Cr3), "a RAW's embedded full-size preview"); + Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Dng)); + Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([]), ImageFileFormat.Jpeg), "proxies are JPEGs"); + Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Unknown)); + // A desktop whose decoders read HEIF makes the proxies a server asks for. + Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([new NativeHeif()]), ImageFileFormat.Heif)); + } + + [Test] + public async Task AProxySentForAFrameThisHostCannotDecodeIsCheckedKeptAndThumbnailed() + { + await using var fx = await CreateAsync(context); + var (sequence, frames) = await HeifSequenceAsync(fx); + var folder = fx.Paths.Sequence(sequence.Folder); + var images = fx.Service.Images; + + Assert.Equal(ProxyStoreResult.Invalid, images.StoreProxy(fx.Archive, sequence, frames[0], "not a photo"u8.ToArray())); + Assert.Equal(ProxyStoreResult.WrongFrame, images.StoreProxy(fx.Archive, sequence, frames[0], Proxy(frames[1].CapturedAt!.Value, 1))); + Assert.FileMissing(folder.Proxy(0)); + var proxy = Proxy(frames[0].CapturedAt!.Value, 2); + Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frames[0], proxy)); + Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct)); + + // The proxy pass keeps what was sent and makes none from the originals here, even when forced. + var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: true, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((0, 1, 1), (proxies.Built, proxies.Reused, proxies.Undecodable)); + Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct)); + Assert.FileMissing(folder.Proxy(1)); + + // The thumbnail comes from the proxy; the frame without one has none yet. + var thumbs = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((1, 0, 1), (thumbs.Built, thumbs.Reused, thumbs.Undecodable)); + using (var pack = ThumbPackReader.Open(folder.ThumbsPackPath)) + { + var entry = pack.Entries[0]; + Assert.Equal((FrameImageRenderer.ThumbnailLongEdge, 240), (entry.Width, entry.Height)); + } + var again = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable)); + } + + [Test] + public async Task AProxyLeftByARenumberedFrameIsAskedForAgain() + { + await using var fx = await CreateAsync(context); + var (sequence, frames) = await HeifSequenceAsync(fx); + var folder = fx.Paths.Sequence(sequence.Folder); + Directory.CreateDirectory(folder.Proxies); + await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[0].CapturedAt!.Value, 3), fx.Ct); + + var proxies = await fx.Service.Images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + + Assert.Equal(2, proxies.Undecodable); + Assert.FileMissing(folder.Proxy(1)); + Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt)); + } + + [Test] + public async Task AnArchiveWithoutProxiesRefusesThem() + { + await using var fx = await CreateAsync(context); + var (sequence, frames) = await HeifSequenceAsync(fx); + var manifest = ManifestStore.ReadArchive(fx.Paths.ManifestPath); + ManifestStore.WriteArchive(fx.Paths.ManifestPath, manifest with { ProxyPolicy = new ProxyPolicy { LongEdge = ProxyPolicy.Disabled } }); + + Assert.Equal(ProxyStoreResult.Disabled, fx.Service.Images.StoreProxy(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 4))); + Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).Proxy(0)); + } + + /// Two ingested frames whose originals the archive engine takes for HEIF (its decoders read none). + private static async Task<(Sequence Sequence, IReadOnlyList Frames)> HeifSequenceAsync(ArchiveFixture fx) + { + Guid id = default; + for (var i = 0; i < 2; i++) + { + var staged = fx.Stage(Start.AddSeconds(10 * i), 40 + i); + id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "heif", staged.Path, $"IMG_{i:0000}.HEIC", + staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId; + } + var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var sidecar = ManifestStore.ReadSequence(folder.ManifestPath); + ManifestStore.WriteSequence(folder.ManifestPath, sidecar with { Frames = [.. sidecar.Frames.Select(f => f with { Format = ImageFileFormat.Heif })] }); + return (sequence, await fx.Db.Frames.ListBySequenceAsync(id)); + } + + /// A 320 × 240 JPEG proxy with in its Exif, as a client makes it. + private static byte[] Proxy(DateTime captured, int seed) + { + using var image = new ImageBuffer(320, 240); + var level = 0.2f + seed % 5 * 0.1f; + for (var c = 0; c < 3; c++) image.Plane(c).Fill(level + c * 0.05f); + var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Titano" }); + return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); + } + + /// A provider that decodes HEIF natively (WIC with the HEIF extension on a desktop). + private sealed class NativeHeif : IImageDecoderProvider + { + public string Name => "native-heif"; + + public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null; + + public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null; + } +} diff --git a/tests/Titano.Data.Tests/Processing/RenderJobTests.cs b/tests/Titano.Data.Tests/Processing/RenderJobTests.cs index 656032ef..f0a0aa41 100644 --- a/tests/Titano.Data.Tests/Processing/RenderJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/RenderJobTests.cs @@ -63,6 +63,33 @@ public sealed class RenderJobTests(TestContext context) Assert.Equal(3, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount); } + [Test] + public async Task FramesThisHostCannotDecodeRenderFromTheirProxies() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Scene()); + var folder = fx.Folder(sequence); + Directory.CreateDirectory(folder.Proxies); + foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)) + { + // The originals are HEIF this host cannot read (BRIEF §2.3): only the proxies a client sent have pixels here. + var original = fx.Archive.Paths.FromRelative(frame.RelativePath); + File.Copy(original, folder.Proxy(frame.Index)); + await File.WriteAllBytesAsync(original, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8], fx.Ct); + } + await using (var session = await fx.Db.Database.OpenAsync(fx.Ct)) + await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE sequence_id = $2", [(int)ImageFileFormat.Heif, sequence.Id], fx.Ct); + await fx.StartEngineAsync(); + + // The job does not ask for proxies: the host takes them because it cannot decode the originals. + var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64)); + await fx.WaitForAsync(job.Id, JobState.Done); + + Assert.FileExists(Path.Combine(folder.Exports, "clip-MJPEG.avi")); + var decoded = fx.Decoder.Requests.Where(r => r.Purpose == DecodePurpose.Render).Select(r => Path.GetDirectoryName(r.Path)).Distinct().ToList(); + Assert.SequenceEqual([folder.Proxies], decoded); + } + [Test] public async Task ACancelledRenderLeavesNothingBehind() { diff --git a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs index c05213d4..275565b2 100644 --- a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs +++ b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs @@ -38,6 +38,9 @@ internal sealed class FakeSyncRemote : ISyncRemote /// Proxy and thumbnail downloads fail with a server error. public bool FailAssets { get; set; } + /// Proxies clients sent, by frame id (for sequences the server cannot decode). + public Dictionary SentProxies { get; } = []; + public FakeSequence Add(string? title, IEnumerable originals, string? settingsJson = null, SequenceStatus status = SequenceStatus.ProxiesBuilt | SequenceStatus.ThumbsBuilt) { @@ -96,7 +99,7 @@ internal sealed class FakeSyncRemote : ISyncRemote { ManifestCalls++; if (Refused.Contains(id)) throw new TitanoApiException(HttpStatusCode.Forbidden, new ApiError("forbidden", "refused")); - return Task.FromResult(Find(id).Manifest()); + return Task.FromResult(Find(id).Manifest(SentProxies.ContainsKey)); } } @@ -171,6 +174,18 @@ internal sealed class FakeSyncRemote : ISyncRemote await destination.WriteAsync(bytes.AsMemory(offset), cancellationToken); } + /// Like the server: only a frame whose proxy it asks for takes one, and only a JPEG. + public Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken cancellationToken) + { + lock (_gate) + { + if (!Sequences.Values.Any(s => s.Undecodable && s.Frames.Any(f => f.Id == frameId))) throw NotFound(); + if (jpeg is not [0xFF, 0xD8, ..]) throw new TitanoApiException(HttpStatusCode.UnprocessableEntity, new ApiError("invalid_proxy", "not a JPEG")); + SentProxies[frameId] = jpeg; + } + return Task.CompletedTask; + } + public async Task DownloadRenderAsync(Guid renderId, Stream destination, CancellationToken cancellationToken) { FakeRender render; @@ -225,6 +240,10 @@ internal sealed class FakeSequence public int SettingsVersion { get; set; } = 1; public int FramesVersion { get; set; } public SequenceStatus Status { get; set; } + + /// The server cannot decode these originals (HEIF): its manifest asks for proxies of them. + public bool Undecodable { get; set; } + public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow; public DateTime? FirstCapture { get; set; } public string? CameraModel { get; set; } = "Canon EOS R5"; @@ -243,9 +262,10 @@ internal sealed class FakeSequence public SequenceDto Dto() => new(Id, Title, Title ?? "Untitled", "2026/" + Id.ToString("N"), "Canon", CameraModel, null, FirstCapture, null, Frames.Count, 10, Frames.Sum(f => (long)f.Bytes.Length), (long)Status, Renders.Count, null, Tags, Rating, Notes, SettingsVersion, FramesVersion, 0, null, UpdatedAt); - public SequenceManifest Manifest() => new(Id, FramesVersion, SettingsVersion, SettingsJson, Tags, - [.. Frames.Select((f, i) => new ManifestFrame(f.Id, i, f.FileName, f.Bytes.Length, f.Sha256, (Status & SequenceStatus.ProxiesBuilt) != 0, - (Status & SequenceStatus.ThumbsBuilt) != 0))], [.. Renders.Select(r => r.Dto())]); + public SequenceManifest Manifest(Func proxySent) => new(Id, FramesVersion, SettingsVersion, SettingsJson, Tags, + [.. Frames.Select((f, i) => new ManifestFrame(f.Id, i, f.FileName, f.Bytes.Length, f.Sha256, (Status & SequenceStatus.ProxiesBuilt) != 0 || proxySent(f.Id), + (Status & SequenceStatus.ThumbsBuilt) != 0, WantsProxy: Undecodable && !proxySent(f.Id)))], [.. Renders.Select(r => r.Dto())], + Undecodable ? new ProxySpec(1280, 90) : null); } internal sealed record FakeRender(Guid Id, string FileName, string PresetName, byte[] Bytes) diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs index e4f523c5..129fd092 100644 --- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs @@ -1,5 +1,7 @@ +using Titano.Core.Codecs.Jpeg; using Titano.Core.Imaging; using Titano.Core.Imaging.Decoders; +using Titano.Core.Localization; using Titano.Core.Model; using Titano.Core.Render; using Titano.Data.Sync; @@ -32,6 +34,34 @@ public sealed class SyncTransferTests(TestContext context) Assert.Equal(remote.Id, entry.Link!.RemoteId); } + [Test] + public async Task AnUploadSendsTheProxiesTheServerCannotMakeItself() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Holy grail", photos); + // The server has the originals already (the Uploader sent them) but cannot decode them (HEIF, BRIEF §2.3). + var remote = f.Remote.Add("Holy grail", photos); + remote.Undecodable = true; + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var ahead = await f.EntryAsync("Holy grail"); + Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); + + Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); + foreach (var proxy in f.Remote.SentProxies.Values) + { + var info = JpegDecoder.ReadInfo(proxy); + Assert.True(info.Width <= 1280 && info.Height <= 1280, "made as the server's policy says"); + } + Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status); + } + [Test] public async Task AResumedUploadSendsOnlyWhatTheServerLacks() { diff --git a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs index c32fecf2..5b35d8a7 100644 --- a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs +++ b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs @@ -1,10 +1,14 @@ +using System.Collections.Concurrent; using System.Text; using Microsoft.Extensions.DependencyInjection; +using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; using Titano.Core.Diagnostics.Synthetic; using Titano.Core.Export; +using Titano.Core.Imaging; using Titano.Core.Imaging.Decoders; using Titano.Core.Localization; +using Titano.Core.Metadata; using Titano.Core.Model; using Titano.Core.Project; using Titano.Data.Archiving; @@ -57,6 +61,40 @@ public sealed class RenderNodeClientTests(TestContext context) Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed"); } + [Test] + public async Task ANodeSkipsMissingFramesFetchesUnhashedOnesAndSendsTheProxiesAskedFor() + { + await using var server = await TestServer.StartAsync(context); + var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); + var sequenceId = await UploadSequenceAsync(editor, Photos()[..3]); + await Uploads.UploadAsync(editor, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, 9)], "IMG_0004.HEIC", sequenceId); + var frames = await server.Database.Frames.ListBySequenceAsync(sequenceId); + // Frame 1 is gone from the server, frame 2 was adopted and never hashed, frame 4 is a HEIF the server cannot decode. + File.Delete(Path.Combine(server.Options.ArchivePath, frames[0].RelativePath)); + await server.Database.Frames.UpdateFlagsAsync([frames[0].Id], FrameFlags.Missing, FrameFlags.None); + await using (var session = await server.Database.Database.OpenAsync(context.Cancellation)) + { + await session.ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE id = $1", [frames[1].Id], context.Cancellation); + await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE id = $2", [(int)ImageFileFormat.Heif, frames[3].Id], context.Cancellation); + } + var job = await CreateRenderAsync(server, editor, sequenceId); + + var requests = new RecordPaths(new SocketsHttpHandler()); + using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", requests) { Token = token }; + // This PC decodes HEIF (WIC with the HEIF extension on Windows). + await using var node = Node(client, context.CreateTempDirectory("node"), decoder: new ImageDecoder([new SolidHeif(), new ManagedDecoderProvider()])); + Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); + + var done = await JobApiTests.GetAsync(editor, job.Id); + Assert.Equal("done", done.State, done.Error); + Assert.DoesNotContain($"/api/v1/frames/{frames[0].Id}/original", requests.Paths); + Assert.Contains($"/api/v1/frames/{frames[1].Id}/original", requests.Paths); + var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!; + var proxy = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder).Proxy(frames[3].Index); + Assert.Equal(frames[3].CapturedAt, MetadataReader.Read(proxy).CapturedAt, "the proxy the server asked for, with the frame's capture time"); + Assert.Equal((96, 64), MetadataReader.Read(proxy).OrientedSize); + } + [Test] public async Task ANodeReportsWhyItCouldNotRender() { @@ -263,8 +301,8 @@ public sealed class RenderNodeClientTests(TestContext context) Assert.FileExists(part[..^".part".Length]); } - private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null) => - new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } }, + private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null, ImageDecoder? decoder = null) => + new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = decoder ?? DecoderFactory.CreateServer() } }, new RenderNodeOptions { ClientName = "studio-pc", @@ -314,6 +352,35 @@ public sealed class RenderNodeClientTests(TestContext context) } } + /// Records the path of every request. + private sealed class RecordPaths(HttpMessageHandler inner) : DelegatingHandler(inner) + { + private readonly ConcurrentQueue _paths = new(); + + public IReadOnlyCollection Paths => _paths; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + _paths.Enqueue(request.RequestUri!.AbsolutePath); + return base.SendAsync(request, cancellationToken); + } + } + + /// Decodes every HEIF as a mid-grey 96 × 64 frame, as a native HEIF decoder would decode a real one. + private sealed class SolidHeif : IImageDecoderProvider + { + public string Name => "heif-test"; + + public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null; + + public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) + { + var image = new ImageBuffer(96, 64); + for (var c = 0; c < 3; c++) image.Plane(c).Fill(0.2f); + return new DecodedImage(image, DecodeTier.Native, Name, 96, 64); + } + } + /// Counts the node's claims the server answered. private sealed class CountClaims(HttpMessageHandler inner) : DelegatingHandler(inner) { diff --git a/tests/Titano.Server.Tests/Storage/ProxyUploadTests.cs b/tests/Titano.Server.Tests/Storage/ProxyUploadTests.cs new file mode 100644 index 00000000..688343b9 --- /dev/null +++ b/tests/Titano.Server.Tests/Storage/ProxyUploadTests.cs @@ -0,0 +1,99 @@ +using System.Net; +using System.Net.Http.Headers; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Metadata; +using Titano.Protocol; +using Titano.Server.Storage; +using Titano.Testing; + +namespace Titano.Server.Tests.Storage; + +/// +/// Proxies of originals the server cannot decode (BRIEF §2.3: HEIF) arrive from clients and render nodes through +/// PUT /api/v1/frames/{id}/proxy and become the frame's proxy and thumbnail. +/// +[Category("server")] +public sealed class ProxyUploadTests(TestContext context) +{ + [Test] + public async Task AProxySentForAFrameTheServerCannotDecodeIsCheckedKeptAndThumbnailed() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var (viewer, _, _) = await server.UserAsync("vera", "viewer"); + var upload = await Uploads.UploadAsync(editor, Heif(1), "IMG_0001.HEIC", sessionKey: "roof"); + var frame = Assert.NotNull(await server.Database.Frames.GetAsync(upload.FrameId)); + Assert.Equal(ImageFileFormat.Heif, frame.Format); + + var manifest = await ManifestAsync(editor, upload.SequenceId); + Assert.Equal((true, false), (manifest.Frames.Single().WantsProxy, manifest.Frames.Single().HasProxy)); + Assert.Equal(new ProxySpec(1920, 92), manifest.Proxies); + + using (var garbage = await PutAsync(editor, frame.Id, "not a photo"u8.ToArray())) + await garbage.ExpectErrorAsync(HttpStatusCode.UnprocessableEntity, "invalid_proxy"); + using (var other = await PutAsync(editor, frame.Id, Proxy(frame.CapturedAt!.Value.AddSeconds(1)))) + await other.ExpectErrorAsync(HttpStatusCode.UnprocessableEntity, "proxy_mismatch"); + var proxy = Proxy(frame.CapturedAt!.Value); + using (var refused = await PutAsync(viewer, frame.Id, proxy)) + await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); + using (var missing = await PutAsync(editor, Guid.CreateVersion7(), proxy)) + await missing.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); + using (var stored = await PutAsync(editor, frame.Id, proxy)) + await stored.ExpectAsync(HttpStatusCode.NoContent); + + await Http.BytesEqualAsync(proxy, editor.GetByteArrayAsync($"/api/v1/frames/{frame.Id}/proxy")); + var after = (await ManifestAsync(editor, upload.SequenceId)).Frames.Single(); + Assert.Equal((false, true), (after.WantsProxy, after.HasProxy)); + + // The thumbnail job the proxy queued draws the frame from it. + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (true) + { + using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.Id}/thumb"); + if (thumb.StatusCode == HttpStatusCode.OK) + { + var jpeg = await thumb.Content.ReadAsByteArrayAsync(); + Assert.Equal((320, 240), (JpegDecoder.ReadInfo(jpeg).Width, JpegDecoder.ReadInfo(jpeg).Height)); + break; + } + Assert.True(DateTime.UtcNow < until, $"thumbnail not built: {(int)thumb.StatusCode}"); + await Task.Delay(100); + } + } + + [Test] + public async Task FramesTheServerDecodesAskForNoProxy() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var upload = await Uploads.UploadAsync(editor, Proxy(new DateTime(2026, 9, 28, 18, 2, 11)), "IMG_0001.JPG", sessionKey: "roof"); + + Assert.False((await ManifestAsync(editor, upload.SequenceId)).Frames.Single().WantsProxy); + } + + /// A HEIF file as far as signature sniffing goes (an ftyp box naming heic), with no pixels the server can read. + private static byte[] Heif(int seed) => [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, seed)]; + + /// A 320 × 240 JPEG with in its Exif, as a client makes a proxy. + private static byte[] Proxy(DateTime captured) + { + using var image = new ImageBuffer(320, 240); + for (var c = 0; c < 3; c++) image.Plane(c).Fill(0.3f + c * 0.1f); + var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Titano" }); + return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); + } + + private static Task PutAsync(HttpClient client, Guid frameId, byte[] body) + { + var content = new ByteArrayContent(body); + content.Headers.ContentType = new MediaTypeHeaderValue("image/jpeg"); + return client.PutAsync($"/api/v1/frames/{frameId}/proxy", content); + } + + private static async Task ManifestAsync(HttpClient client, Guid sequenceId) + { + using var response = await client.GetAsync($"/api/v1/sequences/{sequenceId}/manifest"); + return await response.ReadAsync(ProtocolJson.Default.SequenceManifest); + } +} From f215b99ca5aaf88d3cd167bc6533e0c5a0a2797c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 11:06:42 +0000 Subject: [PATCH 6/8] fix(proxies): ask again for preview-sized proxies and refuse unwanted ones Earlier versions made the server's proxies of HEIF frames from their embedded previews. Those carry the frame's capture time, so the proxy pass kept them and the sequence stayed marked as having all its proxies; renders and analysis then read thumbnail-sized files and no client or render node was asked for a real one. The proxy pass now treats a proxy whose long edge falls short of the policy's for the frame (its own size when smaller, a tenth of slack, frames of unknown size not judged) as stale, and the server refuses such a proxy when one is sent (422 proxy_too_small). PUT /api/v1/frames/{id}/proxy also took a proxy of a frame the server decodes itself, replacing its own proxy with any JPEG of the right capture time; it now answers 422 proxy_not_wanted for those and for frames missing or damaged. Its follow-up jobs are queued even when the sender disconnects once the proxy is filed. A render node cleared the server's Undecodable flag on every frame, so a damaged frame the server's own render skips was downloaded and failed the node's job. Only a format the server cannot decode and the node can loses the flag now; other undecodable originals are neither fetched nor rendered unless read from their proxy. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- i18n/en/Server.resx | 2 + i18n/it/Server.it.resx | 2 + src/Titano.Data/Archiving/ArchiveImages.cs | 39 ++++++++++++++-- .../Processing/RenderNodeClient.cs | 12 +++-- src/Titano.Server/Api/FileEndpoints.cs | 18 ++++++-- src/Titano.Server/Storage/ServerProxies.cs | 8 +--- .../Titano.Data.Tests/Archiving/ProxyTests.cs | 46 ++++++++++++++++--- .../Jobs/RenderNodeClientTests.cs | 7 ++- .../Storage/ProxyUploadTests.cs | 3 ++ 9 files changed, 112 insertions(+), 25 deletions(-) diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 2f5002b7..22398fac 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -70,6 +70,8 @@ Change your password first.long This archive makes no proxies.long The proxy was made from another photo.long + The server does not need a proxy of this photo.long + The proxy is smaller than this archive's proxies.long This upload would exceed your quota.long Too many attempts; wait a minute and try again.long Choose a password different from the current one.long diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index 4ed52416..9a74fb04 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -68,6 +68,8 @@ Cambia prima la password. Questo archivio non crea proxy. Il proxy è stato creato da un'altra foto. + Al server non serve un proxy di questa foto. + Il proxy è più piccolo dei proxy di questo archivio. Questo caricamento supererebbe la tua quota. Troppi tentativi; attendi un minuto e riprova. Scegli una password diversa da quella attuale. diff --git a/src/Titano.Data/Archiving/ArchiveImages.cs b/src/Titano.Data/Archiving/ArchiveImages.cs index f7fb6dc4..099c6333 100644 --- a/src/Titano.Data/Archiving/ArchiveImages.cs +++ b/src/Titano.Data/Archiving/ArchiveImages.cs @@ -2,6 +2,7 @@ using System.Diagnostics; using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; using Titano.Core.Imaging; +using Titano.Core.Imaging.Decoders; using Titano.Core.IO; using Titano.Core.Metadata; using Titano.Core.Model; @@ -24,6 +25,9 @@ public enum ProxyStoreResult /// Its Exif capture time is not the frame's: it was made from another photo. WrongFrame, + + /// Smaller than the archive's proxies of that frame: made from an embedded preview, not the original. + TooSmall, } /// @@ -41,6 +45,7 @@ public sealed class ArchiveImages private const int CommitFrames = 200; private const int ProbeLongEdge = 64; private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10); + private static readonly ImageDecoder ServerDecoders = DecoderFactory.CreateServer(); private readonly TitanoDatabase _db; private readonly SequenceWriter _writer; @@ -72,6 +77,12 @@ public sealed class ArchiveImages /// with this archive engine's decoders. public bool NeedsProxy(ImageFileFormat format) => NeedsProxy(_writer.Options.Decoder, format); + /// + /// on a Titano Server, whose decoders are managed only (BRIEF + /// §4.2): it asks clients and render nodes for the proxies of those originals. + /// + public static bool ServerNeedsProxy(ImageFileFormat format) => NeedsProxy(ServerDecoders, format); + /// /// A proxy of the original at for a server that cannot decode it (): /// at (0 = full size) and , with 's capture @@ -109,7 +120,8 @@ public sealed class ArchiveImages /// /// Files a proxy made elsewhere (a client or render node whose decoders read the original) as the frame's proxy, /// replacing the one there: a JPEG of at most that decodes, whose Exif capture time is the - /// frame's (proxies carry it, so the proxy counts as current and cannot belong to another photo), written atomically. + /// frame's (proxies carry it, so the proxy counts as current and cannot belong to another photo) and whose size is + /// the policy's (), written atomically. /// /// The archive is offline or read-only. public ProxyStoreResult StoreProxy(Archive archive, Sequence sequence, Frame frame, byte[] jpeg) @@ -117,8 +129,11 @@ public sealed class ArchiveImages ArgumentNullException.ThrowIfNull(jpeg); var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); - if (ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy.LongEdge == ProxyPolicy.Disabled) return ProxyStoreResult.Disabled; + var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy; + if (policy.LongEdge == ProxyPolicy.Disabled) return ProxyStoreResult.Disabled; if (jpeg.Length is 0 or > (int)MaxProxyBytes || !Decodes(jpeg)) return ProxyStoreResult.Invalid; + var info = JpegDecoder.ReadInfo(jpeg); + if (TooSmall(System.Math.Max(info.Width, info.Height), frame.Width, frame.Height, policy)) return ProxyStoreResult.TooSmall; if (frame.CapturedAt is { } captured) { using var stream = new MemoryStream(jpeg, writable: false); @@ -208,8 +223,10 @@ public sealed class ArchiveImages if (NeedsProxy(frame.Format)) { // Only a client or render node that decodes the original makes this proxy: a preview would make one the - // size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again. - var check = ReadProxy(path, frame).Check; + // size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again, and so + // does one smaller than the policy's (made here from a preview by an earlier version). + var (check, longEdge) = ReadProxy(path, frame); + if (check is ProxyCheck.Current or ProxyCheck.Unverifiable && TooSmall(longEdge, frame.W, frame.H, policy)) check = ProxyCheck.Stale; if (check is ProxyCheck.Current or ProxyCheck.Unverifiable) { reused++; @@ -272,6 +289,20 @@ public sealed class ArchiveImages } } + /// + /// Whether a proxy whose long edge is falls short of the policy's for a frame of + /// × (the policy's long edge, or the frame's own when smaller or + /// at full size): one made from an embedded preview. A frame of unknown size is never judged (a small original's + /// proxy would be refused again and again); a tenth of slack spares decoders that crop a few pixels. + /// + private static bool TooSmall(int longEdge, int width, int height, ProxyPolicy policy) + { + var own = System.Math.Max(width, height); + if (own <= 0) return false; + var expected = policy.LongEdge == ProxyPolicy.FullSize ? own : System.Math.Min(own, policy.LongEdge); + return longEdge * 10L < expected * 9L; + } + /// The frame's proxy as the frame to draw: a JPEG already oriented, under proxies/. private static SidecarFrame ProxySource(SidecarFrame frame) => frame with { diff --git a/src/Titano.Data/Processing/RenderNodeClient.cs b/src/Titano.Data/Processing/RenderNodeClient.cs index 602548a4..3ca316bf 100644 --- a/src/Titano.Data/Processing/RenderNodeClient.cs +++ b/src/Titano.Data/Processing/RenderNodeClient.cs @@ -245,9 +245,13 @@ public sealed class RenderNodeClient : IAsyncDisposable var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export }; var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false)); - // This PC's decoders judge what they can read: the server's Undecodable does not keep a frame out of the render here. + // The server's Undecodable stands, as in its own render, except for a format only its decoders lack (HEIF) that + // this PC's read: such a frame renders here from its original. One read from its proxy renders either way. + var decoder = _processing.Archive.Decoder; var frames = (await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false)) - .ConvertAll(f => f with { Flags = f.Flags & ~FrameFlags.Undecodable }); + .ConvertAll(f => ArchiveImages.ServerNeedsProxy(f.Format) && !ArchiveImages.NeedsProxy(decoder, f.Format) + ? f with { Flags = f.Flags & ~FrameFlags.Undecodable } + : f); var files = await DownloadAsync(claimed, frames, work, reporter, cancellationToken).ConfigureAwait(false); var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id).Path)]) { @@ -357,10 +361,12 @@ public sealed class RenderNodeClient : IAsyncDisposable NodeReporter reporter, CancellationToken cancellationToken) { var formats = rows.ToDictionary(f => f.Id, f => f.Format); + var undecodable = rows.Where(f => (f.Flags & FrameFlags.Undecodable) != 0).Select(f => f.Id).ToHashSet(); bool Proxy(ManifestFrame f) => f.HasProxy && (claimed.UseProxies || (formats.TryGetValue(f.Id, out var format) && ArchiveImages.NeedsProxy(_processing.Archive.Decoder, format))); var files = new Dictionary(); - var fetched = claimed.Frames.Where(f => ((FrameFlags)f.Flags & NotFetched) == 0).ToList(); + // An original that stays undecodable would only be skipped by the render. + var fetched = claimed.Frames.Where(f => ((FrameFlags)f.Flags & NotFetched) == 0 && (Proxy(f) || !undecodable.Contains(f.Id))).ToList(); var done = 0; var total = fetched.Count; await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false); diff --git a/src/Titano.Server/Api/FileEndpoints.cs b/src/Titano.Server/Api/FileEndpoints.cs index 42f273f7..d7090edb 100644 --- a/src/Titano.Server/Api/FileEndpoints.cs +++ b/src/Titano.Server/Api/FileEndpoints.cs @@ -58,10 +58,12 @@ public static class FileEndpoints } /// - /// Files the body (a JPEG made from the original, its Exif capture time the frame's, at most - /// ) as the frame's proxy; the sequence's thumbnails and proxy state follow as - /// background jobs. 204; 413 too_large, 422 invalid_proxy or proxy_mismatch, 409 - /// proxies_disabled, 503 archive_offline. + /// Files the body (a JPEG made from the original, its Exif capture time the frame's, as large as the archive's proxies, + /// at most ) as the proxy of a frame the server cannot decode + /// (); the sequence's thumbnails and proxy state follow as background jobs. 204; 413 + /// too_large; 422 proxy_not_wanted (the server decodes that original itself, or it is missing or + /// damaged), invalid_proxy, proxy_mismatch or proxy_too_small, which concern that proxy only; 409 + /// proxies_disabled, which concerns them all; 503 archive_offline. /// private static async Task StoreProxyAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive, [FromServices] ArchiveService? archives, SequenceJobs follow) @@ -69,6 +71,9 @@ public static class FileEndpoints var ct = context.RequestAborted; if (await db.Frames.GetAsync(id, ct) is not { } frame || await db.Sequences.GetAsync(frame.SequenceId, ct) is not { } sequence) return ApiErrors.NotFound(context, "frame"); + // A proxy replaces the server's own of an original it decodes, which renders and remote views would then show. + if (!ServerProxies.CannotDecode(frame.Format) || (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0) + return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_not_wanted"); if (context.Request.ContentLength is > ArchiveImages.MaxProxyBytes) return ApiErrors.Result(context, StatusCodes.Status413PayloadTooLarge, "too_large"); // Proxies are larger than the JSON bodies Kestrel is limited to; reading past this answers 413. if (context.Features.Get() is { IsReadOnly: false } limit) limit.MaxRequestBodySize = ArchiveImages.MaxProxyBytes; @@ -93,8 +98,11 @@ public static class FileEndpoints return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "invalid_proxy"); case ProxyStoreResult.WrongFrame: return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_mismatch"); + case ProxyStoreResult.TooSmall: + return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_too_small"); } - await follow.ProxyArrivedAsync(sequence, ct); + // The proxy is filed: its follow-up is queued even when the sender is gone by now. + await follow.ProxyArrivedAsync(sequence, CancellationToken.None); return Results.NoContent(); } diff --git a/src/Titano.Server/Storage/ServerProxies.cs b/src/Titano.Server/Storage/ServerProxies.cs index fd70e4e5..34e73cc3 100644 --- a/src/Titano.Server/Storage/ServerProxies.cs +++ b/src/Titano.Server/Storage/ServerProxies.cs @@ -1,6 +1,5 @@ using Titano.Core.Archiving; using Titano.Core.Imaging; -using Titano.Core.Imaging.Decoders; using Titano.Core.Model; using Titano.Data; using Titano.Data.Archiving; @@ -17,11 +16,8 @@ public static class ServerProxies { private const int Page = 1000; - /// The server's decoders (BRIEF §4.2: managed only). - private static readonly ImageDecoder Decoder = DecoderFactory.CreateServer(); - - /// The server takes this format's pixels from proxies made elsewhere. - public static bool CannotDecode(ImageFileFormat format) => ArchiveImages.NeedsProxy(Decoder, format); + /// The server takes this format's pixels from proxies made elsewhere (its decoders are managed only, BRIEF §4.2). + public static bool CannotDecode(ImageFileFormat format) => ArchiveImages.ServerNeedsProxy(format); /// How proxies of the archive are made; null when it makes none (or its manifest cannot be read now). public static ProxySpec? Spec(Archive archive) diff --git a/tests/Titano.Data.Tests/Archiving/ProxyTests.cs b/tests/Titano.Data.Tests/Archiving/ProxyTests.cs index f7b86b17..0fc493fa 100644 --- a/tests/Titano.Data.Tests/Archiving/ProxyTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ProxyTests.cs @@ -78,6 +78,34 @@ public sealed class ProxyTests(TestContext context) Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt)); } + [Test] + public async Task AProxyMadeFromAPreviewIsAskedForAgain() + { + await using var fx = await CreateAsync(context); + var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024); + var folder = fx.Paths.Sequence(sequence.Folder); + var images = fx.Service.Images; + // An earlier version made the frame's proxy here from the original's embedded preview: the frame's capture time, + // a thumbnail's size. + Directory.CreateDirectory(folder.Proxies); + await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 5, 160, 120), fx.Ct); + await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, SequenceStatus.ProxiesBuilt, SequenceStatus.None, fx.Ct); + + var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + + Assert.Equal((0, 0, 2), (proxies.Built, proxies.Reused, proxies.Undecodable)); + Assert.FileMissing(folder.Proxy(0)); + Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt), "asked for again"); + + // A client or render node sending one that small is refused; one of the policy's size (1920 px) is kept. + var frame = frames[0] with { Width = 4032, Height = 3024 }; + Assert.Equal(ProxyStoreResult.TooSmall, images.StoreProxy(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 6, 160, 120))); + Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 7, 1920, 1440))); + var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable)); + Assert.FileExists(folder.Proxy(0)); + } + [Test] public async Task AnArchiveWithoutProxiesRefusesThem() { @@ -90,8 +118,11 @@ public sealed class ProxyTests(TestContext context) Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).Proxy(0)); } - /// Two ingested frames whose originals the archive engine takes for HEIF (its decoders read none). - private static async Task<(Sequence Sequence, IReadOnlyList Frames)> HeifSequenceAsync(ArchiveFixture fx) + /// + /// Two ingested frames whose originals the archive engine takes for HEIF (its decoders read none), their size in the + /// sidecar × when given. + /// + private static async Task<(Sequence Sequence, IReadOnlyList Frames)> HeifSequenceAsync(ArchiveFixture fx, int? width = null, int? height = null) { Guid id = default; for (var i = 0; i < 2; i++) @@ -103,14 +134,17 @@ public sealed class ProxyTests(TestContext context) var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!; var folder = fx.Paths.Sequence(sequence.Folder); var sidecar = ManifestStore.ReadSequence(folder.ManifestPath); - ManifestStore.WriteSequence(folder.ManifestPath, sidecar with { Frames = [.. sidecar.Frames.Select(f => f with { Format = ImageFileFormat.Heif })] }); + ManifestStore.WriteSequence(folder.ManifestPath, sidecar with + { + Frames = [.. sidecar.Frames.Select(f => f with { Format = ImageFileFormat.Heif, W = width ?? f.W, H = height ?? f.H })], + }); return (sequence, await fx.Db.Frames.ListBySequenceAsync(id)); } - /// A 320 × 240 JPEG proxy with in its Exif, as a client makes it. - private static byte[] Proxy(DateTime captured, int seed) + /// A JPEG proxy (320 × 240 unless sized) with in its Exif, as a client makes it. + private static byte[] Proxy(DateTime captured, int seed, int width = 320, int height = 240) { - using var image = new ImageBuffer(320, 240); + using var image = new ImageBuffer(width, height); var level = 0.2f + seed % 5 * 0.1f; for (var c = 0; c < 3; c++) image.Plane(c).Fill(level + c * 0.05f); var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Titano" }); diff --git a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs index 5b35d8a7..6f417329 100644 --- a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs +++ b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs @@ -69,9 +69,11 @@ public sealed class RenderNodeClientTests(TestContext context) var sequenceId = await UploadSequenceAsync(editor, Photos()[..3]); await Uploads.UploadAsync(editor, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, 9)], "IMG_0004.HEIC", sequenceId); var frames = await server.Database.Frames.ListBySequenceAsync(sequenceId); - // Frame 1 is gone from the server, frame 2 was adopted and never hashed, frame 4 is a HEIF the server cannot decode. + // Frame 1 is gone from the server, frame 2 was adopted and never hashed, frame 3 is damaged (the server could not + // decode it) and frame 4 is a HEIF the server cannot decode. File.Delete(Path.Combine(server.Options.ArchivePath, frames[0].RelativePath)); await server.Database.Frames.UpdateFlagsAsync([frames[0].Id], FrameFlags.Missing, FrameFlags.None); + await server.Database.Frames.UpdateFlagsAsync([frames[2].Id, frames[3].Id], FrameFlags.Undecodable, FrameFlags.None); await using (var session = await server.Database.Database.OpenAsync(context.Cancellation)) { await session.ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE id = $1", [frames[1].Id], context.Cancellation); @@ -89,6 +91,9 @@ public sealed class RenderNodeClientTests(TestContext context) Assert.Equal("done", done.State, done.Error); Assert.DoesNotContain($"/api/v1/frames/{frames[0].Id}/original", requests.Paths); Assert.Contains($"/api/v1/frames/{frames[1].Id}/original", requests.Paths); + // The damaged frame stays out, as in the server's own render; the HEIF renders from its original here. + Assert.DoesNotContain($"/api/v1/frames/{frames[2].Id}/original", requests.Paths); + Assert.Contains($"/api/v1/frames/{frames[3].Id}/original", requests.Paths); var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!; var proxy = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder).Proxy(frames[3].Index); Assert.Equal(frames[3].CapturedAt, MetadataReader.Read(proxy).CapturedAt, "the proxy the server asked for, with the frame's capture time"); diff --git a/tests/Titano.Server.Tests/Storage/ProxyUploadTests.cs b/tests/Titano.Server.Tests/Storage/ProxyUploadTests.cs index 688343b9..2e0811ba 100644 --- a/tests/Titano.Server.Tests/Storage/ProxyUploadTests.cs +++ b/tests/Titano.Server.Tests/Storage/ProxyUploadTests.cs @@ -70,6 +70,9 @@ public sealed class ProxyUploadTests(TestContext context) var upload = await Uploads.UploadAsync(editor, Proxy(new DateTime(2026, 9, 28, 18, 2, 11)), "IMG_0001.JPG", sessionKey: "roof"); Assert.False((await ManifestAsync(editor, upload.SequenceId)).Frames.Single().WantsProxy); + // Nor takes one: renders and remote views keep the proxy the server makes itself. + using var refused = await PutAsync(editor, upload.FrameId, Proxy(new DateTime(2026, 9, 28, 18, 2, 11))); + await refused.ExpectErrorAsync(HttpStatusCode.UnprocessableEntity, "proxy_not_wanted"); } /// A HEIF file as far as signature sniffing goes (an ftyp box naming heic), with no pixels the server can read. From 17311cc459b1da33686662c317064b2c1fb5a836 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 11:06:53 +0000 Subject: [PATCH 7/8] fix(sync): send wanted proxies of unverified originals and recount them Wanted proxies were matched to this PC's originals by SHA-256 only, and frames adopted from a folder stay unhashed until verified. A desktop that uploaded an adopted HEIC folder counted none of the proxies the server asked for, showed the link in sync and sent none of them. The upload now keeps the hash it computes for each original the index has none for and makes the proxies from those too; the comparison counts a wanted frame no hash matched against an unhashed original this PC reads by size (the server names its copy by its own template, so not by name). The wanted count was recomputed only when frame or render counters moved, so proxies a render node sent left the row "Local ahead" with an upload that sent nothing, and proxies the server started asking for later were never counted. A link whose copy may make proxies for the server now keeps the server's update time of the last comparison (sync_links.diff_remote_updated_at, migration 0006) and is compared again when it moved. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../postgres/0006_sync_proxy_compare.sql | 5 ++ .../sqlite/0006_sync_proxy_compare.sql | 5 ++ src/Titano.Data/Sync/SyncLinkStore.cs | 6 +- src/Titano.Data/Sync/SyncModel.cs | 8 +++ src/Titano.Data/Sync/SyncRules.cs | 33 ++++++++-- src/Titano.Data/Sync/SyncService.cs | 15 ++++- src/Titano.Data/Sync/SyncTransfers.cs | 31 +++++++--- .../Titano.Data.Tests/Sync/SyncRulesTests.cs | 60 +++++++++++++++++++ .../Sync/SyncTransferTests.cs | 53 ++++++++++++++++ 9 files changed, 198 insertions(+), 18 deletions(-) create mode 100644 src/Titano.Data/Migrations/postgres/0006_sync_proxy_compare.sql create mode 100644 src/Titano.Data/Migrations/sqlite/0006_sync_proxy_compare.sql diff --git a/src/Titano.Data/Migrations/postgres/0006_sync_proxy_compare.sql b/src/Titano.Data/Migrations/postgres/0006_sync_proxy_compare.sql new file mode 100644 index 00000000..63764a45 --- /dev/null +++ b/src/Titano.Data/Migrations/postgres/0006_sync_proxy_compare.sql @@ -0,0 +1,5 @@ +-- Client sync state (BRIEF §2.3, §14.5): the server's update time a linked copy's originals were compared at, kept +-- while proxies the server asks for may be made on this PC; they change without the frame counters moving, so the +-- copy is compared again when the server's sequence changed since. Mirrors sqlite/0006_sync_proxy_compare.sql. + +ALTER TABLE sync_links ADD COLUMN diff_remote_updated_at timestamptz; diff --git a/src/Titano.Data/Migrations/sqlite/0006_sync_proxy_compare.sql b/src/Titano.Data/Migrations/sqlite/0006_sync_proxy_compare.sql new file mode 100644 index 00000000..6088e40b --- /dev/null +++ b/src/Titano.Data/Migrations/sqlite/0006_sync_proxy_compare.sql @@ -0,0 +1,5 @@ +-- Client sync state (BRIEF §2.3, §14.5): the server's update time a linked copy's originals were compared at, kept +-- while proxies the server asks for may be made on this PC; they change without the frame counters moving, so the +-- copy is compared again when the server's sequence changed since. Mirrors postgres/0006_sync_proxy_compare.sql. + +ALTER TABLE sync_links ADD COLUMN diff_remote_updated_at TEXT; diff --git a/src/Titano.Data/Sync/SyncLinkStore.cs b/src/Titano.Data/Sync/SyncLinkStore.cs index 1d28e3f4..9c7f0c75 100644 --- a/src/Titano.Data/Sync/SyncLinkStore.cs +++ b/src/Titano.Data/Sync/SyncLinkStore.cs @@ -11,7 +11,8 @@ internal sealed class SyncLinkStore(TitanoDatabase db) "server, remote_id, local_id, title, frame_count, total_bytes, render_count, remote_status, remote_settings_version, " + "remote_frames_version, remote_updated_at, base_local_settings, base_remote_settings, diff_local_frames, diff_remote_frames, " + "diff_local_renders, diff_remote_renders, missing_remote, missing_local, exports_to_get, exports_to_get_bytes, exports_to_send, " + - "exports_to_send_bytes, conflict_local_settings, conflict_remote_settings, conflict_remote, proxies_cached, proxies_frames_version, error, synced_at"; + "exports_to_send_bytes, conflict_local_settings, conflict_remote_settings, conflict_remote, proxies_cached, proxies_frames_version, error, synced_at, " + + "diff_remote_updated_at"; private static readonly string UpsertSql = SqlDialect.Upsert("sync_links", Columns.Split(", "), 2); @@ -36,7 +37,7 @@ internal sealed class SyncLinkStore(TitanoDatabase db) l.DiffLocalRenders, l.DiffRemoteRenders, l.MissingRemote, l.MissingLocal, l.ExportsToGet, l.ExportsToGetBytes, l.ExportsToSend, l.ExportsToSendBytes, l.ConflictLocalSettings, l.ConflictRemoteSettings, l.ConflictRemote is { } remote ? JsonSerializer.Serialize(remote, SyncPayloadJson.Default.SyncPayload) : null, - l.ProxiesCached, l.ProxiesFramesVersion, l.Error, l.SyncedAt, + l.ProxiesCached, l.ProxiesFramesVersion, l.Error, l.SyncedAt, l.DiffRemoteUpdatedAt, ], cancellationToken); public Task DeleteAsync(string server, Guid remoteId, CancellationToken cancellationToken) => @@ -103,5 +104,6 @@ internal sealed class SyncLinkStore(TitanoDatabase db) ProxiesFramesVersion = r.GetInt32OrNull(27), Error = r.GetStringOrNull(28), SyncedAt = r.GetDateTimeOffsetOrNull(29), + DiffRemoteUpdatedAt = r.GetDateTimeOffsetOrNull(30), }; } diff --git a/src/Titano.Data/Sync/SyncModel.cs b/src/Titano.Data/Sync/SyncModel.cs index eb842ff5..a379a54d 100644 --- a/src/Titano.Data/Sync/SyncModel.cs +++ b/src/Titano.Data/Sync/SyncModel.cs @@ -91,6 +91,14 @@ public sealed record SyncLink public int? DiffLocalRenders { get; init; } public int? DiffRemoteRenders { get; init; } + /// + /// The server's update time the originals were compared at, kept while proxies it asks for may be made here (some + /// were counted, or the copy holds originals the server cannot decode and this PC can); null otherwise. The proxies + /// it asks for change without its frame counters moving (one arrives from a render node, its proxy policy changes), + /// and every such change leaves a new update time, so the originals are compared again when it moved. + /// + public DateTimeOffset? DiffRemoteUpdatedAt { get; init; } + /// /// Local originals the server does not have, and proxies it asks for that this PC can make (it cannot decode those /// originals): an upload sends both. diff --git a/src/Titano.Data/Sync/SyncRules.cs b/src/Titano.Data/Sync/SyncRules.cs index ccda9f46..e0852bce 100644 --- a/src/Titano.Data/Sync/SyncRules.cs +++ b/src/Titano.Data/Sync/SyncRules.cs @@ -134,16 +134,41 @@ public static class SyncRules /// /// Frames whose proxy the server asks for (it cannot decode their originals, BRIEF §2.3) that this PC holds and its - /// reads: an upload makes and sends them. + /// reads: an upload makes and sends them. A local original not hashed yet (a folder adopted + /// and not verified) stands for a wanted frame of its size that no hash matched (by size, not name: the server names + /// its copy by its own template); the upload hashes it before making the proxy. /// public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection local, ImageDecoder decoder) { ArgumentNullException.ThrowIfNull(manifest); - var readable = local.Where(f => !string.IsNullOrEmpty(f.Sha256) && !ArchiveImages.NeedsProxy(decoder, f.Format)) - .Select(f => f.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); - return manifest.Frames.Count(f => f.WantsProxy && readable.Contains(f.Sha256.ToLowerInvariant())); + var hashed = local.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal); + var unhashed = local.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f)).GroupBy(f => f.Size).ToDictionary(g => g.Key, g => g.Count()); + var count = 0; + foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length > 0)) + { + var sha = frame.Sha256.ToLowerInvariant(); + if (hashed.Contains(sha)) + { + if (hashed[sha].Any(Readable)) count++; + } + else if (unhashed.TryGetValue(frame.Size, out var left) && left > 0) + { + unhashed[frame.Size] = left - 1; + count++; + } + } + return count; + + bool Readable(Frame frame) => !ArchiveImages.NeedsProxy(decoder, frame.Format); } + /// + /// Whether holds originals a server cannot decode (it asks for their proxies, BRIEF §2.3, + /// §4.2: managed decoders only) that this PC's reads, so it can make those proxies. + /// + public static bool MakesProxiesFor(IEnumerable local, ImageDecoder decoder) => + local.Select(f => f.Format).Distinct().Any(format => ArchiveImages.ServerNeedsProxy(format) && !ArchiveImages.NeedsProxy(decoder, format)); + internal static string Blank(string? text) => text?.Trim() ?? string.Empty; private static JsonNode? Parse(string json) diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index a1e655f7..cc5e6071 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -490,7 +490,8 @@ public sealed class SyncService } SequenceManifest? manifest = null; if (link.DiffLocalFrames != local.FramesVersion || link.DiffRemoteFrames != link.RemoteFramesVersion - || link.DiffLocalRenders != local.RenderedCount || link.DiffRemoteRenders != link.RenderCount) + || link.DiffLocalRenders != local.RenderedCount || link.DiffRemoteRenders != link.RenderCount + || (link.DiffRemoteUpdatedAt is { } compared && !SameMoment(compared, link.RemoteUpdatedAt))) { manifest = await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false); @@ -498,7 +499,10 @@ public sealed class SyncService var present = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList(); var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]); // Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3). - if (policy.CanEdit && manifest.Proxies is not null) missingRemote += SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder); + var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder) : 0; + // Those change without the server's frame counters moving: while some may be made here, the server's next + // change of the sequence brings a new comparison. + var watch = policy.CanEdit && (proxies > 0 || SyncRules.MakesProxiesFor(present, Options.Archive.Decoder)); link = await CompareExportsAsync(link, local, manifest.Renders, cancellationToken).ConfigureAwait(false) with { DiffLocalFrames = local.FramesVersion, @@ -506,7 +510,8 @@ public sealed class SyncService RemoteFramesVersion = manifest.FramesVersion, DiffLocalRenders = local.RenderedCount, DiffRemoteRenders = manifest.Renders.Count, - MissingRemote = missingRemote, + DiffRemoteUpdatedAt = watch ? link.RemoteUpdatedAt : null, + MissingRemote = missingRemote + proxies, MissingLocal = missingLocal, }; } @@ -639,6 +644,9 @@ public sealed class SyncService private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified); + /// The same instant as databases keep it (to the microsecond). + private static bool SameMoment(DateTimeOffset a, DateTimeOffset b) => a.UtcTicks / 10 == b.UtcTicks / 10; + /// /// Writes a payload into the local index, project and sidecar in one step (settings version + 1) under the sequence's /// lease, which the editor's saves take too, and only while the local copy is still at the settings version @@ -871,6 +879,7 @@ public sealed class SyncService DiffRemoteFrames = null, DiffLocalRenders = null, DiffRemoteRenders = null, + DiffRemoteUpdatedAt = null, MissingRemote = 0, MissingLocal = 0, ExportsToGet = 0, diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs index 95e42f10..ae361dbc 100644 --- a/src/Titano.Data/Sync/SyncTransfers.cs +++ b/src/Titano.Data/Sync/SyncTransfers.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Globalization; using System.Net; using Titano.Core.Archiving; @@ -50,12 +51,16 @@ internal sealed class SyncTransfers(SyncService sync) var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count); await progress.ReportAsync().ConfigureAwait(false); var skipped = 0; + // The SHA-256 of each original the index has none for (a folder adopted and not verified), as its upload computes + // it: the proxies the server asks for are matched to this PC's originals by content. Every such original is + // pending, so each one still here is hashed. + var hashes = new ConcurrentDictionary(); // Without a server sequence the originals go out one by one under this sequence's session key until the // server has filed one, which creates the sequence (an original it already holds elsewhere creates nothing). while (remoteId is null && pending.TryDequeue(out var first)) { - var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, ct) + var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, hashes, ct) .ConfigureAwait(false); if (sent is null) skipped++; else if (sent.SequenceId != Guid.Empty) @@ -70,11 +75,11 @@ internal sealed class SyncTransfers(SyncService sync) await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) => { - if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, token).ConfigureAwait(false) is null) + if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, hashes, token).ConfigureAwait(false) is null) Interlocked.Increment(ref skipped); await progress.StepAsync().ConfigureAwait(false); }).ConfigureAwait(false); - await SendProxiesAsync(context, remote, target, local, paths, frames).ConfigureAwait(false); + await SendProxiesAsync(context, remote, target, local, paths, frames, hashes).ConfigureAwait(false); if (created) await sync.PushAllAsync(remote, target, ct).ConfigureAwait(false); await sync.RecompareAsync(remote, target, ct).ConfigureAwait(false); @@ -83,17 +88,23 @@ internal sealed class SyncTransfers(SyncService sync) /// /// The proxies the server asks for () that this PC's decoders can make from its - /// originals, made as the server's proxy policy says with the server's capture data, and sent. One the server refuses - /// is counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a + /// originals (by their archived SHA-256, or the one holds for an original the index has none + /// for), made as the server's proxy policy says with the server's capture data, and sent. One the server refuses is + /// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a /// server busy or out of reach fails the job like an original would. /// - private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList frames) + private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList frames, + IReadOnlyDictionary hashes) { var ct = context.Cancellation; var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); var wanted = manifest.Frames.Where(f => f.WantsProxy).ToList(); if (manifest.Proxies is not { } spec || wanted.Count == 0) return; - var originals = frames.Where(f => f.Sha256 is not null).GroupBy(f => f.Sha256!.ToLowerInvariant()).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + var originals = new Dictionary(StringComparer.Ordinal); + foreach (var frame in frames) + { + if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame); + } var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id); var camera = new SidecarCamera { Make = local.CameraMake, Model = local.CameraModel, Serial = local.CameraSerial }; var decoder = sync.Options.Archive.Decoder; @@ -443,10 +454,11 @@ internal sealed class SyncTransfers(SyncService sync) /// /// Hashes the original again and sends it into (or the sequence of - /// ); null (logged) when it is gone or no longer the archived file. + /// ); null (logged) when it is gone or no longer the archived file. The hash of an + /// original the index has none for goes into . /// private static async Task UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path, - Frame frame, CancellationToken cancellationToken) + Frame frame, ConcurrentDictionary hashes, CancellationToken cancellationToken) { if (!File.Exists(path)) { @@ -454,6 +466,7 @@ internal sealed class SyncTransfers(SyncService sync) return null; } var sha = await TitanoClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false); + if (frame.Sha256 is null) hashes[frame.Id] = sha; if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase)) { context.Log($"{frame.FileName}: changed since it was archived, not uploaded"); diff --git a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs index f2b7fa91..da763383 100644 --- a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs @@ -1,5 +1,8 @@ +using Titano.Core.Imaging; +using Titano.Core.Imaging.Decoders; using Titano.Core.Model; using Titano.Data.Sync; +using Titano.Protocol; using Titano.Testing; namespace Titano.Data.Tests.Sync; @@ -63,6 +66,38 @@ public sealed class SyncRulesTests Assert.Equal((1, 0), SyncRules.CompareOriginals([null, null], ["bb"])); } + [Test] + public void ProxiesTheServerAsksForAreCountedForOriginalsThisPcReads() + { + var heif = new ImageDecoder([new NativeHeif()]); + List local = + [ + Original("aa", 100, ImageFileFormat.Heif), + Original(null, 200, ImageFileFormat.Heif), + Original(null, 300, ImageFileFormat.Heif), + Original("dd", 400, ImageFileFormat.Avif), + ]; + var manifest = new SequenceManifest(Guid.CreateVersion7(), 1, 1, null, [], + [ + Wanted("AA", 100, "2026-10-01_0001.heic"), + // Not hashed here: matched by size, whatever name the server gave its copy. + Wanted("bb", 200, "2026-10-01_0002.heic"), + Wanted("cc", 999, "IMG_0003.HEIC"), + // The server has no hash of it yet: nothing tells which original it is. + Wanted(string.Empty, 300, "IMG_0004.HEIC"), + // This PC cannot decode AVIF. + Wanted("dd", 400, "IMG_0005.AVIF"), + Wanted("ee", 200, "IMG_0006.HEIC") with { WantsProxy = false }, + ], [], new ProxySpec(1920, 90)); + + Assert.Equal(2, SyncRules.ProxiesToSend(manifest, local, heif)); + Assert.Equal(0, SyncRules.ProxiesToSend(manifest, local, DecoderFactory.CreateServer())); + + Assert.True(SyncRules.MakesProxiesFor(local, heif), "HEIF the server cannot decode and this PC can"); + Assert.False(SyncRules.MakesProxiesFor(local, DecoderFactory.CreateServer()), "a PC that reads no more than the server"); + Assert.False(SyncRules.MakesProxiesFor([Original("aa", 100, ImageFileFormat.Jpeg)], heif), "the server decodes JPEG itself"); + } + [Test] public void SettingsAreComparedAsProjectsNotAsText() { @@ -92,4 +127,29 @@ public sealed class SyncRulesTests Assert.True(mine.SameMetadata(mine with { Tags = ["HARBOUR", "sunset"], Notes = "mine " }), "tags ignore case and order, blanks are trimmed"); Assert.Empty(SyncRules.Differences(mine, mine with { Tags = ["harbour", "SUNSET"] })); } + + private static Frame Original(string? sha, long size, ImageFileFormat format) => new() + { + Id = Guid.CreateVersion7(), + SequenceId = Local.Id, + Index = (int)(size / 100), + RelativePath = $"2026/a/{size}", + FileName = $"{size}", + Size = size, + Sha256 = sha, + Format = format, + }; + + private static ManifestFrame Wanted(string sha, long size, string name) => + new(Guid.CreateVersion7(), (int)(size / 100), name, size, sha, HasProxy: false, HasThumb: true, WantsProxy: true); + + /// A provider that decodes HEIF natively (WIC with the HEIF extension on a desktop). + private sealed class NativeHeif : IImageDecoderProvider + { + public string Name => "native-heif"; + + public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null; + + public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null; + } } diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs index 129fd092..174aa73e 100644 --- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs @@ -62,6 +62,59 @@ public sealed class SyncTransferTests(TestContext context) Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status); } + [Test] + public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + // An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds + // under the names it gave them, and cannot decode (HEIF, BRIEF §2.3). + var folder = Path.Combine(f.Archive.Paths.Root, "Adopted"); + Directory.CreateDirectory(folder); + for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct); + await f.Archive.ScanAsync(); + var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Titano.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted"); + Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed"); + var remote = f.Remote.Add("Adopted", photos); + remote.Undecodable = true; + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var ahead = await f.EntryAsync(local.DisplayTitle); + Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); + + Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); + Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status); + } + + [Test] + public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Holy grail", photos); + var remote = f.Remote.Add("Holy grail", photos); + remote.Undecodable = true; + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote); + var calls = f.Remote.ManifestCalls; + await f.RefreshAsync(); + Assert.Equal(calls, f.Remote.ManifestCalls); + + // A render node sends them after a job: the server's frame counters stand still, its sequence changes. + foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy; + remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1); + await f.RefreshAsync(); + + var entry = await f.EntryAsync("Holy grail"); + Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote)); + } + [Test] public async Task AResumedUploadSendsOnlyWhatTheServerLacks() { From e3719dc1586be7c72e9fc0497dadb3722d0e7d27 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 11:07:02 +0000 Subject: [PATCH 8/8] fix(archive): lock the sequences a renamed folder carries A mass rename moved the folders of nested sequences while holding only the lease of the sequence whose folder moved, so an ingest, an editor save or a thumbnail pass of a nested sequence kept writing in the folder as it moved: on Windows the move failed and left the rename pending; elsewhere the writer recreated the old folder and the nested step then failed. A moving step now takes the sequence lease, then the thumbnail lease, of itself and of every sequence it carries, by id, checks them all before anything moves, and records the carried sequences' sidecars and index rows before letting go, so they are journaled together. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/MassRenamer.cs | 117 +++++++++++++----- .../Archiving/MassRenameTests.cs | 24 ++++ 2 files changed, 108 insertions(+), 33 deletions(-) diff --git a/src/Titano.Data/Archiving/MassRenamer.cs b/src/Titano.Data/Archiving/MassRenamer.cs index ad46a81b..6846e70d 100644 --- a/src/Titano.Data/Archiving/MassRenamer.cs +++ b/src/Titano.Data/Archiving/MassRenamer.cs @@ -93,55 +93,106 @@ public sealed class MassRenamer } } RenameJournal.Write(paths, journal); - for (var i = 0; i < journal.Plan.Steps.Count; i++) + var steps = journal.Plan.Steps; + foreach (var step in steps) { - var step = journal.Plan.Steps[i]; if (completed.Contains(step.SequenceId)) continue; cancellationToken.ThrowIfCancellationRequested(); - try - { - await ApplyStepAsync(archive, paths, step, cancellationToken).ConfigureAwait(false); - } - catch (StaleRenamePlanException) when (resuming) - { - Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there"); - } + // A folder that moves carries the sequences inside it: they are locked, moved and recorded with it. + IReadOnlyList carried = step.MovesFolder && !step.Carried + ? [.. steps.Where(c => c.Carried && !completed.Contains(c.SequenceId) && Carrier(steps, c) == step)] + : []; + await ApplyStepAsync(archive, paths, step, carried, resuming, cancellationToken).ConfigureAwait(false); completed.Add(step.SequenceId); + foreach (var c in carried) completed.Add(c.SequenceId); RenameJournal.Write(paths, journal with { Completed = [.. completed] }); - progress?.Report(i + 1); + progress?.Report(completed.Count); } RenameJournal.Delete(paths, journal.Plan.Id); } - /// Folder, then sidecar title, then index; each part is idempotent so a resumed step can repeat it. - private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken) + /// The step whose folder move carries : the nearest moving folder around it (as planned). + private static RenameStep? Carrier(IReadOnlyList steps, RenameStep carried) => + steps.Where(m => !m.Carried && m.MovesFolder && RenamePlanner.IsInside(m.FromFolder, carried.FromFolder)).MaxBy(m => m.FromFolder.Length); + + /// + /// Applies a step, and the steps of the sequences its folder carries, under all their leases: sequence leases, then + /// for a move the thumbnail leases, each by sequence id (the order trash and pack compaction take them in). No + /// writer of a carried sequence (ingest, an editor save, a thumbnail pass) works in the folder while it moves or + /// before its index row follows. + /// + /// A step whose sequence the index no longer has where the plan says is dropped instead of + /// failing the rename: its sidecar still carries the id, so the next scan indexes it wherever it ended up. + private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, IReadOnlyList carried, bool resuming, + CancellationToken cancellationToken) { - await using (await _writer.LockAsync(paths, step.SequenceId, cancellationToken).ConfigureAwait(false)) + var ids = carried.Select(c => c.SequenceId).Append(step.SequenceId).Order().ToList(); + var locks = new List(ids.Count); + var leases = new List(ids.Count); + try { - var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); - if (row is null || (row.Folder != step.FromFolder && row.Folder != step.ToFolder)) throw new StaleRenamePlanException(step.SequenceId); - RenamePlanner.MoveFolder(paths, step); - var sequence = paths.Sequence(step.ToFolder); - var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId); - if (manifest.Title != step.ToTitle) + foreach (var id in ids) locks.Add(await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false)); + if (step.MovesFolder) { - manifest = manifest with + foreach (var id in ids) { - Title = step.ToTitle, - Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 }, - UpdatedAt = DateTimeOffset.UtcNow, - }; - ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + leases.Add(await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{id:N}", _writer.Options.Owner, _writer.Options.LeaseWait, + _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)); + } } - await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + // Checked before anything moves; a dropped step drops the sequences it would have carried, which stay inside it. + if (!await StandsAsync(step, resuming, cancellationToken).ConfigureAwait(false)) return; + var follow = new List(carried.Count); + foreach (var c in carried) { - var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); - await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); - if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false); - await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false); - await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + if (await StandsAsync(c, resuming, cancellationToken).ConfigureAwait(false)) follow.Add(c); } - await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false); + RenamePlanner.MoveFolder(paths, step); + await RecordAsync(archive, paths, step, cancellationToken).ConfigureAwait(false); + // Moved with their carrier: only their sidecars and index rows follow. + foreach (var c in follow) await RecordAsync(archive, paths, c, cancellationToken).ConfigureAwait(false); + } + finally + { + for (var i = leases.Count - 1; i >= 0; i--) leases[i].Dispose(); + for (var i = locks.Count - 1; i >= 0; i--) await locks[i].DisposeAsync().ConfigureAwait(false); } } + + /// The index still has the step's sequence where the plan says (or where it already took it); otherwise stale. + /// Not resuming, and the sequence moved or vanished since the plan was made. + private async Task StandsAsync(RenameStep step, bool resuming, CancellationToken cancellationToken) + { + var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); + if (row is not null && (row.Folder == step.FromFolder || row.Folder == step.ToFolder)) return true; + if (!resuming) throw new StaleRenamePlanException(step.SequenceId); + Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there"); + return false; + } + + /// Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it. + private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken) + { + var sequence = paths.Sequence(step.ToFolder); + var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId); + if (manifest.Title != step.ToTitle) + { + manifest = manifest with + { + Title = step.ToTitle, + Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }; + ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + } + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false); + await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false); + } } diff --git a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs index c54ca153..d15b5036 100644 --- a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs +++ b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs @@ -136,6 +136,30 @@ public sealed class MassRenameTests(TestContext context) Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).MissingSequences); } + [Test] + public async Task AFolderMovesOnceTheWritersOfTheSequencesInsideAreDone() + { + await using var fx = await CreateAsync(context); + var ids = await NestedAsync(fx); + var before = await StateAsync(fx, ids); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "moved/{date}", null, fx.Ct); + + // An ingest into trips/day2 holds that sequence's lease; a thumbnail pass of trips/day2/dawn holds its pack's. + Task apply; + using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{ids[1]:N}", "ingest", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) + using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{ids[2]:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) + { + apply = fx.Service.ApplyRenameAsync(plan, null, fx.Ct); + await Task.Delay(500, fx.Ct); + Assert.False(apply.IsCompleted); + Assert.True(Directory.Exists(Path.Combine(fx.Paths.Root, "trips", "day2", "dawn")), "nothing moved while they write"); + } + + await apply; + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + } + [Test] public async Task NestedSequencesMovedTogetherLeaveTheirFolderFirst() {