diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index cdc3820a..b07fdc62 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -81,6 +81,7 @@ The copy of {0} does not match the original.long: error sentence {0} could not be added to the sequence.long: error sentence {0} is already in the library.long: error sentence + {0} would be inside the sequence folder {1}.long: error sentence {0} was written by a newer Titano (format {1}).long: error sentence The archive {0} is offline.long: error sentence The archive {0} is read-only.long: error sentence @@ -133,6 +134,7 @@ Index compacted: {0} freedlong: job log line {0} stale locks, {1} temporary files, {2} packs compacted, {3} duplicate flags changedlong: job log line {0} unindexed photos, {1} missing photoslong: job log line + Proxies for the server: {0} sent, {1} not sentlong: job log line; {0} proxies sent; {1} proxies this PC could not make or the server refused {0} folders, {1} indexed, {2} adopted, {3} removedlong: job log line Sequence not found: {0}long: job log line {0} photos checked, {1} intact, {2} hashed, {3} with problemslong: job log line diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index d8fe6c8a..441e327a 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -56,6 +56,7 @@ This job kind cannot be started here.long The job parameters are not valid.long The priority must be background, normal or user.long + The proxy is not a JPEG the server can read.long The role must be admin, editor, uploader or viewer.long The setting {0} is not valid.long The settings must be valid JSON.long @@ -70,6 +71,10 @@ That address is not a Titano Server.long Change your password first.long This page did not come through a trusted proxy over HTTPS. Open it through your proxy, or add the proxy's address to TITANO_TRUSTED_PROXIES.long + This archive makes no proxies.long + The proxy was made from another photo.long + The server does not need a proxy of this photo.long + The proxy is smaller than this archive's proxies.long This upload would exceed your quota.long Too many attempts; wait a minute and try again.long Choose a password different from the current one.long diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 8c2c51d1..c75b1e65 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -76,6 +76,7 @@ La copia di {0} non corrisponde all'originale. Impossibile aggiungere {0} alla sequenza. {0} è già nella libreria. + {0} finirebbe dentro la cartella della sequenza {1}. {0} è stato scritto da un Titano più recente (formato {1}). L'archivio {0} non è raggiungibile. L'archivio {0} è di sola lettura. @@ -128,6 +129,7 @@ Indice compattato: {0} liberati {0} blocchi scaduti, {1} file temporanei, {2} pacchetti compattati, {3} segni di duplicato cambiati {0} foto non indicizzate, {1} foto mancanti + Proxy per il server: {0} inviati, {1} non inviati {0} cartelle, {1} indicizzate, {2} adottate, {3} rimosse Sequenza non trovata: {0} {0} foto controllate, {1} integre, {2} con impronta nuova, {3} con problemi diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index 32545f94..de2bba4d 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -54,6 +54,7 @@ Questo tipo di job non si può avviare qui. I parametri del job non sono validi. La priorità deve essere background, normal o user. + Il proxy non è un JPEG che il server sa leggere. Il ruolo deve essere admin, editor, uploader o viewer. L'impostazione {0} non è valida. Le impostazioni devono essere JSON valido. @@ -68,6 +69,10 @@ Questo indirizzo non è un Titano Server. Cambia prima la password. Questa pagina non è arrivata da un proxy attendibile in HTTPS. Aprila tramite il proxy o aggiungi l'indirizzo del proxy a TITANO_TRUSTED_PROXIES. + Questo archivio non crea proxy. + Il proxy è stato creato da un'altra foto. + Al server non serve un proxy di questa foto. + Il proxy è più piccolo dei proxy di questo archivio. Questo caricamento supererebbe la tua quota. Troppi tentativi; attendi un minuto e riprova. Scegli una password diversa da quella attuale. diff --git a/src/Titano.App/Archive/ArchiveFailures.cs b/src/Titano.App/Archive/ArchiveFailures.cs index b7013175..94565389 100644 --- a/src/Titano.App/Archive/ArchiveFailures.cs +++ b/src/Titano.App/Archive/ArchiveFailures.cs @@ -17,7 +17,8 @@ public static class ArchiveFailures ArchiveOfflineException offline => Loc.L(offline.Availability == ArchiveAvailability.ReadOnly ? "Archive.Error.ReadOnly" : "Archive.Error.Offline", Formatting.LeftToRight(offline.Root)), // Already one localized sentence each. - LeaseUnavailableException or NewerFormatException or StaleRenamePlanException or StaleDeletePlanException or SequenceInLibraryException => ex.Message, + LeaseUnavailableException or NewerFormatException or StaleRenamePlanException or NestedRenameException or StaleDeletePlanException + or SequenceInLibraryException => ex.Message, UnauthorizedAccessException => Loc.L("Archive.Failed.Access"), DatabaseException => Loc.L("Archive.Failed.Index"), ArgumentException => Loc.L("Archive.Failed.Gone"), diff --git a/src/Titano.App/Archive/MassOperations.cs b/src/Titano.App/Archive/MassOperations.cs index 4bcb2d3b..7be16553 100644 --- a/src/Titano.App/Archive/MassOperations.cs +++ b/src/Titano.App/Archive/MassOperations.cs @@ -41,8 +41,8 @@ public static class MassRename : new RenameRow(s.SequenceId, s.FromTitle ?? string.Empty, s.ToTitle ?? string.Empty)), ]; - /// "12 of 36 sequences change". - public static string Summary(RenamePlan plan, int setSize) => Loc.Plural("Archive.Rename.Summary", plan.Steps.Count, setSize); + /// "12 of 36 sequences change"; sequences that only move with a folder they are in are listed, not counted. + public static string Summary(RenamePlan plan, int setSize) => Loc.Plural("Archive.Rename.Summary", plan.Steps.Count(s => !s.Carried), setSize); } /// Adding and removing tags on a set of sequences (BRIEF §12.5 Retag). diff --git a/src/Titano.Core/Archiving/MassRename.cs b/src/Titano.Core/Archiving/MassRename.cs index caf423c9..c653b694 100644 --- a/src/Titano.Core/Archiving/MassRename.cs +++ b/src/Titano.Core/Archiving/MassRename.cs @@ -1,6 +1,7 @@ using System.Globalization; using System.Text.Json; using Titano.Core.IO; +using Titano.Core.Localization; using Titano.Core.Naming; namespace Titano.Core.Archiving; @@ -9,11 +10,20 @@ namespace Titano.Core.Archiving; public sealed record RenameSubject(Guid SequenceId, string Folder, string? Title, DateTime? FirstCapture, string? CameraMake, string? CameraModel, string? Lens); /// One sequence's change: its folder, its title, or both. -public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle) +/// The folder moves only because a folder it is inside moves (a sequence nested in another). +public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle, bool Carried = false) { public bool MovesFolder => !string.Equals(FromFolder, ToFolder, StringComparison.Ordinal); } +/// A planned folder would lie inside a sequence's folder (nesting one sequence in another); nothing is planned. +public sealed class NestedRenameException(string target, string folder) + : ArgumentException(Loc.L("Archive.Error.NestedTarget", target, folder)) +{ + public string Target { get; } = target; + public string Folder { get; } = folder; +} + /// The exact changes a mass rename will make (its dry run); applying it makes exactly these. public sealed record RenamePlan(Guid Id, Guid ArchiveId, IReadOnlyList Steps); @@ -50,16 +60,21 @@ public static class RenamePlanner /// Expands (where {title} is the current title) and then /// (where {title} is the new title) for every subject. A target folder /// that is any current sequence folder, exists on disk or was given to an earlier subject gets " (2)", " (3)"…. - /// Null templates leave that part unchanged; subjects without a change produce no step. + /// Null templates leave that part unchanged; subjects without a change produce no step. A folder that moves takes + /// the sequences inside it along: each gets a step after every move, so the index + /// follows its files; a selected sequence inside a moving folder that moves elsewhere leaves first. /// /// Every sequence folder of the archive (renamed or not). + /// The archive's other sequences inside a subject's folder, with their titles. + /// A target lies inside a sequence's folder, or inside another target. public static RenamePlan Plan(Guid archiveId, IReadOnlyList subjects, string? folderTemplate, string? titleTemplate, - IEnumerable occupied, Func existsOnDisk) + IEnumerable occupied, Func existsOnDisk, IReadOnlyList? nested = null) { var folder = folderTemplate is null ? null : PathTemplate.Parse(folderTemplate); var title = titleTemplate is null ? null : PathTemplate.Parse(titleTemplate); - var claimed = new HashSet(occupied, StringComparer.OrdinalIgnoreCase); - var steps = new List(); + var taken = occupied.ToList(); + var claimed = new HashSet(taken, StringComparer.OrdinalIgnoreCase); + var own = new List(subjects.Count); foreach (var s in subjects) { var context = new NamingContext @@ -80,11 +95,39 @@ public static class RenamePlanner newFolder = expanded.Equals(s.Folder, StringComparison.OrdinalIgnoreCase) ? expanded : Unique(expanded, claimed, existsOnDisk); claimed.Add(newFolder); } - if (newFolder != s.Folder || newTitle != s.Title) steps.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle)); + own.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle)); } - return new RenamePlan(Guid.CreateVersion7(), archiveId, steps); + + var moving = own.Where(s => s.MovesFolder).ToList(); + foreach (var step in moving) + { + // Into a sequence's folder (its own included) or another target: one sequence would end up inside another. + var host = taken.Concat(moving.Where(m => m != step).Select(m => m.ToFolder)).FirstOrDefault(f => IsInside(f, step.ToFolder)); + if (host is not null) throw new NestedRenameException(step.ToFolder, host); + } + var steps = new List<(RenameStep Step, int Order)>(); + foreach (var step in own.Concat((nested ?? []).Select(n => new RenameStep(n.SequenceId, n.Folder, n.Folder, n.Title, n.Title)))) + { + // The nearest folder around it that moves carries it; a sequence that moves elsewhere leaves before its folder does. + var carrier = step.MovesFolder ? null : moving.Where(m => IsInside(m.FromFolder, step.FromFolder)).MaxBy(m => m.FromFolder.Length); + if (carrier is not null) + steps.Add((step with { ToFolder = carrier.ToFolder + step.FromFolder[carrier.FromFolder.Length..], Carried = true }, 1)); + else if (step.MovesFolder) + steps.Add((step, -moving.Count(m => m != step && IsInside(m.FromFolder, step.FromFolder)))); + else if (step.FromTitle != step.ToTitle) + steps.Add((step, 0)); + } + // Stable: subjects keep their order, sequences inside moving folders move out first, carried ones come last. + return new RenamePlan(Guid.CreateVersion7(), archiveId, [.. steps.OrderBy(s => s.Order).Select(s => s.Step)]); } + /// + /// Whether lies inside (archive-relative, forward slashes); the + /// archive root, a sequence only when a folder was adopted in place, counts as containing nothing. + /// + public static bool IsInside(string folder, string path) => + folder.Length > 0 && path.Length > folder.Length + 1 && path[folder.Length] == '/' && path.StartsWith(folder, StringComparison.OrdinalIgnoreCase); + /// /// Moves the folder of a step; idempotent so a resumed plan can repeat it: an already moved folder is accepted, /// a case-only change goes through a temporary name (case-insensitive file systems). diff --git a/src/Titano.Core/Archiving/ThumbPack.cs b/src/Titano.Core/Archiving/ThumbPack.cs index 48bf29df..c2bf8eaa 100644 --- a/src/Titano.Core/Archiving/ThumbPack.cs +++ b/src/Titano.Core/Archiving/ThumbPack.cs @@ -279,7 +279,7 @@ public sealed class ThumbPackReader : IDisposable public void Dispose() => _stream?.Dispose(); } -/// Appends thumbnails to a pack; one writer per pack at a time (the sequence lease guarantees it). +/// Appends thumbnails to a pack; one writer per pack at a time (the thumbs-<sequence id> lease guarantees it). public sealed class ThumbPackWriter : IDisposable { private readonly FileStream _stream; diff --git a/src/Titano.Data/Archiving/ArchiveImages.cs b/src/Titano.Data/Archiving/ArchiveImages.cs index 408e766c..7844f230 100644 --- a/src/Titano.Data/Archiving/ArchiveImages.cs +++ b/src/Titano.Data/Archiving/ArchiveImages.cs @@ -1,5 +1,8 @@ using System.Diagnostics; using Titano.Core.Archiving; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Decoders; using Titano.Core.IO; using Titano.Core.Metadata; using Titano.Core.Model; @@ -9,15 +12,40 @@ namespace Titano.Data.Archiving; /// Frames processed by one thumbnail or proxy pass over a sequence. public sealed record ImagePassResult(int Built, int Reused, int Undecodable); +/// What became of a proxy sent by a client or render node (). +public enum ProxyStoreResult +{ + Stored, + + /// The archive's proxy policy makes no proxies. + Disabled, + + /// Not a JPEG that decodes, or larger than . + Invalid, + + /// Its Exif capture time is not the frame's: it was made from another photo. + WrongFrame, + + /// Smaller than the archive's proxies of that frame: made from an embedded preview, not the original. + TooSmall, +} + /// /// Thumbnails (thumbs.pack) and proxies (proxies/0001.jpg) of sequences (BRIEF §8.5). Both are /// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after -/// frames were renumbered the stale ones are rebuilt and the rest reused. +/// frames were renumbered the stale ones are rebuilt and the rest reused. A frame whose original this host cannot +/// decode () gets its proxy from a client or render node that can +/// (, BRIEF §2.3) and its thumbnail from that proxy. /// public sealed class ArchiveImages { + /// Largest proxy a client or render node may send: a full-size proxy of a large sensor at q92 fits. + public const long MaxProxyBytes = 64L << 20; + private const int CommitFrames = 200; + private const int ProbeLongEdge = 64; private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10); + private static readonly ImageDecoder ServerDecoders = DecoderFactory.CreateServer(); private readonly TitanoDatabase _db; private readonly SequenceWriter _writer; @@ -32,6 +60,91 @@ public sealed class ArchiveImages _renderer = new FrameImageRenderer(writer.Options.Decoder); } + /// + /// Whether a host with takes the pixels of a original from its + /// proxy: no decoder reads the format, or only previews of a HEIF, which embeds thumbnails at best (BRIEF §2.3, + /// §8.1: the server keeps HEIF originals and relies on proxies made by a Windows client or render node). Never for a + /// JPEG (proxies are JPEGs: a host that reads them reads it) or a file no format was recognised in. + /// + public static bool NeedsProxy(ImageDecoder decoder, ImageFileFormat format) + { + ArgumentNullException.ThrowIfNull(decoder); + if (format is ImageFileFormat.Unknown or ImageFileFormat.Jpeg) return false; + var tier = decoder.TierFor(format); + return tier == DecodeTier.Unavailable || (tier != DecodeTier.Native && ImageFormatInfo.Of(format).Family == ImageFamily.Heif); + } + + /// with this archive engine's decoders. + public bool NeedsProxy(ImageFileFormat format) => NeedsProxy(_writer.Options.Decoder, format); + + /// + /// on a Titano Server, whose decoders are managed only (BRIEF + /// §4.2): it asks clients and render nodes for the proxies of those originals. + /// + public static bool ServerNeedsProxy(ImageFileFormat format) => NeedsProxy(ServerDecoders, format); + + /// + /// A proxy of the original at for a server that cannot decode it (): + /// at (0 = full size) and , with 's capture + /// data (the server's record, which it checks) and in its Exif. Null when + /// cannot decode the original either. + /// + /// The original cannot be read. + public static byte[]? MakeProxy(ImageDecoder decoder, string path, Frame frame, SidecarCamera? camera, int longEdge, int quality, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(frame); + if (NeedsProxy(decoder, frame.Format)) return null; + // The original's folder stands in for a sequence folder: the renderer reads frames relative to one. + var folder = new ArchivePaths(Path.GetDirectoryName(Path.GetFullPath(path))!).Sequence(string.Empty); + var source = new SidecarFrame + { + Id = frame.Id, + I = frame.Index, + File = Path.GetFileName(path), + Size = frame.Size, + CapturedAt = frame.CapturedAt, + UtcOffset = frame.UtcOffsetMinutes, + Exposure = frame.ExposureTime, + FNumber = frame.FNumber, + Iso = frame.Iso, + Focal = frame.FocalLength, + Bias = frame.ExposureBias, + Orientation = frame.Orientation, + Format = frame.Format, + }; + var policy = new ProxyPolicy { LongEdge = longEdge, Quality = quality }; + return new FrameImageRenderer(decoder).Proxy(folder, source, camera, policy, cancellationToken)?.Jpeg; + } + + /// + /// Files a proxy made elsewhere (a client or render node whose decoders read the original) as the frame's proxy, + /// replacing the one there: a JPEG of at most that decodes, whose Exif capture time is the + /// frame's (proxies carry it, so the proxy counts as current and cannot belong to another photo) and whose size is + /// the policy's (), written atomically. + /// + /// The archive is offline or read-only. + public ProxyStoreResult StoreProxy(Archive archive, Sequence sequence, Frame frame, byte[] jpeg) + { + ArgumentNullException.ThrowIfNull(jpeg); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy; + if (policy.LongEdge == ProxyPolicy.Disabled) return ProxyStoreResult.Disabled; + if (jpeg.Length is 0 or > (int)MaxProxyBytes || !Decodes(jpeg)) return ProxyStoreResult.Invalid; + var info = JpegDecoder.ReadInfo(jpeg); + if (TooSmall(System.Math.Max(info.Width, info.Height), frame.Width, frame.Height, policy)) return ProxyStoreResult.TooSmall; + if (frame.CapturedAt is { } captured) + { + using var stream = new MemoryStream(jpeg, writable: false); + if (MetadataReader.Read(stream, "proxy.jpg").CapturedAt != captured) return ProxyStoreResult.WrongFrame; + } + var folder = paths.Sequence(sequence.Folder); + Directory.CreateDirectory(folder.Proxies); + AtomicFile.WriteAllBytes(folder.Proxy(frame.Index), jpeg); + return ProxyStoreResult.Stored; + } + /// Resume point inside the sequence (frames before it were done by an earlier run). /// Called with the next frame index whenever the pack has been committed. public async Task BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame, @@ -54,12 +167,22 @@ public sealed class ArchiveImages cancellationToken.ThrowIfCancellationRequested(); var frame = manifest.Frames[i]; var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size); - if (!force && pack.Entries.TryGetValue(frame.I, out var entry) && entry.SourceHash.AsSpan().SequenceEqual(fingerprint)) + var entry = pack.Entries.GetValueOrDefault(frame.I); + var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height); + var current = !force && entry is not null && entry.SourceHash.AsSpan().SequenceEqual(fingerprint); + // A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded + // preview, whose smaller thumbnail the proxy's replaces. + var fromProxy = false; + if (NeedsProxy(frame.Format) && !(current && edge >= FrameImageRenderer.ThumbnailLongEdge) + && ReadProxy(folder.Proxy(frame.I), frame) is { Check: ProxyCheck.Current or ProxyCheck.Unverifiable } proxy) + fromProxy = !current || edge < System.Math.Min(FrameImageRenderer.ThumbnailLongEdge, proxy.LongEdge); + if (current && !fromProxy) { reused++; continue; } - if (Render(() => _renderer.Thumbnail(folder, frame, policy, cancellationToken)) is { } image) + var source = fromProxy ? ProxySource(frame) : frame; + if (Render(() => _renderer.Thumbnail(folder, source, policy, cancellationToken)) is { } image) { pack.Append(frame.I, image.Width, image.Height, fingerprint, image.Jpeg); built++; @@ -98,12 +221,29 @@ public sealed class ArchiveImages cancellationToken.ThrowIfCancellationRequested(); var frame = manifest.Frames[i]; var path = folder.Proxy(frame.I); - if (!force && IsCurrent(path, frame)) + if (NeedsProxy(frame.Format)) + { + // Only a client or render node that decodes the original makes this proxy: a preview would make one the + // size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again, and so + // does one smaller than the policy's (made here from a preview by an earlier version). + var (check, longEdge) = ReadProxy(path, frame); + if (check is ProxyCheck.Current or ProxyCheck.Unverifiable && TooSmall(longEdge, frame.W, frame.H, policy)) check = ProxyCheck.Stale; + if (check is ProxyCheck.Current or ProxyCheck.Unverifiable) + { + reused++; + } + else + { + if (check == ProxyCheck.Stale) File.Delete(path); + undecodable++; + } + } + else if (!force && ReadProxy(path, frame).Check == ProxyCheck.Current) { reused++; continue; } - if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image) + else if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image) { AtomicFile.WriteAllBytes(path, image.Jpeg); built++; @@ -118,15 +258,73 @@ public sealed class ArchiveImages return new ImagePassResult(built, reused, undecodable); } - /// A proxy is current when its Exif capture time is the frame's (frames renumbered since make it stale). - private static bool IsCurrent(string proxy, SidecarFrame frame) + /// Where a frame's proxy stands; only a capture time tells whether it is the frame's. + private enum ProxyCheck { - if (!File.Exists(proxy)) return false; + Missing, + + /// Its Exif capture time is the frame's. + Current, + + /// The frame has no capture time to compare with. + Unverifiable, + + /// Made from another frame (frames were renumbered since). + Stale, + } + + /// Where the proxy at stands for , and its long edge. + private static (ProxyCheck Check, int LongEdge) ReadProxy(string proxy, SidecarFrame frame) + { + if (!File.Exists(proxy)) return (ProxyCheck.Missing, 0); try { - return MetadataReader.Read(proxy).CapturedAt == frame.CapturedAt && frame.CapturedAt is not null; + var metadata = MetadataReader.Read(proxy); + var (width, height) = metadata.OrientedSize; + var check = frame.CapturedAt is null ? ProxyCheck.Unverifiable : metadata.CapturedAt == frame.CapturedAt ? ProxyCheck.Current : ProxyCheck.Stale; + return (check, System.Math.Max(width, height)); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return (ProxyCheck.Missing, 0); + } + } + + /// + /// Whether a proxy whose long edge is falls short of the policy's for a frame of + /// × (the policy's long edge, or the frame's own when smaller or + /// at full size): one made from an embedded preview. A frame of unknown size is never judged (a small original's + /// proxy would be refused again and again); a tenth of slack spares decoders that crop a few pixels. + /// + private static bool TooSmall(int longEdge, int width, int height, ProxyPolicy policy) + { + var own = System.Math.Max(width, height); + if (own <= 0) return false; + var expected = policy.LongEdge == ProxyPolicy.FullSize ? own : System.Math.Min(own, policy.LongEdge); + return longEdge * 10L < expected * 9L; + } + + /// The frame's proxy as the frame to draw: a JPEG already oriented, under proxies/. + private static SidecarFrame ProxySource(SidecarFrame frame) => frame with + { + File = ArchiveLayout.ProxiesFolder + "/" + ArchiveLayout.ProxyFileName(frame.I), + Format = ImageFileFormat.Jpeg, + Orientation = 1, + PreviewOffset = null, + PreviewLength = null, + }; + + /// A colour or grey JPEG whose headers parse and whose scans decode (at a small DCT-scaled size). + private static bool Decodes(byte[] jpeg) + { + try + { + var info = JpegDecoder.ReadInfo(jpeg); + if (info.Process == JpegProcess.Lossless || info.Components is not (1 or 3) || info.Width < 1 || info.Height < 1) return false; + using var probe = JpegDecoder.DecodeToLinear(jpeg, new JpegDecodeOptions { MaxLongEdge = ProbeLongEdge }); + return true; + } + catch (Exception ex) when (ex is InvalidDataException or NotSupportedException) { return false; } diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs index 74b1bf56..c3cc7dc7 100644 --- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -215,7 +215,11 @@ public sealed class ArchiveMaintenance if (length == 0 || garbage < length * CompactionRatio) continue; compacted.Add(location.Folder); if (dryRun) continue; + // The thumbnail builder appends under the thumbs lease only: holding it too keeps the compacted pack from + // replacing the file under a writer, whose later thumbnails would land in the orphaned one. await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) + using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{location.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait, + _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) ThumbPackWriter.Compact(sequence.ThumbsPackPath); } return compacted; diff --git a/src/Titano.Data/Archiving/MassRenamer.cs b/src/Titano.Data/Archiving/MassRenamer.cs index 9cc0af2e..6846e70d 100644 --- a/src/Titano.Data/Archiving/MassRenamer.cs +++ b/src/Titano.Data/Archiving/MassRenamer.cs @@ -32,6 +32,8 @@ public sealed class MassRenamer /// New folder template ({title} = the new title), or null to keep folders. /// New title template ({title} = the current title), or null to keep titles. + /// Sequences nested in a folder that moves are part of the plan (they move with it); see . + /// A new folder would lie inside a sequence's folder. public async Task PlanAsync(Archive archive, IReadOnlyList sequenceIds, string? folderTemplate, string? titleTemplate, CancellationToken cancellationToken = default) { @@ -42,12 +44,23 @@ public sealed class MassRenamer { if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } s || s.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(sequenceIds)); - subjects.Add(new RenameSubject(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens)); + subjects.Add(Subject(s)); + } + var nested = new List(); + if (folderTemplate is not null) + { + var selected = sequenceIds.ToHashSet(); + foreach (var location in locations.Where(l => !selected.Contains(l.Id) && subjects.Any(s => RenamePlanner.IsInside(s.Folder, l.Folder)))) + { + if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is { } inner) nested.Add(Subject(inner)); + } } return RenamePlanner.Plan(archive.Id, subjects, folderTemplate, titleTemplate, locations.Select(l => l.Folder), - folder => Directory.Exists(paths.Sequence(folder).Folder)); + folder => Directory.Exists(paths.Sequence(folder).Folder), nested); } + private static RenameSubject Subject(Sequence s) => new(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens); + /// Applies a plan; each completed step is journaled before the next starts. /// A sequence is no longer where the plan expects it; nothing of that step was changed. public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress? progress = null, CancellationToken cancellationToken = default) => @@ -80,55 +93,106 @@ public sealed class MassRenamer } } RenameJournal.Write(paths, journal); - for (var i = 0; i < journal.Plan.Steps.Count; i++) + var steps = journal.Plan.Steps; + foreach (var step in steps) { - var step = journal.Plan.Steps[i]; if (completed.Contains(step.SequenceId)) continue; cancellationToken.ThrowIfCancellationRequested(); - try - { - await ApplyStepAsync(archive, paths, step, cancellationToken).ConfigureAwait(false); - } - catch (StaleRenamePlanException) when (resuming) - { - Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there"); - } + // A folder that moves carries the sequences inside it: they are locked, moved and recorded with it. + IReadOnlyList carried = step.MovesFolder && !step.Carried + ? [.. steps.Where(c => c.Carried && !completed.Contains(c.SequenceId) && Carrier(steps, c) == step)] + : []; + await ApplyStepAsync(archive, paths, step, carried, resuming, cancellationToken).ConfigureAwait(false); completed.Add(step.SequenceId); + foreach (var c in carried) completed.Add(c.SequenceId); RenameJournal.Write(paths, journal with { Completed = [.. completed] }); - progress?.Report(i + 1); + progress?.Report(completed.Count); } RenameJournal.Delete(paths, journal.Plan.Id); } - /// Folder, then sidecar title, then index; each part is idempotent so a resumed step can repeat it. - private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken) + /// The step whose folder move carries : the nearest moving folder around it (as planned). + private static RenameStep? Carrier(IReadOnlyList steps, RenameStep carried) => + steps.Where(m => !m.Carried && m.MovesFolder && RenamePlanner.IsInside(m.FromFolder, carried.FromFolder)).MaxBy(m => m.FromFolder.Length); + + /// + /// Applies a step, and the steps of the sequences its folder carries, under all their leases: sequence leases, then + /// for a move the thumbnail leases, each by sequence id (the order trash and pack compaction take them in). No + /// writer of a carried sequence (ingest, an editor save, a thumbnail pass) works in the folder while it moves or + /// before its index row follows. + /// + /// A step whose sequence the index no longer has where the plan says is dropped instead of + /// failing the rename: its sidecar still carries the id, so the next scan indexes it wherever it ended up. + private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, IReadOnlyList carried, bool resuming, + CancellationToken cancellationToken) { - await using (await _writer.LockAsync(paths, step.SequenceId, cancellationToken).ConfigureAwait(false)) + var ids = carried.Select(c => c.SequenceId).Append(step.SequenceId).Order().ToList(); + var locks = new List(ids.Count); + var leases = new List(ids.Count); + try { - var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); - if (row is null || (row.Folder != step.FromFolder && row.Folder != step.ToFolder)) throw new StaleRenamePlanException(step.SequenceId); - RenamePlanner.MoveFolder(paths, step); - var sequence = paths.Sequence(step.ToFolder); - var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId); - if (manifest.Title != step.ToTitle) + foreach (var id in ids) locks.Add(await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false)); + if (step.MovesFolder) { - manifest = manifest with + foreach (var id in ids) { - Title = step.ToTitle, - Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 }, - UpdatedAt = DateTimeOffset.UtcNow, - }; - ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + leases.Add(await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{id:N}", _writer.Options.Owner, _writer.Options.LeaseWait, + _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)); + } } - await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + // Checked before anything moves; a dropped step drops the sequences it would have carried, which stay inside it. + if (!await StandsAsync(step, resuming, cancellationToken).ConfigureAwait(false)) return; + var follow = new List(carried.Count); + foreach (var c in carried) { - var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); - await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); - if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false); - await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false); - await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + if (await StandsAsync(c, resuming, cancellationToken).ConfigureAwait(false)) follow.Add(c); } - await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false); + RenamePlanner.MoveFolder(paths, step); + await RecordAsync(archive, paths, step, cancellationToken).ConfigureAwait(false); + // Moved with their carrier: only their sidecars and index rows follow. + foreach (var c in follow) await RecordAsync(archive, paths, c, cancellationToken).ConfigureAwait(false); + } + finally + { + for (var i = leases.Count - 1; i >= 0; i--) leases[i].Dispose(); + for (var i = locks.Count - 1; i >= 0; i--) await locks[i].DisposeAsync().ConfigureAwait(false); } } + + /// The index still has the step's sequence where the plan says (or where it already took it); otherwise stale. + /// Not resuming, and the sequence moved or vanished since the plan was made. + private async Task StandsAsync(RenameStep step, bool resuming, CancellationToken cancellationToken) + { + var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); + if (row is not null && (row.Folder == step.FromFolder || row.Folder == step.ToFolder)) return true; + if (!resuming) throw new StaleRenamePlanException(step.SequenceId); + Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there"); + return false; + } + + /// Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it. + private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken) + { + var sequence = paths.Sequence(step.ToFolder); + var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId); + if (manifest.Title != step.ToTitle) + { + manifest = manifest with + { + Title = step.ToTitle, + Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }; + ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + } + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false); + await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false); + } } diff --git a/src/Titano.Data/Migrations/postgres/0006_sync_proxy_compare.sql b/src/Titano.Data/Migrations/postgres/0006_sync_proxy_compare.sql new file mode 100644 index 00000000..63764a45 --- /dev/null +++ b/src/Titano.Data/Migrations/postgres/0006_sync_proxy_compare.sql @@ -0,0 +1,5 @@ +-- Client sync state (BRIEF §2.3, §14.5): the server's update time a linked copy's originals were compared at, kept +-- while proxies the server asks for may be made on this PC; they change without the frame counters moving, so the +-- copy is compared again when the server's sequence changed since. Mirrors sqlite/0006_sync_proxy_compare.sql. + +ALTER TABLE sync_links ADD COLUMN diff_remote_updated_at timestamptz; diff --git a/src/Titano.Data/Migrations/sqlite/0006_sync_proxy_compare.sql b/src/Titano.Data/Migrations/sqlite/0006_sync_proxy_compare.sql new file mode 100644 index 00000000..6088e40b --- /dev/null +++ b/src/Titano.Data/Migrations/sqlite/0006_sync_proxy_compare.sql @@ -0,0 +1,5 @@ +-- Client sync state (BRIEF §2.3, §14.5): the server's update time a linked copy's originals were compared at, kept +-- while proxies the server asks for may be made on this PC; they change without the frame counters moving, so the +-- copy is compared again when the server's sequence changed since. Mirrors postgres/0006_sync_proxy_compare.sql. + +ALTER TABLE sync_links ADD COLUMN diff_remote_updated_at TEXT; diff --git a/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs b/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs index 359361d5..8c1f1145 100644 --- a/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs +++ b/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs @@ -62,7 +62,7 @@ internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions opti var targets = all.Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).Select(t => (t.i, adapt(t.t))).ToList(); if (targets.Count == 0) return; - var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, cancellation).ConfigureAwait(false); + var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, options.Archive.Decoder, cancellation).ConfigureAwait(false); IReadOnlyList outcomes; await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames.Count), options)) { diff --git a/src/Titano.Data/Processing/RenderNodeClient.cs b/src/Titano.Data/Processing/RenderNodeClient.cs index c8938ad7..3ca316bf 100644 --- a/src/Titano.Data/Processing/RenderNodeClient.cs +++ b/src/Titano.Data/Processing/RenderNodeClient.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Net; using System.Text.Json; +using Titano.Core.Archiving; using Titano.Core.Diagnostics; using Titano.Core.Export; using Titano.Core.Imaging; @@ -8,6 +9,7 @@ using Titano.Core.Localization; using Titano.Core.Model; using Titano.Core.Render; using Titano.Core.Video; +using Titano.Data.Archiving; using Titano.Protocol; using Titano.Protocol.Client; @@ -47,8 +49,10 @@ public sealed record RenderNodeOptions /// /// "Offer this PC as render node" (BRIEF §14.7): registers this machine's single-file encoders with the server, /// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals -/// hash-verified into a content-addressed cache, or proxies when the job asks; a dropped download resumes), renders the -/// job's preset with the host's encoders and uploads the output, which the server files as a render of the sequence. +/// hash-verified into a content-addressed cache, or proxies when the job asks or this PC cannot decode the original; a +/// dropped download resumes; frames missing, corrupt or excluded on the server are left out), renders the job's preset +/// with the host's encoders and uploads the output, which the server files as a render of the sequence. Then it sends +/// the proxies the server asks for, made from the originals it fetched (BRIEF §2.3: the server cannot decode HEIF). /// Progress and render remarks go to the server; a job cancelled or reassigned there stops here; a failure is reported /// with its reason. Stopping abandons the job in flight without reporting it: the server requeues it when the node /// registers again or its heartbeats stop. Image sequences are not offered and batches are refused: the protocol uploads @@ -58,6 +62,12 @@ public sealed class RenderNodeClient : IAsyncDisposable { private const string LogCategory = "render-node"; + /// + /// Frames no node renders: absent on the server, not the archived file, or excluded. Undecodable is the server's + /// verdict on its own decoders, so this PC's decoders try those frames. + /// + internal const FrameFlags NotFetched = FrameFlags.Missing | FrameFlags.Corrupt | FrameFlags.Excluded; + private readonly TitanoClient _client; private readonly ProcessingOptions _processing; private readonly RenderNodeOptions _options; @@ -127,7 +137,9 @@ public sealed class RenderNodeClient : IAsyncDisposable try { Directory.CreateDirectory(work); - await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false); + var input = await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false); + // The job is done on the server; the proxies it asked for follow, on the node's own time. + await SendProxiesAsync(claimed, input, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (job.IsCancellationRequested) { @@ -221,7 +233,8 @@ public sealed class RenderNodeClient : IAsyncDisposable } } - private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken) + /// Renders the job and uploads its output; returns the frames it read. + private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken) { var export = ProcessingJobs.ReadExport(claimed.PresetJson); var resolved = ExportResolver.Resolve(export); @@ -232,9 +245,18 @@ public sealed class RenderNodeClient : IAsyncDisposable var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export }; var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false)); - var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false); - var files = await DownloadAsync(claimed, work, reporter, cancellationToken).ConfigureAwait(false); - var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id))]); + // The server's Undecodable stands, as in its own render, except for a format only its decoders lack (HEIF) that + // this PC's read: such a frame renders here from its original. One read from its proxy renders either way. + var decoder = _processing.Archive.Decoder; + var frames = (await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false)) + .ConvertAll(f => ArchiveImages.ServerNeedsProxy(f.Format) && !ArchiveImages.NeedsProxy(decoder, f.Format) + ? f with { Flags = f.Flags & ~FrameFlags.Undecodable } + : f); + var files = await DownloadAsync(claimed, frames, work, reporter, cancellationToken).ConfigureAwait(false); + var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id).Path)]) + { + Proxied = [.. frames.Select(f => files.GetValueOrDefault(f.Id).Proxy)], + }; IReadOnlyList outcomes; await using (var render = new SequenceRender(input, project, null, _processing)) @@ -253,6 +275,56 @@ public sealed class RenderNodeClient : IAsyncDisposable cancellationToken).ConfigureAwait(false); } if (outcomes[0].Error is { } error) throw new InvalidOperationException(error); + return input; + } + + /// + /// Sends the proxies the server asks for () that this PC can make from the + /// originals it fetched, as the server's proxy policy says; a failure is logged and never fails the finished job. + /// + private async Task SendProxiesAsync(ClaimedJob claimed, SequenceFrames input, CancellationToken cancellationToken) + { + if (claimed.Proxies is not { } spec) return; + var wanted = claimed.Frames.Where(f => f.WantsProxy).Select(f => f.Id).ToHashSet(); + if (wanted.Count == 0) return; + var camera = new SidecarCamera { Make = input.Sequence.CameraMake, Model = input.Sequence.CameraModel }; + int sent = 0, unsent = 0; + for (var i = 0; i < input.Count; i++) + { + var frame = input.Frames[i]; + if (!wanted.Remove(frame.Id)) continue; + if (input.Proxied?[i] == true || input.Files[i] is not { } original) + { + unsent++; + continue; + } + try + { + var jpeg = await Task.Run(() => ArchiveImages.MakeProxy(_processing.Archive.Decoder, original, frame, camera, spec.LongEdge, spec.Quality, + cancellationToken), cancellationToken).ConfigureAwait(false); + if (jpeg is null) + { + unsent++; + continue; + } + await _client.UploadProxyAsync(frame.Id, jpeg, cancellationToken).ConfigureAwait(false); + sent++; + } + catch (TitanoApiException ex) when (ex.Status is HttpStatusCode.Conflict or HttpStatusCode.Forbidden) + { + // Proxies turned off on the server, or this account may no longer send them: none of the rest would go either. + Log.Warning(LogCategory, $"The server refused the proxies of job {claimed.Job.Id}", ex); + return; + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + // Best effort (a timed-out upload included): the job is done, and a proxy the server still lacks is asked for + // again with the next job. + Log.Warning(LogCategory, $"The proxy of {frame.FileName} could not be sent", ex); + unsent++; + } + } + Log.Info(LogCategory, $"Proxies of job {claimed.Job.Id}: {sent} sent, {unsent + wanted.Count} not"); } private async Task UploadAsync(Guid nodeId, Guid jobId, ExportTarget target, RenderResult result, CancellationToken cancellationToken) @@ -280,27 +352,35 @@ public sealed class RenderNodeClient : IAsyncDisposable } /// - /// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused - /// across jobs, fetched once per content), or proxies into the job folder when the job asks for them and the server - /// has them. + /// The files of the job's frames (none for frames missing, corrupt or excluded on the server): originals verified + /// against their SHA-256 into the content-addressed cache (reused across jobs, fetched once per content), originals the + /// server has not hashed yet into the job folder, or proxies into the job folder when the job asks for them or this PC + /// cannot decode the original, and the server has them. /// - private async Task> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken) + private async Task> DownloadAsync(ClaimedJob claimed, IReadOnlyList rows, string work, + NodeReporter reporter, CancellationToken cancellationToken) { - var files = new Dictionary(); + var formats = rows.ToDictionary(f => f.Id, f => f.Format); + var undecodable = rows.Where(f => (f.Flags & FrameFlags.Undecodable) != 0).Select(f => f.Id).ToHashSet(); + bool Proxy(ManifestFrame f) => + f.HasProxy && (claimed.UseProxies || (formats.TryGetValue(f.Id, out var format) && ArchiveImages.NeedsProxy(_processing.Archive.Decoder, format))); + var files = new Dictionary(); + // An original that stays undecodable would only be skipped by the render. + var fetched = claimed.Frames.Where(f => ((FrameFlags)f.Flags & NotFetched) == 0 && (Proxy(f) || !undecodable.Contains(f.Id))).ToList(); var done = 0; - var total = claimed.Frames.Count; + var total = fetched.Count; await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false); - var downloads = claimed.Frames.GroupBy(f => claimed.UseProxies && f.HasProxy ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList(); + var downloads = fetched.GroupBy(f => Proxy(f) || f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList(); await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken }, async (frames, ct) => { var frame = frames.First(); - var path = claimed.UseProxies && frame.HasProxy + var file = Proxy(frame) ? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false) - : await DownloadOriginalAsync(frame, ct).ConfigureAwait(false); + : (await DownloadOriginalAsync(frame, work, ct).ConfigureAwait(false), false); lock (files) { - foreach (var f in frames) files[f.Id] = path; + foreach (var f in frames) files[f.Id] = file; } await reporter.ReportAsync(Interlocked.Add(ref done, frames.Count()), total, null).ConfigureAwait(false); }).ConfigureAwait(false); @@ -312,25 +392,29 @@ public sealed class RenderNodeClient : IAsyncDisposable /// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server /// has none for the frame. /// - private async Task DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) + private async Task<(string Path, bool Proxy)> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) { var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg"); Directory.CreateDirectory(Path.GetDirectoryName(path)!); try { await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false); - return path; + return (path, true); } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) { File.Delete(path); - return await DownloadOriginalAsync(frame, cancellationToken).ConfigureAwait(false); + return (await DownloadOriginalAsync(frame, work, cancellationToken).ConfigureAwait(false), false); } } - /// A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256. - private async Task DownloadOriginalAsync(ManifestFrame frame, CancellationToken cancellationToken) + /// + /// A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256; an original the + /// server has not hashed yet goes into the job folder and must have the manifest's size. + /// + private async Task DownloadOriginalAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) { + if (frame.Sha256.Length == 0) return await DownloadUnhashedAsync(frame, work, cancellationToken).ConfigureAwait(false); var sha = frame.Sha256.ToLowerInvariant(); if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName)); var path = Path.Combine(_options.CacheDirectory, "originals", sha[..2], sha + Path.GetExtension(frame.FileName).ToLowerInvariant()); @@ -355,6 +439,23 @@ public sealed class RenderNodeClient : IAsyncDisposable } } + /// + /// An original without a SHA-256 on the server (adopted, not verified yet): nothing to check it against but its size, + /// so it is neither cached nor shared with other frames; a copy of another size is fetched again once. + /// + private async Task DownloadUnhashedAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) + { + var path = Path.Combine(work, "originals", frame.Id.ToString("N") + Path.GetExtension(frame.FileName).ToLowerInvariant()); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + for (var attempt = 1; ; attempt++) + { + if (!File.Exists(path) || new FileInfo(path).Length < frame.Size) await FetchAsync(frame, "original", path, resume: true, cancellationToken).ConfigureAwait(false); + if (new FileInfo(path).Length == frame.Size) return path; + File.Delete(path); + if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName)); + } + } + /// /// Fetches a frame resource into . A dropped connection or a busy server is retried (up to /// times, the wait doubling from ); diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index 8ded6dad..37e68042 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -36,7 +36,7 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); - var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, cancellationToken).ConfigureAwait(false); + var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false); var metadata = frames.LightFrames(); var light = LightAnalyzer.Analyze(metadata); diff --git a/src/Titano.Data/Processing/SequenceFrames.cs b/src/Titano.Data/Processing/SequenceFrames.cs index 6d9f7cc0..98ea6134 100644 --- a/src/Titano.Data/Processing/SequenceFrames.cs +++ b/src/Titano.Data/Processing/SequenceFrames.cs @@ -1,8 +1,10 @@ using Titano.Core.Analysis; using Titano.Core.Archiving; using Titano.Core.Diagnostics; +using Titano.Core.Imaging; using Titano.Core.Metadata; using Titano.Core.Model; +using Titano.Data.Archiving; namespace Titano.Data.Processing; @@ -20,22 +22,39 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList Fr public int Count => Frames.Count; - /// Every frame row of with its archive file (the proxy when asked and built). - public static async Task LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies, + /// + /// Per frame, whether its file is its proxy. is this host's verdict on the + /// original, so it does not keep a frame read from its proxy out of a render. Null: none is. + /// + public IReadOnlyList? Proxied { get; init; } + + /// The originals, for metadata (proxies carry no GPS); null when are the originals. + public IReadOnlyList? Originals { get; init; } + + /// + /// Every frame row of with its archive file: the proxy when asked, and whenever the + /// original is one cannot decode (: + /// HEIF on the server, BRIEF §2.3, §8.5), as long as the proxy exists; else the original. + /// + public static async Task LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies, ImageDecoder decoder, CancellationToken cancellationToken) { var frames = await ListAsync(db, sequence.Id, cancellationToken).ConfigureAwait(false); var paths = new ArchivePaths(archive.RootPath); var folder = paths.Sequence(sequence.Folder); var files = new string?[frames.Count]; + var originals = new string?[frames.Count]; + var proxied = new bool[frames.Count]; for (var i = 0; i < files.Length; i++) { var frame = frames[i]; if ((frame.Flags & FrameFlags.Missing) != 0) continue; - var proxy = useProxies ? folder.Proxy(frame.Index) : null; - files[i] = proxy is not null && File.Exists(proxy) ? proxy : paths.FromRelative(frame.RelativePath); + originals[i] = paths.FromRelative(frame.RelativePath); + var proxy = useProxies || ArchiveImages.NeedsProxy(decoder, frame.Format) ? folder.Proxy(frame.Index) : null; + proxied[i] = proxy is not null && File.Exists(proxy); + files[i] = proxied[i] ? proxy : originals[i]; } - return new SequenceFrames(sequence, frames, files); + return new SequenceFrames(sequence, frames, files) { Proxied = proxied, Originals = originals }; } public static async Task> ListAsync(TitanoDatabase db, Guid sequenceId, CancellationToken cancellationToken) @@ -80,15 +99,17 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList Fr { for (var i = 0; i < Frames.Count; i++) { - if (Files[i] is null || (Frames[i].Flags & NotRendered) != 0) yield return i; + var skip = Proxied is { } proxied && proxied[i] ? NotRendered & ~FrameFlags.Undecodable : NotRendered; + if (Files[i] is null || (Frames[i].Flags & skip) != 0) yield return i; } } private (int Index, GpsPosition Gps)? FirstGpsFix() { - for (var i = 0; i < Files.Count; i++) + var files = Originals ?? Files; + for (var i = 0; i < files.Count; i++) { - if (Files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue; + if (files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue; try { return MetadataReader.Read(file).Gps is { } gps ? (i, gps) : null; diff --git a/src/Titano.Data/Sync/SyncLinkStore.cs b/src/Titano.Data/Sync/SyncLinkStore.cs index 1d28e3f4..9c7f0c75 100644 --- a/src/Titano.Data/Sync/SyncLinkStore.cs +++ b/src/Titano.Data/Sync/SyncLinkStore.cs @@ -11,7 +11,8 @@ internal sealed class SyncLinkStore(TitanoDatabase db) "server, remote_id, local_id, title, frame_count, total_bytes, render_count, remote_status, remote_settings_version, " + "remote_frames_version, remote_updated_at, base_local_settings, base_remote_settings, diff_local_frames, diff_remote_frames, " + "diff_local_renders, diff_remote_renders, missing_remote, missing_local, exports_to_get, exports_to_get_bytes, exports_to_send, " + - "exports_to_send_bytes, conflict_local_settings, conflict_remote_settings, conflict_remote, proxies_cached, proxies_frames_version, error, synced_at"; + "exports_to_send_bytes, conflict_local_settings, conflict_remote_settings, conflict_remote, proxies_cached, proxies_frames_version, error, synced_at, " + + "diff_remote_updated_at"; private static readonly string UpsertSql = SqlDialect.Upsert("sync_links", Columns.Split(", "), 2); @@ -36,7 +37,7 @@ internal sealed class SyncLinkStore(TitanoDatabase db) l.DiffLocalRenders, l.DiffRemoteRenders, l.MissingRemote, l.MissingLocal, l.ExportsToGet, l.ExportsToGetBytes, l.ExportsToSend, l.ExportsToSendBytes, l.ConflictLocalSettings, l.ConflictRemoteSettings, l.ConflictRemote is { } remote ? JsonSerializer.Serialize(remote, SyncPayloadJson.Default.SyncPayload) : null, - l.ProxiesCached, l.ProxiesFramesVersion, l.Error, l.SyncedAt, + l.ProxiesCached, l.ProxiesFramesVersion, l.Error, l.SyncedAt, l.DiffRemoteUpdatedAt, ], cancellationToken); public Task DeleteAsync(string server, Guid remoteId, CancellationToken cancellationToken) => @@ -103,5 +104,6 @@ internal sealed class SyncLinkStore(TitanoDatabase db) ProxiesFramesVersion = r.GetInt32OrNull(27), Error = r.GetStringOrNull(28), SyncedAt = r.GetDateTimeOffsetOrNull(29), + DiffRemoteUpdatedAt = r.GetDateTimeOffsetOrNull(30), }; } diff --git a/src/Titano.Data/Sync/SyncModel.cs b/src/Titano.Data/Sync/SyncModel.cs index ca465d6c..a379a54d 100644 --- a/src/Titano.Data/Sync/SyncModel.cs +++ b/src/Titano.Data/Sync/SyncModel.cs @@ -91,7 +91,18 @@ public sealed record SyncLink public int? DiffLocalRenders { get; init; } public int? DiffRemoteRenders { get; init; } - /// Local originals the server does not have. + /// + /// The server's update time the originals were compared at, kept while proxies it asks for may be made here (some + /// were counted, or the copy holds originals the server cannot decode and this PC can); null otherwise. The proxies + /// it asks for change without its frame counters moving (one arrives from a render node, its proxy policy changes), + /// and every such change leaves a new update time, so the originals are compared again when it moved. + /// + public DateTimeOffset? DiffRemoteUpdatedAt { get; init; } + + /// + /// Local originals the server does not have, and proxies it asks for that this PC can make (it cannot decode those + /// originals): an upload sends both. + /// public int MissingRemote { get; init; } /// Server originals this PC does not have. diff --git a/src/Titano.Data/Sync/SyncRemote.cs b/src/Titano.Data/Sync/SyncRemote.cs index 76268b4f..08537a27 100644 --- a/src/Titano.Data/Sync/SyncRemote.cs +++ b/src/Titano.Data/Sync/SyncRemote.cs @@ -42,6 +42,13 @@ public interface ISyncRemote : IDisposable /// Streams a frame's original, proxy or thumb, resuming with Range from the destination's length. Task DownloadFrameAsync(Guid frameId, string kind, Stream destination, CancellationToken cancellationToken); + /// + /// Sends the proxy of a frame the server cannot decode (); a 422 + /// means it was refused (not a JPEG of that frame), 409 that the server makes no + /// proxies. A remote without the call (the screenshots' sample server) throws . + /// + Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken cancellationToken) => throw new NotSupportedException(); + /// Streams a render's file, resuming with Range from the destination's length; returns the file name the server gives it. Task DownloadRenderAsync(Guid renderId, Stream destination, CancellationToken cancellationToken); @@ -116,6 +123,8 @@ public sealed class TitanoSyncRemote(TitanoClient client, bool ownsClient) : ISy public Task DownloadFrameAsync(Guid frameId, string kind, Stream destination, CancellationToken cancellationToken) => client.DownloadFrameAsync(frameId, kind, destination, null, cancellationToken); + public Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken cancellationToken) => client.UploadProxyAsync(frameId, jpeg, cancellationToken); + public Task DownloadRenderAsync(Guid renderId, Stream destination, CancellationToken cancellationToken) => client.DownloadRenderAsync(renderId, destination, cancellationToken); diff --git a/src/Titano.Data/Sync/SyncRules.cs b/src/Titano.Data/Sync/SyncRules.cs index a8aa0e12..e0852bce 100644 --- a/src/Titano.Data/Sync/SyncRules.cs +++ b/src/Titano.Data/Sync/SyncRules.cs @@ -2,8 +2,10 @@ using System.Globalization; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; +using Titano.Core.Imaging; using Titano.Core.Model; using Titano.Core.Project; +using Titano.Data.Archiving; using Titano.Protocol; namespace Titano.Data.Sync; @@ -130,6 +132,43 @@ public static class SyncRules return (hashed.Count - matched + Math.Max(0, unhashed - unmatchedRemote), Math.Max(0, unmatchedRemote - unhashed)); } + /// + /// Frames whose proxy the server asks for (it cannot decode their originals, BRIEF §2.3) that this PC holds and its + /// reads: an upload makes and sends them. A local original not hashed yet (a folder adopted + /// and not verified) stands for a wanted frame of its size that no hash matched (by size, not name: the server names + /// its copy by its own template); the upload hashes it before making the proxy. + /// + public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection local, ImageDecoder decoder) + { + ArgumentNullException.ThrowIfNull(manifest); + var hashed = local.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal); + var unhashed = local.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f)).GroupBy(f => f.Size).ToDictionary(g => g.Key, g => g.Count()); + var count = 0; + foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length > 0)) + { + var sha = frame.Sha256.ToLowerInvariant(); + if (hashed.Contains(sha)) + { + if (hashed[sha].Any(Readable)) count++; + } + else if (unhashed.TryGetValue(frame.Size, out var left) && left > 0) + { + unhashed[frame.Size] = left - 1; + count++; + } + } + return count; + + bool Readable(Frame frame) => !ArchiveImages.NeedsProxy(decoder, frame.Format); + } + + /// + /// Whether holds originals a server cannot decode (it asks for their proxies, BRIEF §2.3, + /// §4.2: managed decoders only) that this PC's reads, so it can make those proxies. + /// + public static bool MakesProxiesFor(IEnumerable local, ImageDecoder decoder) => + local.Select(f => f.Format).Distinct().Any(format => ArchiveImages.ServerNeedsProxy(format) && !ArchiveImages.NeedsProxy(decoder, format)); + internal static string Blank(string? text) => text?.Trim() ?? string.Empty; private static JsonNode? Parse(string json) diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index 947e706c..cc5e6071 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -43,8 +43,9 @@ public sealed record SyncOptions /// conflict the user settles with , against the versions the user was shown. Uploads and /// downloads run as jobs (): uploads use the chunked, hash-verified, resumable protocol; /// downloaded originals are verified and committed through the archive ingest, so sidecars stay right; proxies go to a -/// cache and open as a ; renders travel on request. Proxies and thumbnails never travel -/// upward. A local copy and a server sequence holding the same originals are linked by content when no link joins +/// cache and open as a ; renders travel on request. Thumbnails never travel upward, and +/// proxies only for originals the server cannot decode (BRIEF §2.3: HEIF), made here and counted with the originals the +/// server lacks. A local copy and a server sequence holding the same originals are linked by content when no link joins /// them. The state of each local sequence is written to its sync_state for the Library. A failure while /// comparing one sequence is recorded on its link and does not stop the others. /// @@ -58,7 +59,7 @@ public sealed class SyncService private readonly TitanoDatabase _db; private readonly SyncLinkStore _links; - private readonly ProjectStore _projects; + private readonly SequenceWriter _writer; private readonly SemaphoreSlim _gate = new(1, 1); /// Unlinked pairs whose manifests were compared at these frame versions without a match (not fetched again). @@ -70,7 +71,7 @@ public sealed class SyncService Archives = archives; Options = options; _links = new SyncLinkStore(db); - _projects = new ProjectStore(db, new SequenceWriter(options.Archive)); + _writer = new SequenceWriter(options.Archive); Proxies = new ProxyCache(options.ProxyCacheDirectory); } @@ -211,8 +212,17 @@ public sealed class SyncService else { if (resolution == SyncResolution.SaveBoth) saved = await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false); - var applied = await ApplyLocallyAsync(local, theirs, cancellationToken).ConfigureAwait(false); - link = Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, seenRemote); + if (await ApplyLocallyAsync(local, theirs, cancellationToken).ConfigureAwait(false) is { } applied) + { + link = Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, seenRemote); + } + else + { + // Edited on this PC since it was read: the copy of a version the user did not choose to keep goes. + changed = true; + if (saved is not null) File.Delete(saved); + saved = null; + } } } if (!changed) @@ -359,7 +369,8 @@ public sealed class SyncService var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false); var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false); - var applied = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false); + // Edited on this PC meanwhile: nothing is agreed yet, so the next comparison settles both sides' content. + if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false) is not { } applied) return; await _links.UpsertAsync(Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, manifest.SettingsVersion), cancellationToken) .ConfigureAwait(false); } @@ -479,13 +490,19 @@ public sealed class SyncService } SequenceManifest? manifest = null; if (link.DiffLocalFrames != local.FramesVersion || link.DiffRemoteFrames != link.RemoteFramesVersion - || link.DiffLocalRenders != local.RenderedCount || link.DiffRemoteRenders != link.RenderCount) + || link.DiffLocalRenders != local.RenderedCount || link.DiffRemoteRenders != link.RenderCount + || (link.DiffRemoteUpdatedAt is { } compared && !SameMoment(compared, link.RemoteUpdatedAt))) { manifest = await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false); // A frame flagged missing here counts as absent: downloading the originals restores it from the server. - var (missingRemote, missingLocal) = SyncRules.CompareOriginals( - [.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]); + var present = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList(); + var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]); + // Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3). + var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder) : 0; + // Those change without the server's frame counters moving: while some may be made here, the server's next + // change of the sequence brings a new comparison. + var watch = policy.CanEdit && (proxies > 0 || SyncRules.MakesProxiesFor(present, Options.Archive.Decoder)); link = await CompareExportsAsync(link, local, manifest.Renders, cancellationToken).ConfigureAwait(false) with { DiffLocalFrames = local.FramesVersion, @@ -493,7 +510,8 @@ public sealed class SyncService RemoteFramesVersion = manifest.FramesVersion, DiffLocalRenders = local.RenderedCount, DiffRemoteRenders = manifest.Renders.Count, - MissingRemote = missingRemote, + DiffRemoteUpdatedAt = watch ? link.RemoteUpdatedAt : null, + MissingRemote = missingRemote + proxies, MissingLocal = missingLocal, }; } @@ -547,8 +565,13 @@ public sealed class SyncService return (policy.CanEdit ? await PushAsync(remote, link, local, link.BaseRemoteSettings, cancellationToken).ConfigureAwait(false) : link, local); manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); if (manifest.SettingsVersion != dto.SettingsVersion) return (link, local); - var pulled = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false); - return (Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion), pulled); + if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false) is { } pulled) + return (Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion), pulled); + // The local copy was read when the comparison began and an edit was saved on this PC since: both sides changed, + // so the server's version must not replace it unseen (BRIEF §14.5). + var fresh = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false); + if (fresh is null || fresh.SettingsVersion == local.SettingsVersion) return (link, local); + return await SettleSettingsAsync(remote, link, dto, fresh, manifest, policy, cancellationToken).ConfigureAwait(false); } /// Sends this PC's payload unless the server moved past , which shows up as a conflict next. @@ -621,20 +644,67 @@ public sealed class SyncService private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified); - /// Writes a payload into the local sidecar, project and index (each write bumps the local settings version). - private async Task ApplyLocallyAsync(Sequence local, SyncPayload payload, CancellationToken cancellationToken) + /// The same instant as databases keep it (to the microsecond). + private static bool SameMoment(DateTimeOffset a, DateTimeOffset b) => a.UtcTicks / 10 == b.UtcTicks / 10; + + /// + /// Writes a payload into the local index, project and sidecar in one step (settings version + 1) under the sequence's + /// lease, which the editor's saves take too, and only while the local copy is still at the settings version + /// was read at. Null when an edit was saved on this PC since: nothing is written then. + /// + private async Task ApplyLocallyAsync(Sequence local, SyncPayload payload, CancellationToken cancellationToken) { - var current = local; - if (!payload.SameMetadata(SyncPayload.Of(local))) - current = await Archives.SetSequenceInfoAsync(local.Id, payload.Title, payload.Tags, payload.Rating, payload.Notes, cancellationToken).ConfigureAwait(false); - if (!SyncRules.SameSettings(payload.SettingsJson, current.SettingsJson)) + var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed."); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var folder = paths.Sequence(local.Folder); + await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false)) { - var archive = await _db.Archives.GetAsync(current.ArchiveId, cancellationToken).ConfigureAwait(false) - ?? throw new InvalidDataException($"Archive {current.ArchiveId} is not indexed."); - var project = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson)); - await _projects.UpdateAsync(new ArchivePaths(archive.RootPath), current, _ => project, cancellationToken).ConfigureAwait(false); + var row = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidDataException($"Sequence {local.Id} is not indexed."); + if (row.SettingsVersion != local.SettingsVersion) return null; + var metadata = !payload.SameMetadata(SyncPayload.Of(row)); + var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson); + if (!metadata && !settings) return row; + + string? project = null; + Guid? hero = null; + if (settings) + { + var edited = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson)); + project = edited.ToJson(); + hero = await HeroFrame.EditAsync(_db, row.Id, ProjectStore.Read(row, folder).HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false); + } + var edit = new SequenceEdit + { + Title = metadata ? payload.Title?.Trim() ?? string.Empty : null, + Tags = metadata ? payload.Tags : null, + Rating = metadata ? payload.Rating : null, + Notes = metadata ? payload.Notes ?? string.Empty : null, + SettingsJson = project, + HeroFrameId = hero, + }; + // The index first, at the version read under the lease: a writer that skips the lease still cannot be overwritten. + var result = await _db.Sequences.EditAsync(row.Id, edit, row.SettingsVersion, cancellationToken).ConfigureAwait(false); + if (result is not { Outcome: SequenceEditOutcome.Updated, Sequence: { } updated }) return null; + if (project is not null) AtomicFile.WriteAllText(folder.ProjectPath, project); + if (metadata) + { + var manifest = SequenceWriter.ReadLocked(folder, row.Id); + ManifestStore.WriteSequence(folder.ManifestPath, manifest with + { + Title = updated.Title, + Tags = updated.Tags, + Rating = updated.Rating, + Notes = updated.Notes, + Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) }, + UpdatedAt = DateTimeOffset.UtcNow, + }); + await Archives.Indexer.RecordStampsAsync(archive, folder, row.Id, cancellationToken).ConfigureAwait(false); + } + return updated; } - return await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) ?? current; } /// @@ -809,6 +879,7 @@ public sealed class SyncService DiffRemoteFrames = null, DiffLocalRenders = null, DiffRemoteRenders = null, + DiffRemoteUpdatedAt = null, MissingRemote = 0, MissingLocal = 0, ExportsToGet = 0, diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs index 5f0f356d..ae361dbc 100644 --- a/src/Titano.Data/Sync/SyncTransfers.cs +++ b/src/Titano.Data/Sync/SyncTransfers.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Globalization; using System.Net; using Titano.Core.Archiving; @@ -16,7 +17,8 @@ namespace Titano.Data.Sync; /// /// The work of upload and download jobs. Uploads send every local original the server lacks through the chunked, /// resumable protocol after hashing it again (an original that no longer matches its archived hash is never sent), -/// creating the server sequence first and sending its settings when the upload made it. Downloads fetch originals the +/// creating the server sequence first and sending its settings when the upload made it, then the proxies the server asks +/// for (originals it cannot decode, BRIEF §2.3), made here from this PC's originals. Downloads fetch originals the /// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 and commit /// them through the archive ingest, which names them and writes the sidecar; the first original creates the local /// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the @@ -49,12 +51,16 @@ internal sealed class SyncTransfers(SyncService sync) var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count); await progress.ReportAsync().ConfigureAwait(false); var skipped = 0; + // The SHA-256 of each original the index has none for (a folder adopted and not verified), as its upload computes + // it: the proxies the server asks for are matched to this PC's originals by content. Every such original is + // pending, so each one still here is hashed. + var hashes = new ConcurrentDictionary(); // Without a server sequence the originals go out one by one under this sequence's session key until the // server has filed one, which creates the sequence (an original it already holds elsewhere creates nothing). while (remoteId is null && pending.TryDequeue(out var first)) { - var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, ct) + var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, hashes, ct) .ConfigureAwait(false); if (sent is null) skipped++; else if (sent.SequenceId != Guid.Empty) @@ -69,16 +75,100 @@ internal sealed class SyncTransfers(SyncService sync) await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) => { - if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, token).ConfigureAwait(false) is null) + if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, hashes, token).ConfigureAwait(false) is null) Interlocked.Increment(ref skipped); await progress.StepAsync().ConfigureAwait(false); }).ConfigureAwait(false); + await SendProxiesAsync(context, remote, target, local, paths, frames, hashes).ConfigureAwait(false); if (created) await sync.PushAllAsync(remote, target, ct).ConfigureAwait(false); await sync.RecompareAsync(remote, target, ct).ConfigureAwait(false); if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped)); } + /// + /// The proxies the server asks for () that this PC's decoders can make from its + /// originals (by their archived SHA-256, or the one holds for an original the index has none + /// for), made as the server's proxy policy says with the server's capture data, and sent. One the server refuses is + /// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a + /// server busy or out of reach fails the job like an original would. + /// + private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList frames, + IReadOnlyDictionary hashes) + { + var ct = context.Cancellation; + var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); + var wanted = manifest.Frames.Where(f => f.WantsProxy).ToList(); + if (manifest.Proxies is not { } spec || wanted.Count == 0) return; + var originals = new Dictionary(StringComparer.Ordinal); + foreach (var frame in frames) + { + if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame); + } + var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id); + var camera = new SidecarCamera { Make = local.CameraMake, Model = local.CameraModel, Serial = local.CameraSerial }; + var decoder = sync.Options.Archive.Decoder; + var progress = new TransferProgress(context, wanted.Count, 0); + await progress.ReportAsync().ConfigureAwait(false); + int sent = 0, unsent = 0; + using var refused = CancellationTokenSource.CreateLinkedTokenSource(ct); + try + { + await Parallel.ForEachAsync(wanted, Workers(sync.Options.Uploads, refused.Token), async (frame, token) => + { + var jpeg = originals.TryGetValue(frame.Sha256.ToLowerInvariant(), out var original) && records.TryGetValue(frame.Id, out var record) + ? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName, + token), token).ConfigureAwait(false) + : null; + try + { + if (jpeg is not null) + { + await remote.UploadProxyAsync(frame.Id, jpeg, token).ConfigureAwait(false); + Interlocked.Increment(ref sent); + } + else + { + Interlocked.Increment(ref unsent); + } + } + catch (Exception ex) when (ex is NotSupportedException or TitanoApiException { Status: HttpStatusCode.Conflict or HttpStatusCode.Forbidden }) + { + // The server takes no proxies (or none from this account): nothing more to send. + Core.Diagnostics.Log.Info("sync", $"Proxies of {local.DisplayTitle} not sent: {ex.Message}"); + await refused.CancelAsync().ConfigureAwait(false); + } + catch (TitanoApiException ex) when (!ex.IsTransient) + { + // This proxy only (not a JPEG of that frame, a frame gone meanwhile): the others still go. + Core.Diagnostics.Log.Warning("sync", $"The proxy of {frame.FileName} was refused", ex); + Interlocked.Increment(ref unsent); + } + await progress.StepAsync().ConfigureAwait(false); + }).ConfigureAwait(false); + } + catch (OperationCanceledException) when (refused.IsCancellationRequested && !ct.IsCancellationRequested) + { + unsent = wanted.Count - sent; + } + context.Log(Loc.L("Archive.Log.ProxiesSent", sent, unsent)); + } + + /// A proxy made here for the server; null (logged) when this PC cannot decode or read the original either. + private static byte[]? MakeProxy(Core.Imaging.ImageDecoder decoder, string path, Frame frame, SidecarCamera camera, ProxySpec spec, string name, + CancellationToken cancellationToken) + { + try + { + return ArchiveImages.MakeProxy(decoder, path, frame, camera, spec.LongEdge, spec.Quality, cancellationToken); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or NotSupportedException) + { + Core.Diagnostics.Log.Warning("sync", $"No proxy of {name} could be made", ex); + return null; + } + } + public async Task DownloadOriginalsAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid? targetArchive) { var ct = context.Cancellation; @@ -364,10 +454,11 @@ internal sealed class SyncTransfers(SyncService sync) /// /// Hashes the original again and sends it into (or the sequence of - /// ); null (logged) when it is gone or no longer the archived file. + /// ); null (logged) when it is gone or no longer the archived file. The hash of an + /// original the index has none for goes into . /// private static async Task UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path, - Frame frame, CancellationToken cancellationToken) + Frame frame, ConcurrentDictionary hashes, CancellationToken cancellationToken) { if (!File.Exists(path)) { @@ -375,6 +466,7 @@ internal sealed class SyncTransfers(SyncService sync) return null; } var sha = await TitanoClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false); + if (frame.Sha256 is null) hashes[frame.Id] = sha; if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase)) { context.Log($"{frame.FileName}: changed since it was archived, not uploaded"); @@ -397,7 +489,11 @@ internal sealed class SyncTransfers(SyncService sync) return result; } - /// A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two attempts). + /// + /// A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two + /// attempts). A part as long as the file (a job stopped while hashing it) is only hashed again; a part the server can + /// no longer resume (416) starts over. + /// private static async Task FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging, CancellationToken cancellationToken) { var sha = frame.Sha256.ToLowerInvariant(); @@ -407,8 +503,19 @@ internal sealed class SyncTransfers(SyncService sync) var part = target + ".part"; for (var attempt = 1; ; attempt++) { - await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) - await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false); + if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) + { + try + { + await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false); + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1) + { + File.Delete(part); + continue; + } + } if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha) { File.Move(part, target, overwrite: true); diff --git a/src/Titano.Protocol/Client/TitanoClient.cs b/src/Titano.Protocol/Client/TitanoClient.cs index 704ab62a..2d8c3dfa 100644 --- a/src/Titano.Protocol/Client/TitanoClient.cs +++ b/src/Titano.Protocol/Client/TitanoClient.cs @@ -142,6 +142,25 @@ public sealed class TitanoClient : IDisposable } } + /// + /// Sends the proxy of a frame whose original the server cannot decode (): a JPEG + /// with the frame's capture time in its Exif, replacing the server's. A transient failure is retried. + /// + public Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(jpeg); + return RetryAsync(async () => + { + await Bandwidth.AcquireAsync(jpeg.Length, ct).ConfigureAwait(false); + using var content = new ByteArrayContent(jpeg); + content.Headers.ContentType = new MediaTypeHeaderValue("image/jpeg"); + using var response = await SendRawAsync(new HttpRequestMessage(HttpMethod.Put, $"api/v1/frames/{frameId}/proxy") { Content = content }, ct) + .ConfigureAwait(false); + await EnsureSuccessAsync(response, ct).ConfigureAwait(false); + return true; + }, ct); + } + // ---------------- Uploads ---------------- /// diff --git a/src/Titano.Protocol/Jobs.cs b/src/Titano.Protocol/Jobs.cs index 38ba16cf..398b4d73 100644 --- a/src/Titano.Protocol/Jobs.cs +++ b/src/Titano.Protocol/Jobs.cs @@ -37,7 +37,10 @@ public sealed record NodeEncoder(string Codec, string Name, bool Hardware, bool public sealed record RenderNodeDto(Guid Id, string ClientName, string Username, IReadOnlyList Encoders, DateTimeOffset LastHeartbeat, string State); /// POST /api/v1/nodes/{id}/claim: the job and everything needed to run it, or 204 when none matches. -public sealed record ClaimedJob(JobDto Job, Guid SequenceId, string SettingsJson, string PresetJson, bool UseProxies, IReadOnlyList Frames); +/// Every frame of the sequence; a node renders none flagged missing, corrupt or excluded. +/// How to make the proxies the server asks for (); null when it makes none. +public sealed record ClaimedJob(JobDto Job, Guid SequenceId, string SettingsJson, string PresetJson, bool UseProxies, IReadOnlyList Frames, + ProxySpec? Proxies = null); /// POST /api/v1/nodes/{id}/jobs/{jobId}/progress. public sealed record NodeProgress(long Done, long Total, double? ItemsPerSecond, string? LogLine); diff --git a/src/Titano.Protocol/Sequences.cs b/src/Titano.Protocol/Sequences.cs index 8ad09422..45a91f17 100644 --- a/src/Titano.Protocol/Sequences.cs +++ b/src/Titano.Protocol/Sequences.cs @@ -65,6 +65,7 @@ public sealed record FrameDto( long Flags); /// GET /api/v1/sequences/{id}/manifest: what a client needs to diff a sequence (BRIEF §14.5). +/// How to make the proxies the server asks for (); null when it makes none. public sealed record SequenceManifest( Guid SequenceId, int FramesVersion, @@ -72,9 +73,21 @@ public sealed record SequenceManifest( string? SettingsJson, IReadOnlyList Tags, IReadOnlyList Frames, - IReadOnlyList Renders); + IReadOnlyList Renders, + ProxySpec? Proxies = null); -public sealed record ManifestFrame(Guid Id, int Index, string FileName, long Size, string Sha256, bool HasProxy, bool HasThumb); +/// One frame of a manifest: its content, whether the server has a proxy and thumbnail of it, and its flags. +/// Lower-case hex SHA-256; empty while the server has not hashed it (a folder adopted and not verified yet). +/// The frame's FrameFlags bits (missing, corrupt, excluded…). +/// +/// The server cannot decode this original and has no proxy of it (BRIEF §2.3, §8.1: HEIF): a client or render node that +/// decodes it sends one with PUT /api/v1/frames/{id}/proxy, made as says. +/// +public sealed record ManifestFrame(Guid Id, int Index, string FileName, long Size, string Sha256, bool HasProxy, bool HasThumb, long Flags = 0, + bool WantsProxy = false); + +/// How the server's proxies are made (BRIEF §8.5): long edge in pixels (0 = full size) and JPEG quality. +public sealed record ProxySpec(int LongEdge, int Quality); public sealed record RenderDto( Guid Id, diff --git a/src/Titano.Server/Api/FileEndpoints.cs b/src/Titano.Server/Api/FileEndpoints.cs index 76cf182f..d7090edb 100644 --- a/src/Titano.Server/Api/FileEndpoints.cs +++ b/src/Titano.Server/Api/FileEndpoints.cs @@ -1,6 +1,12 @@ using System.Globalization; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.Mvc; +using Titano.Core.Archiving; +using Titano.Core.Imaging.Decoders; using Titano.Core.Model; using Titano.Data; +using Titano.Data.Archiving; +using Titano.Server.Jobs; using Titano.Server.Security; using Titano.Server.Storage; @@ -8,7 +14,8 @@ namespace Titano.Server.Api; /// /// Streamed archive files with Range support (BRIEF §14.3): originals, proxies, thumbnails and exports. Paths come -/// from the database by id and are resolved under the archive root only. +/// from the database by id and are resolved under the archive root only. Proxies of frames the server cannot decode +/// arrive from clients and render nodes (PUT /frames/{id}/proxy, BRIEF §2.3). /// public static class FileEndpoints { @@ -21,6 +28,8 @@ public static class FileEndpoints api.MapGet("/frames/{id:guid}/original", OriginalAsync).Requires(Permissions.Edit | Permissions.RenderNode); api.MapGet("/frames/{id:guid}/proxy", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Proxy, db, a, l)) .Requires(Permissions.Read | Permissions.RenderNode); + // Made by clients that edit and by render nodes, which decode what the server cannot. + api.MapPut("/frames/{id:guid}/proxy", StoreProxyAsync).Requires(Permissions.Edit | Permissions.RenderNode); api.MapGet("/frames/{id:guid}/thumb", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Thumbnail, db, a, l)) .Requires(Permissions.Read); api.MapGet("/renders/{id:guid}/file", RenderAsync).Requires(Permissions.Read); @@ -48,6 +57,55 @@ public static class FileEndpoints ?? ApiErrors.NotFound(context, "asset_missing"); } + /// + /// Files the body (a JPEG made from the original, its Exif capture time the frame's, as large as the archive's proxies, + /// at most ) as the proxy of a frame the server cannot decode + /// (); the sequence's thumbnails and proxy state follow as background jobs. 204; 413 + /// too_large; 422 proxy_not_wanted (the server decodes that original itself, or it is missing or + /// damaged), invalid_proxy, proxy_mismatch or proxy_too_small, which concern that proxy only; 409 + /// proxies_disabled, which concerns them all; 503 archive_offline. + /// + private static async Task StoreProxyAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive, + [FromServices] ArchiveService? archives, SequenceJobs follow) + { + var ct = context.RequestAborted; + if (await db.Frames.GetAsync(id, ct) is not { } frame || await db.Sequences.GetAsync(frame.SequenceId, ct) is not { } sequence) + return ApiErrors.NotFound(context, "frame"); + // A proxy replaces the server's own of an original it decodes, which renders and remote views would then show. + if (!ServerProxies.CannotDecode(frame.Format) || (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0) + return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_not_wanted"); + if (context.Request.ContentLength is > ArchiveImages.MaxProxyBytes) return ApiErrors.Result(context, StatusCodes.Status413PayloadTooLarge, "too_large"); + // Proxies are larger than the JSON bodies Kestrel is limited to; reading past this answers 413. + if (context.Features.Get() is { IsReadOnly: false } limit) limit.MaxRequestBodySize = ArchiveImages.MaxProxyBytes; + using var body = new MemoryStream(); + await context.Request.Body.CopyToAsync(body, ct); + + var images = (archives ?? new ArchiveService(db, new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = ArchiveModule.Owner })).Images; + ProxyStoreResult result; + try + { + result = images.StoreProxy(archive.Current, sequence, frame, body.ToArray()); + } + catch (ArchiveOfflineException) + { + return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline"); + } + switch (result) + { + case ProxyStoreResult.Disabled: + return ApiErrors.Conflict(context, "proxies_disabled"); + case ProxyStoreResult.Invalid: + return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "invalid_proxy"); + case ProxyStoreResult.WrongFrame: + return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_mismatch"); + case ProxyStoreResult.TooSmall: + return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_too_small"); + } + // The proxy is filed: its follow-up is queued even when the sender is gone by now. + await follow.ProxyArrivedAsync(sequence, CancellationToken.None); + return Results.NoContent(); + } + private static async Task RenderAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive) { if (await db.Renders.GetAsync(id, context.RequestAborted) is not { } render) return ApiErrors.NotFound(context, "render"); diff --git a/src/Titano.Server/Api/SequenceEndpoints.cs b/src/Titano.Server/Api/SequenceEndpoints.cs index c84b82ad..f3ec9993 100644 --- a/src/Titano.Server/Api/SequenceEndpoints.cs +++ b/src/Titano.Server/Api/SequenceEndpoints.cs @@ -213,29 +213,24 @@ public static class SequenceEndpoints _ => ApiErrors.Conflict(context, "folder_missing"), }; - /// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5); ETag over the version counters. - private static async Task ManifestAsync(HttpContext context, Guid id, TitanoDatabase db) + /// + /// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5), with the proxies the server asks + /// for (); ETag over the version counters and the number of proxies asked for. + /// + private static async Task ManifestAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive) { var ct = context.RequestAborted; if (await db.Sequences.GetAsync(id, ct) is not { } sequence) return ApiErrors.NotFound(context, "sequence"); - var etag = string.Create(CultureInfo.InvariantCulture, $"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}\""); + var spec = ServerProxies.Spec(archive.Current); + var frames = await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, ct); + var etag = string.Create(CultureInfo.InvariantCulture, + $"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}-{frames.Count(f => f.WantsProxy)}\""); if (context.Request.Headers.IfNoneMatch == etag) return Results.StatusCode(StatusCodes.Status304NotModified); - var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt); - var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt); - var frames = new List(sequence.FrameCount); - var after = -1; - while (true) - { - var batch = await db.Frames.ListBySequenceAsync(id, after, 1000, ct); - frames.AddRange(batch.Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb))); - if (batch.Count < 1000) break; - after = batch[^1].Index; - } var renders = await db.Renders.ListBySequenceAsync(id, ct); context.Response.Headers.ETag = etag; return Results.Json(new SequenceManifest(id, sequence.FramesVersion, sequence.SettingsVersion, sequence.SettingsJson, sequence.Tags, frames, - [.. renders.Select(Dto.Render)]), ProtocolJson.Default.SequenceManifest); + [.. renders.Select(Dto.Render)], spec), ProtocolJson.Default.SequenceManifest); } private static async Task FramesAsync(HttpContext context, Guid id, string? cursor, int? limit, TitanoDatabase db) diff --git a/src/Titano.Server/Jobs/RenderNodeService.cs b/src/Titano.Server/Jobs/RenderNodeService.cs index b9f22d6f..4bb8a0d8 100644 --- a/src/Titano.Server/Jobs/RenderNodeService.cs +++ b/src/Titano.Server/Jobs/RenderNodeService.cs @@ -16,11 +16,14 @@ namespace Titano.Server.Jobs; /// /// Render nodes (BRIEF §14.7): clients with the render-node right register their encoders, heartbeat, claim queued /// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of -/// the sequence. A node that stops heart-beating loses its jobs to the queue (). +/// the sequence. A claimed job lists every frame with its flags (a node renders none missing, corrupt or excluded) and +/// the proxies the server asks for (), which the node sends from the originals it fetched. +/// A node that stops heart-beating loses its jobs to the queue (). /// public sealed class RenderNodeService( TitanoDatabase db, ServerOptions options, + ServerArchive archive, UploadService uploads, UploadStore store, RenderStore renders, @@ -84,8 +87,9 @@ public sealed class RenderNodeService( continue; } await TouchAsync(node, cancellationToken); + var spec = ServerProxies.Spec(archive.Current); return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson, - parameters.UseProxies, await ManifestFramesAsync(sequence, cancellationToken)); + parameters.UseProxies, await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, cancellationToken), spec); } await TouchAsync(node, cancellationToken); return (ClaimedJob?)null; @@ -234,22 +238,6 @@ public sealed class RenderNodeService( if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state)); } - private async Task> ManifestFramesAsync(Sequence sequence, CancellationToken cancellationToken) - { - var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt); - var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt); - var frames = new List(sequence.FrameCount); - var after = -1; - while (true) - { - var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, 1000, cancellationToken); - frames.AddRange(batch.Where(f => !f.Flags.HasFlag(FrameFlags.Excluded)) - .Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb))); - if (batch.Count < 1000) return frames; - after = batch[^1].Index; - } - } - /// /// The node has an encoder for the job's codec that handles its resolution: the encoder's own limit, else the node's /// (0 = no announced limit). diff --git a/src/Titano.Server/Jobs/SequenceJobs.cs b/src/Titano.Server/Jobs/SequenceJobs.cs index f53c962f..d1743b04 100644 --- a/src/Titano.Server/Jobs/SequenceJobs.cs +++ b/src/Titano.Server/Jobs/SequenceJobs.cs @@ -4,21 +4,27 @@ using Titano.Data; namespace Titano.Server.Jobs; /// -/// Follow-up work after frames arrive (BRIEF §14.4): thumbnails, proxies and light analysis of the sequence, queued at -/// background priority once per sequence while one is still waiting, and only for kinds this server can run. +/// Follow-up work after frames arrive (BRIEF §14.4): thumbnails, proxies and light analysis of the sequence; after a +/// proxy sent by a client or render node arrives, the thumbnails and the sequence's proxy state. Queued at background +/// priority once per sequence while one is still waiting, and only for kinds this server can run. /// public sealed class SequenceJobs(TitanoDatabase db, JobEngineHost jobs) { private static readonly JobKind[] AfterNewFrames = [JobKind.Thumbs, JobKind.Proxies, JobKind.LightAnalyse]; + private static readonly JobKind[] AfterProxy = [JobKind.Thumbs, JobKind.Proxies]; private readonly SemaphoreSlim _gate = new(1, 1); - public async Task FramesAddedAsync(Sequence sequence, CancellationToken cancellationToken) + public Task FramesAddedAsync(Sequence sequence, CancellationToken cancellationToken) => QueueAsync(sequence, AfterNewFrames, cancellationToken); + + public Task ProxyArrivedAsync(Sequence sequence, CancellationToken cancellationToken) => QueueAsync(sequence, AfterProxy, cancellationToken); + + private async Task QueueAsync(Sequence sequence, JobKind[] kinds, CancellationToken cancellationToken) { await _gate.WaitAsync(cancellationToken); try { - foreach (var kind in AfterNewFrames) + foreach (var kind in kinds) { if (!jobs.CanRun(kind) || await db.Jobs.IsQueuedAsync(kind, sequence.Id, cancellationToken)) continue; await jobs.Engine.EnqueueAsync(kind, [sequence.Id], sequence.DisplayTitle, priority: JobPriority.Background, cancellationToken: cancellationToken); diff --git a/src/Titano.Server/Storage/ArchiveModule.cs b/src/Titano.Server/Storage/ArchiveModule.cs index bc58510a..ad5bebc2 100644 --- a/src/Titano.Server/Storage/ArchiveModule.cs +++ b/src/Titano.Server/Storage/ArchiveModule.cs @@ -7,8 +7,9 @@ namespace Titano.Server.Storage; /// /// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and -/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance). Decoding is -/// managed only (BRIEF §4.2). The daily backup of an SQLite index runs with it (). +/// thumbnails, its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance) and the periodic +/// incremental scan (). Decoding is managed only (BRIEF §4.2). The daily backup of an +/// SQLite index runs with it (). /// public static class ArchiveModule { @@ -23,5 +24,7 @@ public static class ArchiveModule services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); services.AddHostedService(); + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); } } diff --git a/src/Titano.Server/Storage/ArchivePolling.cs b/src/Titano.Server/Storage/ArchivePolling.cs new file mode 100644 index 00000000..fe5bb2e4 --- /dev/null +++ b/src/Titano.Server/Storage/ArchivePolling.cs @@ -0,0 +1,68 @@ +using Titano.Core.Archiving; +using Titano.Core.Model; +using Titano.Data; +using Titano.Data.Archiving; +using Titano.Server.Jobs; + +namespace Titano.Server.Storage; + +/// +/// Polls the archive (BRIEF §7: by modification time and size, no file-system watchers): an incremental scan at start +/// and every while the archive is online, so photos already in an adopted folder and sequences +/// another writer adds (a desktop sharing the archive, files copied in by hand) are indexed without an administrator's +/// rescan. A scan or rebuild of the archive that is waiting, running or paused is not queued again. +/// +public sealed class ArchivePolling(TitanoDatabase db, ServerArchive archive, JobEngineHost jobs, TimeProvider time, ILogger logger) + : BackgroundService +{ + /// The desktop's rhythm (it polls its archive the same way). + public static readonly TimeSpan Interval = TimeSpan.FromMinutes(10); + + private static readonly JobState[] Pending = [JobState.Queued, JobState.Running, JobState.Paused]; + + /// Queues an incremental scan of an online archive unless one is pending; returns it, or null when none was queued. + public async Task PollAsync(CancellationToken cancellationToken) + { + if (!jobs.CanRun(JobKind.Scan)) return null; + var current = archive.Current; + if (ArchiveService.GetAvailability(current) != ArchiveAvailability.Online) return null; + foreach (var kind in new[] { JobKind.Scan, JobKind.Rebuild }) + { + foreach (var state in Pending) + { + if ((await db.Jobs.ListAsync(state, kind, 50, null, cancellationToken)).Items.Any(j => j.TargetIds.Contains(current.Id))) return null; + } + } + var scan = ArchiveJobs.Scan(current); + // Behind what users asked for (BRIEF §14.6). + return await jobs.Engine.EnqueueAsync(scan.Kind, scan.Targets, scan.Label, scan.ParametersJson, JobPriority.Background, + cancellationToken: cancellationToken); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (true) + { + try + { + await PollAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + logger.LogWarning(ex, "The archive could not be polled"); + } + try + { + await Task.Delay(Interval, time, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + } + } +} diff --git a/src/Titano.Server/Storage/ServerProxies.cs b/src/Titano.Server/Storage/ServerProxies.cs new file mode 100644 index 00000000..34e73cc3 --- /dev/null +++ b/src/Titano.Server/Storage/ServerProxies.cs @@ -0,0 +1,62 @@ +using Titano.Core.Archiving; +using Titano.Core.Imaging; +using Titano.Core.Model; +using Titano.Data; +using Titano.Data.Archiving; +using Titano.Protocol; + +namespace Titano.Server.Storage; + +/// +/// Proxies the server asks for (BRIEF §2.3, §8.5): it decodes with managed decoders only, so the frames whose originals +/// it cannot decode (HEIF) get their proxies from a Windows client or render node, made as the archive's proxy policy +/// says. Manifests (sync and render nodes) list every frame with its flags and whether its proxy is wanted. +/// +public static class ServerProxies +{ + private const int Page = 1000; + + /// The server takes this format's pixels from proxies made elsewhere (its decoders are managed only, BRIEF §4.2). + public static bool CannotDecode(ImageFileFormat format) => ArchiveImages.ServerNeedsProxy(format); + + /// How proxies of the archive are made; null when it makes none (or its manifest cannot be read now). + public static ProxySpec? Spec(Archive archive) + { + try + { + var policy = ManifestStore.ReadArchive(new ArchivePaths(archive.RootPath).ManifestPath).ProxyPolicy; + return policy.LongEdge == ProxyPolicy.Disabled ? null : new ProxySpec(policy.LongEdge, policy.Quality); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) + { + return null; + } + } + + /// + /// Every frame of in index order, as a manifest lists it: a proxy is there when the server + /// built them all or one was sent for the frame; it is wanted for a frame the server cannot decode that has none + /// (and is not missing), unless is null. + /// + public static async Task> FramesAsync(TitanoDatabase db, Archive archive, Sequence sequence, ProxySpec? spec, + CancellationToken cancellationToken) + { + var proxies = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt); + var thumbs = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt); + var folder = new ArchivePaths(archive.RootPath).Sequence(sequence.Folder); + var frames = new List(sequence.FrameCount); + var after = -1; + while (true) + { + var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, Page, cancellationToken); + foreach (var f in batch) + { + var sent = !proxies && CannotDecode(f.Format) && File.Exists(folder.Proxy(f.Index)); + var wanted = spec is not null && !proxies && !sent && CannotDecode(f.Format) && (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0; + frames.Add(new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, proxies || sent, thumbs, (long)f.Flags, wanted)); + } + if (batch.Count < Page) return frames; + after = batch[^1].Index; + } + } +} diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs index 87a0215a..e73e5444 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs @@ -150,6 +150,38 @@ public sealed class ArchiveIntegrityTests(TestContext context) Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease")); Assert.True(folder.Frames().All(File.Exists)); } + + [Test] + public async Task PackCompactionWaitsForTheThumbnailWriter() + { + await using var fx = await CreateAsync(context); + var id = await IngestSeriesAsync(fx, "a", 2, 1); + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); + var hash = new byte[ThumbPack.HashLength]; + using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath)) + { + // The same entry written again and again: everything but the last copy is garbage. + for (var i = 0; i < 4; i++) pack.Append(0, 48, 32, hash, Photos.Jpeg(Start, 70 + i)); + pack.Commit(); + } + var length = new FileInfo(folder.ThumbsPackPath).Length; + + // A thumbnail pass holds the pack's own lease (not the sequence's) while it appends. + Task maintenance; + using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{id:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) + { + maintenance = fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); + await Task.Delay(500, fx.Ct); + Assert.False(maintenance.IsCompleted); + Assert.Equal(length, new FileInfo(folder.ThumbsPackPath).Length); + } + + var report = await maintenance; + Assert.SequenceEqual([folder.RelativeFolder], report.CompactedPacks); + using var compacted = ThumbPackReader.Open(folder.ThumbsPackPath); + Assert.Equal(0L, compacted.GarbageBytes); + Assert.NotNull(compacted.Read(0)); + } } internal static class SequencePathsExtensions diff --git a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs index 5cf3130a..d15b5036 100644 --- a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs +++ b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs @@ -105,6 +105,93 @@ public sealed class MassRenameTests(TestContext context) Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder))); } + /// "trips" with photos of its own, "trips/day2" inside it and "trips/day2/dawn" inside that: three nested sequences. + private static async Task> NestedAsync(ArchiveFixture fx) + { + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips"), Start, 2); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips", "day2"), Start.AddDays(1), 2, firstSeed: 10); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips", "day2", "dawn"), Start.AddDays(2), 2, firstSeed: 20); + await fx.ScanAsync(); + var locations = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).OrderBy(l => l.Folder, StringComparer.Ordinal).ToList(); + Assert.SequenceEqual(["trips", "trips/day2", "trips/day2/dawn"], locations.Select(l => l.Folder)); + return [.. locations.Select(l => l.Id)]; + } + + [Test] + public async Task SequencesInsideAMovedFolderAreInTheDryRunAndMoveWithIt() + { + await using var fx = await CreateAsync(context); + var ids = await NestedAsync(fx); + var before = await StateAsync(fx, ids); + + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "moved/{date}", null, fx.Ct); + Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-28/day2", "moved/2026-09-28/day2/dawn"], plan.Steps.Select(s => s.ToFolder)); + Assert.SequenceEqual([false, true, true], plan.Steps.Select(s => s.Carried)); + Assert.SequenceEqual(before, await StateAsync(fx, ids)); + + await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).MissingSequences); + } + + [Test] + public async Task AFolderMovesOnceTheWritersOfTheSequencesInsideAreDone() + { + await using var fx = await CreateAsync(context); + var ids = await NestedAsync(fx); + var before = await StateAsync(fx, ids); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "moved/{date}", null, fx.Ct); + + // An ingest into trips/day2 holds that sequence's lease; a thumbnail pass of trips/day2/dawn holds its pack's. + Task apply; + using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{ids[1]:N}", "ingest", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) + using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{ids[2]:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) + { + apply = fx.Service.ApplyRenameAsync(plan, null, fx.Ct); + await Task.Delay(500, fx.Ct); + Assert.False(apply.IsCompleted); + Assert.True(Directory.Exists(Path.Combine(fx.Paths.Root, "trips", "day2", "dawn")), "nothing moved while they write"); + } + + await apply; + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + } + + [Test] + public async Task NestedSequencesMovedTogetherLeaveTheirFolderFirst() + { + await using var fx = await CreateAsync(context); + var ids = await NestedAsync(fx); + var before = await StateAsync(fx, ids); + + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "moved/{date}", null, fx.Ct); + Assert.SequenceEqual([ids[2], ids[1], ids[0]], plan.Steps.Select(s => s.SequenceId)); + Assert.True(plan.Steps.All(s => !s.Carried)); + + await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); + Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-29", "moved/2026-09-30"], (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)) + .Select(l => l.Folder).Order(StringComparer.Ordinal)); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } + + [Test] + public async Task AFolderInsideASequenceFolderIsRefused() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var before = await StateAsync(fx, ids); + + var into = await Assert.ThrowsAsync(() => fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[1]], "inbox/a/{date}", null, fx.Ct)); + Assert.Equal("inbox/a", into.Folder); + var itself = await Assert.ThrowsAsync(() => fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "inbox/a/{date}", null, fx.Ct)); + Assert.Equal("inbox/a/2026-09-28", itself.Target); + Assert.SequenceEqual(before, await StateAsync(fx, ids)); + } + /// Reports synchronously (Progress<T> would post to the thread pool). private sealed class SyncProgress(Action report) : IProgress { diff --git a/tests/Titano.Data.Tests/Archiving/ProxyTests.cs b/tests/Titano.Data.Tests/Archiving/ProxyTests.cs new file mode 100644 index 00000000..0fc493fa --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ProxyTests.cs @@ -0,0 +1,163 @@ +using Titano.Core.Archiving; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Decoders; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +/// Proxies of originals a host cannot decode (BRIEF §2.3, §8.5): sent by a client or render node, checked, kept, thumbnailed. +[Category("archive")] +public sealed class ProxyTests(TestContext context) +{ + [Test] + public void OnlyFormatsAHostCannotDecodeTakeTheirPixelsFromProxies() + { + var server = DecoderFactory.CreateServer(); + Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Heif), "HEIF previews are thumbnails at best"); + Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Avif)); + Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.WebP), "no decoder at all"); + Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Cr3), "a RAW's embedded full-size preview"); + Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Dng)); + Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([]), ImageFileFormat.Jpeg), "proxies are JPEGs"); + Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Unknown)); + // A desktop whose decoders read HEIF makes the proxies a server asks for. + Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([new NativeHeif()]), ImageFileFormat.Heif)); + } + + [Test] + public async Task AProxySentForAFrameThisHostCannotDecodeIsCheckedKeptAndThumbnailed() + { + await using var fx = await CreateAsync(context); + var (sequence, frames) = await HeifSequenceAsync(fx); + var folder = fx.Paths.Sequence(sequence.Folder); + var images = fx.Service.Images; + + Assert.Equal(ProxyStoreResult.Invalid, images.StoreProxy(fx.Archive, sequence, frames[0], "not a photo"u8.ToArray())); + Assert.Equal(ProxyStoreResult.WrongFrame, images.StoreProxy(fx.Archive, sequence, frames[0], Proxy(frames[1].CapturedAt!.Value, 1))); + Assert.FileMissing(folder.Proxy(0)); + var proxy = Proxy(frames[0].CapturedAt!.Value, 2); + Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frames[0], proxy)); + Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct)); + + // The proxy pass keeps what was sent and makes none from the originals here, even when forced. + var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: true, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((0, 1, 1), (proxies.Built, proxies.Reused, proxies.Undecodable)); + Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct)); + Assert.FileMissing(folder.Proxy(1)); + + // The thumbnail comes from the proxy; the frame without one has none yet. + var thumbs = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((1, 0, 1), (thumbs.Built, thumbs.Reused, thumbs.Undecodable)); + using (var pack = ThumbPackReader.Open(folder.ThumbsPackPath)) + { + var entry = pack.Entries[0]; + Assert.Equal((FrameImageRenderer.ThumbnailLongEdge, 240), (entry.Width, entry.Height)); + } + var again = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable)); + } + + [Test] + public async Task AProxyLeftByARenumberedFrameIsAskedForAgain() + { + await using var fx = await CreateAsync(context); + var (sequence, frames) = await HeifSequenceAsync(fx); + var folder = fx.Paths.Sequence(sequence.Folder); + Directory.CreateDirectory(folder.Proxies); + await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[0].CapturedAt!.Value, 3), fx.Ct); + + var proxies = await fx.Service.Images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + + Assert.Equal(2, proxies.Undecodable); + Assert.FileMissing(folder.Proxy(1)); + Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt)); + } + + [Test] + public async Task AProxyMadeFromAPreviewIsAskedForAgain() + { + await using var fx = await CreateAsync(context); + var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024); + var folder = fx.Paths.Sequence(sequence.Folder); + var images = fx.Service.Images; + // An earlier version made the frame's proxy here from the original's embedded preview: the frame's capture time, + // a thumbnail's size. + Directory.CreateDirectory(folder.Proxies); + await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 5, 160, 120), fx.Ct); + await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, SequenceStatus.ProxiesBuilt, SequenceStatus.None, fx.Ct); + + var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + + Assert.Equal((0, 0, 2), (proxies.Built, proxies.Reused, proxies.Undecodable)); + Assert.FileMissing(folder.Proxy(0)); + Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt), "asked for again"); + + // A client or render node sending one that small is refused; one of the policy's size (1920 px) is kept. + var frame = frames[0] with { Width = 4032, Height = 3024 }; + Assert.Equal(ProxyStoreResult.TooSmall, images.StoreProxy(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 6, 160, 120))); + Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 7, 1920, 1440))); + var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable)); + Assert.FileExists(folder.Proxy(0)); + } + + [Test] + public async Task AnArchiveWithoutProxiesRefusesThem() + { + await using var fx = await CreateAsync(context); + var (sequence, frames) = await HeifSequenceAsync(fx); + var manifest = ManifestStore.ReadArchive(fx.Paths.ManifestPath); + ManifestStore.WriteArchive(fx.Paths.ManifestPath, manifest with { ProxyPolicy = new ProxyPolicy { LongEdge = ProxyPolicy.Disabled } }); + + Assert.Equal(ProxyStoreResult.Disabled, fx.Service.Images.StoreProxy(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 4))); + Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).Proxy(0)); + } + + /// + /// Two ingested frames whose originals the archive engine takes for HEIF (its decoders read none), their size in the + /// sidecar × when given. + /// + private static async Task<(Sequence Sequence, IReadOnlyList Frames)> HeifSequenceAsync(ArchiveFixture fx, int? width = null, int? height = null) + { + Guid id = default; + for (var i = 0; i < 2; i++) + { + var staged = fx.Stage(Start.AddSeconds(10 * i), 40 + i); + id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "heif", staged.Path, $"IMG_{i:0000}.HEIC", + staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId; + } + var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var sidecar = ManifestStore.ReadSequence(folder.ManifestPath); + ManifestStore.WriteSequence(folder.ManifestPath, sidecar with + { + Frames = [.. sidecar.Frames.Select(f => f with { Format = ImageFileFormat.Heif, W = width ?? f.W, H = height ?? f.H })], + }); + return (sequence, await fx.Db.Frames.ListBySequenceAsync(id)); + } + + /// A JPEG proxy (320 × 240 unless sized) with in its Exif, as a client makes it. + private static byte[] Proxy(DateTime captured, int seed, int width = 320, int height = 240) + { + using var image = new ImageBuffer(width, height); + var level = 0.2f + seed % 5 * 0.1f; + for (var c = 0; c < 3; c++) image.Plane(c).Fill(level + c * 0.05f); + var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Titano" }); + return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); + } + + /// A provider that decodes HEIF natively (WIC with the HEIF extension on a desktop). + private sealed class NativeHeif : IImageDecoderProvider + { + public string Name => "native-heif"; + + public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null; + + public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null; + } +} diff --git a/tests/Titano.Data.Tests/Processing/RenderJobTests.cs b/tests/Titano.Data.Tests/Processing/RenderJobTests.cs index 656032ef..f0a0aa41 100644 --- a/tests/Titano.Data.Tests/Processing/RenderJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/RenderJobTests.cs @@ -63,6 +63,33 @@ public sealed class RenderJobTests(TestContext context) Assert.Equal(3, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount); } + [Test] + public async Task FramesThisHostCannotDecodeRenderFromTheirProxies() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Scene()); + var folder = fx.Folder(sequence); + Directory.CreateDirectory(folder.Proxies); + foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)) + { + // The originals are HEIF this host cannot read (BRIEF §2.3): only the proxies a client sent have pixels here. + var original = fx.Archive.Paths.FromRelative(frame.RelativePath); + File.Copy(original, folder.Proxy(frame.Index)); + await File.WriteAllBytesAsync(original, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8], fx.Ct); + } + await using (var session = await fx.Db.Database.OpenAsync(fx.Ct)) + await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE sequence_id = $2", [(int)ImageFileFormat.Heif, sequence.Id], fx.Ct); + await fx.StartEngineAsync(); + + // The job does not ask for proxies: the host takes them because it cannot decode the originals. + var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64)); + await fx.WaitForAsync(job.Id, JobState.Done); + + Assert.FileExists(Path.Combine(folder.Exports, "clip-MJPEG.avi")); + var decoded = fx.Decoder.Requests.Where(r => r.Purpose == DecodePurpose.Render).Select(r => Path.GetDirectoryName(r.Path)).Distinct().ToList(); + Assert.SequenceEqual([folder.Proxies], decoded); + } + [Test] public async Task ACancelledRenderLeavesNothingBehind() { diff --git a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs index c05213d4..275565b2 100644 --- a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs +++ b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs @@ -38,6 +38,9 @@ internal sealed class FakeSyncRemote : ISyncRemote /// Proxy and thumbnail downloads fail with a server error. public bool FailAssets { get; set; } + /// Proxies clients sent, by frame id (for sequences the server cannot decode). + public Dictionary SentProxies { get; } = []; + public FakeSequence Add(string? title, IEnumerable originals, string? settingsJson = null, SequenceStatus status = SequenceStatus.ProxiesBuilt | SequenceStatus.ThumbsBuilt) { @@ -96,7 +99,7 @@ internal sealed class FakeSyncRemote : ISyncRemote { ManifestCalls++; if (Refused.Contains(id)) throw new TitanoApiException(HttpStatusCode.Forbidden, new ApiError("forbidden", "refused")); - return Task.FromResult(Find(id).Manifest()); + return Task.FromResult(Find(id).Manifest(SentProxies.ContainsKey)); } } @@ -171,6 +174,18 @@ internal sealed class FakeSyncRemote : ISyncRemote await destination.WriteAsync(bytes.AsMemory(offset), cancellationToken); } + /// Like the server: only a frame whose proxy it asks for takes one, and only a JPEG. + public Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken cancellationToken) + { + lock (_gate) + { + if (!Sequences.Values.Any(s => s.Undecodable && s.Frames.Any(f => f.Id == frameId))) throw NotFound(); + if (jpeg is not [0xFF, 0xD8, ..]) throw new TitanoApiException(HttpStatusCode.UnprocessableEntity, new ApiError("invalid_proxy", "not a JPEG")); + SentProxies[frameId] = jpeg; + } + return Task.CompletedTask; + } + public async Task DownloadRenderAsync(Guid renderId, Stream destination, CancellationToken cancellationToken) { FakeRender render; @@ -225,6 +240,10 @@ internal sealed class FakeSequence public int SettingsVersion { get; set; } = 1; public int FramesVersion { get; set; } public SequenceStatus Status { get; set; } + + /// The server cannot decode these originals (HEIF): its manifest asks for proxies of them. + public bool Undecodable { get; set; } + public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow; public DateTime? FirstCapture { get; set; } public string? CameraModel { get; set; } = "Canon EOS R5"; @@ -243,9 +262,10 @@ internal sealed class FakeSequence public SequenceDto Dto() => new(Id, Title, Title ?? "Untitled", "2026/" + Id.ToString("N"), "Canon", CameraModel, null, FirstCapture, null, Frames.Count, 10, Frames.Sum(f => (long)f.Bytes.Length), (long)Status, Renders.Count, null, Tags, Rating, Notes, SettingsVersion, FramesVersion, 0, null, UpdatedAt); - public SequenceManifest Manifest() => new(Id, FramesVersion, SettingsVersion, SettingsJson, Tags, - [.. Frames.Select((f, i) => new ManifestFrame(f.Id, i, f.FileName, f.Bytes.Length, f.Sha256, (Status & SequenceStatus.ProxiesBuilt) != 0, - (Status & SequenceStatus.ThumbsBuilt) != 0))], [.. Renders.Select(r => r.Dto())]); + public SequenceManifest Manifest(Func proxySent) => new(Id, FramesVersion, SettingsVersion, SettingsJson, Tags, + [.. Frames.Select((f, i) => new ManifestFrame(f.Id, i, f.FileName, f.Bytes.Length, f.Sha256, (Status & SequenceStatus.ProxiesBuilt) != 0 || proxySent(f.Id), + (Status & SequenceStatus.ThumbsBuilt) != 0, WantsProxy: Undecodable && !proxySent(f.Id)))], [.. Renders.Select(r => r.Dto())], + Undecodable ? new ProxySpec(1280, 90) : null); } internal sealed record FakeRender(Guid Id, string FileName, string PresetName, byte[] Bytes) diff --git a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs index f2b7fa91..da763383 100644 --- a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs @@ -1,5 +1,8 @@ +using Titano.Core.Imaging; +using Titano.Core.Imaging.Decoders; using Titano.Core.Model; using Titano.Data.Sync; +using Titano.Protocol; using Titano.Testing; namespace Titano.Data.Tests.Sync; @@ -63,6 +66,38 @@ public sealed class SyncRulesTests Assert.Equal((1, 0), SyncRules.CompareOriginals([null, null], ["bb"])); } + [Test] + public void ProxiesTheServerAsksForAreCountedForOriginalsThisPcReads() + { + var heif = new ImageDecoder([new NativeHeif()]); + List local = + [ + Original("aa", 100, ImageFileFormat.Heif), + Original(null, 200, ImageFileFormat.Heif), + Original(null, 300, ImageFileFormat.Heif), + Original("dd", 400, ImageFileFormat.Avif), + ]; + var manifest = new SequenceManifest(Guid.CreateVersion7(), 1, 1, null, [], + [ + Wanted("AA", 100, "2026-10-01_0001.heic"), + // Not hashed here: matched by size, whatever name the server gave its copy. + Wanted("bb", 200, "2026-10-01_0002.heic"), + Wanted("cc", 999, "IMG_0003.HEIC"), + // The server has no hash of it yet: nothing tells which original it is. + Wanted(string.Empty, 300, "IMG_0004.HEIC"), + // This PC cannot decode AVIF. + Wanted("dd", 400, "IMG_0005.AVIF"), + Wanted("ee", 200, "IMG_0006.HEIC") with { WantsProxy = false }, + ], [], new ProxySpec(1920, 90)); + + Assert.Equal(2, SyncRules.ProxiesToSend(manifest, local, heif)); + Assert.Equal(0, SyncRules.ProxiesToSend(manifest, local, DecoderFactory.CreateServer())); + + Assert.True(SyncRules.MakesProxiesFor(local, heif), "HEIF the server cannot decode and this PC can"); + Assert.False(SyncRules.MakesProxiesFor(local, DecoderFactory.CreateServer()), "a PC that reads no more than the server"); + Assert.False(SyncRules.MakesProxiesFor([Original("aa", 100, ImageFileFormat.Jpeg)], heif), "the server decodes JPEG itself"); + } + [Test] public void SettingsAreComparedAsProjectsNotAsText() { @@ -92,4 +127,29 @@ public sealed class SyncRulesTests Assert.True(mine.SameMetadata(mine with { Tags = ["HARBOUR", "sunset"], Notes = "mine " }), "tags ignore case and order, blanks are trimmed"); Assert.Empty(SyncRules.Differences(mine, mine with { Tags = ["harbour", "SUNSET"] })); } + + private static Frame Original(string? sha, long size, ImageFileFormat format) => new() + { + Id = Guid.CreateVersion7(), + SequenceId = Local.Id, + Index = (int)(size / 100), + RelativePath = $"2026/a/{size}", + FileName = $"{size}", + Size = size, + Sha256 = sha, + Format = format, + }; + + private static ManifestFrame Wanted(string sha, long size, string name) => + new(Guid.CreateVersion7(), (int)(size / 100), name, size, sha, HasProxy: false, HasThumb: true, WantsProxy: true); + + /// A provider that decodes HEIF natively (WIC with the HEIF extension on a desktop). + private sealed class NativeHeif : IImageDecoderProvider + { + public string Name => "native-heif"; + + public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null; + + public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null; + } } diff --git a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs index 0e28a1e7..f3ed537a 100644 --- a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs @@ -94,6 +94,31 @@ public sealed class SyncServiceTests(TestContext context) Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } + [Test] + public async Task AnEditSavedHereDuringTheComparisonIsNotReplacedByTheServers() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(1); + var local = await f.LocalAsync("Harbour", photos); + var remote = f.Remote.Add("Harbour", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + + // The server's copy changes; the editor saves this PC's edit after the comparison read the local copy, before it + // takes the server's version. + f.Remote.EditSettings(remote.Id, SyncFixture.Window33); + var racing = new RacingRemote(f.Remote, () => f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41).GetAwaiter().GetResult(), + beforeManifest: true); + await f.Sync.RefreshAsync(racing, new SyncPolicy(), f.Ct); + + var entry = await f.EntryAsync("Harbour"); + Assert.Equal(SyncStatus.Conflict, entry.Status); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local!.SettingsJson), "this PC's edit is kept"); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(entry.Local.Folder).ProjectPath, f.Ct))); + Assert.True(SyncRules.SameSettings(SyncFixture.Window33, entry.Link!.ConflictRemote!.SettingsJson), "the user picks"); + Assert.True(SyncRules.SameSettings(SyncFixture.Window33, remote.SettingsJson)); + } + [Test] public async Task WithoutAutomaticSyncAnEditWaitsForTheUser() { @@ -432,8 +457,11 @@ public sealed class SyncServiceTests(TestContext context) return (local, remote); } - /// A remote where someone else edits the settings right after the first read of a sequence, or right before the first edit. - private sealed class RacingRemote(FakeSyncRemote inner, Action race, bool beforeUpdate = false) : ISyncRemote + /// + /// A remote where someone edits right after the first read of a sequence, right before the first edit, or right + /// before the first manifest is read. + /// + private sealed class RacingRemote(FakeSyncRemote inner, Action race, bool beforeUpdate = false, bool beforeManifest = false) : ISyncRemote { private Action? _race = race; @@ -442,12 +470,17 @@ public sealed class SyncServiceTests(TestContext context) public async Task GetSequenceAsync(Guid id, CancellationToken cancellationToken) { var dto = await inner.GetSequenceAsync(id, cancellationToken); - if (!beforeUpdate) Interlocked.Exchange(ref _race, null)?.Invoke(); + if (!beforeUpdate && !beforeManifest) Interlocked.Exchange(ref _race, null)?.Invoke(); return dto; } public Task> ListSequencesAsync(CancellationToken cancellationToken) => inner.ListSequencesAsync(cancellationToken); - public Task GetManifestAsync(Guid id, CancellationToken cancellationToken) => inner.GetManifestAsync(id, cancellationToken); + public Task GetManifestAsync(Guid id, CancellationToken cancellationToken) + { + if (beforeManifest) Interlocked.Exchange(ref _race, null)?.Invoke(); + return inner.GetManifestAsync(id, cancellationToken); + } + public Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken) => inner.ListFramesAsync(sequenceId, cancellationToken); public Task UpdateSequenceAsync(Guid id, UpdateSequenceRequest request, CancellationToken cancellationToken) { diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs index e4f523c5..174aa73e 100644 --- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs @@ -1,5 +1,7 @@ +using Titano.Core.Codecs.Jpeg; using Titano.Core.Imaging; using Titano.Core.Imaging.Decoders; +using Titano.Core.Localization; using Titano.Core.Model; using Titano.Core.Render; using Titano.Data.Sync; @@ -32,6 +34,87 @@ public sealed class SyncTransferTests(TestContext context) Assert.Equal(remote.Id, entry.Link!.RemoteId); } + [Test] + public async Task AnUploadSendsTheProxiesTheServerCannotMakeItself() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Holy grail", photos); + // The server has the originals already (the Uploader sent them) but cannot decode them (HEIF, BRIEF §2.3). + var remote = f.Remote.Add("Holy grail", photos); + remote.Undecodable = true; + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var ahead = await f.EntryAsync("Holy grail"); + Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); + + Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); + foreach (var proxy in f.Remote.SentProxies.Values) + { + var info = JpegDecoder.ReadInfo(proxy); + Assert.True(info.Width <= 1280 && info.Height <= 1280, "made as the server's policy says"); + } + Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status); + } + + [Test] + public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + // An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds + // under the names it gave them, and cannot decode (HEIF, BRIEF §2.3). + var folder = Path.Combine(f.Archive.Paths.Root, "Adopted"); + Directory.CreateDirectory(folder); + for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct); + await f.Archive.ScanAsync(); + var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Titano.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted"); + Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed"); + var remote = f.Remote.Add("Adopted", photos); + remote.Undecodable = true; + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var ahead = await f.EntryAsync(local.DisplayTitle); + Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); + + Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); + Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status); + } + + [Test] + public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Holy grail", photos); + var remote = f.Remote.Add("Holy grail", photos); + remote.Undecodable = true; + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote); + var calls = f.Remote.ManifestCalls; + await f.RefreshAsync(); + Assert.Equal(calls, f.Remote.ManifestCalls); + + // A render node sends them after a job: the server's frame counters stand still, its sequence changes. + foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy; + remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1); + await f.RefreshAsync(); + + var entry = await f.EntryAsync("Holy grail"); + Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote)); + } + [Test] public async Task AResumedUploadSendsOnlyWhatTheServerLacks() { diff --git a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs index c32fecf2..6f417329 100644 --- a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs +++ b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs @@ -1,10 +1,14 @@ +using System.Collections.Concurrent; using System.Text; using Microsoft.Extensions.DependencyInjection; +using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; using Titano.Core.Diagnostics.Synthetic; using Titano.Core.Export; +using Titano.Core.Imaging; using Titano.Core.Imaging.Decoders; using Titano.Core.Localization; +using Titano.Core.Metadata; using Titano.Core.Model; using Titano.Core.Project; using Titano.Data.Archiving; @@ -57,6 +61,45 @@ public sealed class RenderNodeClientTests(TestContext context) Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed"); } + [Test] + public async Task ANodeSkipsMissingFramesFetchesUnhashedOnesAndSendsTheProxiesAskedFor() + { + await using var server = await TestServer.StartAsync(context); + var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); + var sequenceId = await UploadSequenceAsync(editor, Photos()[..3]); + await Uploads.UploadAsync(editor, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, 9)], "IMG_0004.HEIC", sequenceId); + var frames = await server.Database.Frames.ListBySequenceAsync(sequenceId); + // Frame 1 is gone from the server, frame 2 was adopted and never hashed, frame 3 is damaged (the server could not + // decode it) and frame 4 is a HEIF the server cannot decode. + File.Delete(Path.Combine(server.Options.ArchivePath, frames[0].RelativePath)); + await server.Database.Frames.UpdateFlagsAsync([frames[0].Id], FrameFlags.Missing, FrameFlags.None); + await server.Database.Frames.UpdateFlagsAsync([frames[2].Id, frames[3].Id], FrameFlags.Undecodable, FrameFlags.None); + await using (var session = await server.Database.Database.OpenAsync(context.Cancellation)) + { + await session.ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE id = $1", [frames[1].Id], context.Cancellation); + await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE id = $2", [(int)ImageFileFormat.Heif, frames[3].Id], context.Cancellation); + } + var job = await CreateRenderAsync(server, editor, sequenceId); + + var requests = new RecordPaths(new SocketsHttpHandler()); + using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", requests) { Token = token }; + // This PC decodes HEIF (WIC with the HEIF extension on Windows). + await using var node = Node(client, context.CreateTempDirectory("node"), decoder: new ImageDecoder([new SolidHeif(), new ManagedDecoderProvider()])); + Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); + + var done = await JobApiTests.GetAsync(editor, job.Id); + Assert.Equal("done", done.State, done.Error); + Assert.DoesNotContain($"/api/v1/frames/{frames[0].Id}/original", requests.Paths); + Assert.Contains($"/api/v1/frames/{frames[1].Id}/original", requests.Paths); + // The damaged frame stays out, as in the server's own render; the HEIF renders from its original here. + Assert.DoesNotContain($"/api/v1/frames/{frames[2].Id}/original", requests.Paths); + Assert.Contains($"/api/v1/frames/{frames[3].Id}/original", requests.Paths); + var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!; + var proxy = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder).Proxy(frames[3].Index); + Assert.Equal(frames[3].CapturedAt, MetadataReader.Read(proxy).CapturedAt, "the proxy the server asked for, with the frame's capture time"); + Assert.Equal((96, 64), MetadataReader.Read(proxy).OrientedSize); + } + [Test] public async Task ANodeReportsWhyItCouldNotRender() { @@ -263,8 +306,8 @@ public sealed class RenderNodeClientTests(TestContext context) Assert.FileExists(part[..^".part".Length]); } - private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null) => - new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } }, + private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null, ImageDecoder? decoder = null) => + new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = decoder ?? DecoderFactory.CreateServer() } }, new RenderNodeOptions { ClientName = "studio-pc", @@ -314,6 +357,35 @@ public sealed class RenderNodeClientTests(TestContext context) } } + /// Records the path of every request. + private sealed class RecordPaths(HttpMessageHandler inner) : DelegatingHandler(inner) + { + private readonly ConcurrentQueue _paths = new(); + + public IReadOnlyCollection Paths => _paths; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + _paths.Enqueue(request.RequestUri!.AbsolutePath); + return base.SendAsync(request, cancellationToken); + } + } + + /// Decodes every HEIF as a mid-grey 96 × 64 frame, as a native HEIF decoder would decode a real one. + private sealed class SolidHeif : IImageDecoderProvider + { + public string Name => "heif-test"; + + public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null; + + public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) + { + var image = new ImageBuffer(96, 64); + for (var c = 0; c < 3; c++) image.Plane(c).Fill(0.2f); + return new DecodedImage(image, DecodeTier.Native, Name, 96, 64); + } + } + /// Counts the node's claims the server answered. private sealed class CountClaims(HttpMessageHandler inner) : DelegatingHandler(inner) { diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs index 689fbb67..d819d2e6 100644 --- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs @@ -1,6 +1,9 @@ using System.Net; +using Microsoft.Extensions.DependencyInjection; using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; +using Titano.Core.Model; +using Titano.Server.Jobs; using Titano.Server.Storage; using Titano.Testing; @@ -61,11 +64,70 @@ public sealed class ArchiveModuleTests(TestContext context) Assert.Equal(server.Database.SchemaVersion, summary.SchemaVersion); } - private static byte[] Photo() + [Test] + public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPolling() + { + var photo = Photo(); + // An adopted folder that already holds photos. + await using var server = await TestServer.StartAsync(context, options => + { + Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof")); + File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo); + }, ArchiveModule.Register); + var archive = server.Services.GetRequiredService(); + await UntilIndexedAsync(server, archive.Id, "roof"); + await UntilScannedAsync(server); + + // Another writer adds a folder: the next poll queues a scan, but not while one is pending. + Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden")); + File.WriteAllBytes(Path.Combine(server.Options.ArchivePath, "garden", "IMG_0002.JPG"), Photo(seed: 3)); + var polling = server.Services.GetRequiredService(); + var paused = new Job + { + Id = Guid.CreateVersion7(), + Kind = JobKind.Scan, + TargetIds = [archive.Id], + State = JobState.Paused, + Created = DateTimeOffset.UtcNow, + }; + await server.Database.Jobs.EnqueueAsync(paused, context.Cancellation); + Assert.Null(await polling.PollAsync(context.Cancellation)); + await server.Services.GetRequiredService().Engine.CancelAsync(paused.Id, context.Cancellation); + + var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation)); + Assert.Equal(JobPriority.Background, scan.Priority); + await UntilIndexedAsync(server, archive.Id, "garden"); + } + + private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder) + { + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!(await server.Database.Sequences.ListLocationsAsync(archiveId)).Any(l => l.Folder == folder)) + { + Assert.True(DateTime.UtcNow < until, $"{folder} was not indexed"); + await Task.Delay(50); + } + } + + /// Waits for the scan queued at start to finish: its sequences are indexed before it ends. + private static async Task UntilScannedAsync(TestServer server) + { + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + foreach (var state in new[] { JobState.Queued, JobState.Running }) + { + while ((await server.Database.Jobs.ListAsync(state, JobKind.Scan)).Items.Count > 0) + { + Assert.True(DateTime.UtcNow < until, "the scan at start did not finish"); + await Task.Delay(50); + } + } + } + + private static byte[] Photo(int seed = 0) { const int width = 96, height = 64; var pixels = new byte[width * height * 3]; - for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)(i * 7 % 251); + for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed) % 251); return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 }); } } diff --git a/tests/Titano.Server.Tests/Storage/ProxyUploadTests.cs b/tests/Titano.Server.Tests/Storage/ProxyUploadTests.cs new file mode 100644 index 00000000..2e0811ba --- /dev/null +++ b/tests/Titano.Server.Tests/Storage/ProxyUploadTests.cs @@ -0,0 +1,102 @@ +using System.Net; +using System.Net.Http.Headers; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Metadata; +using Titano.Protocol; +using Titano.Server.Storage; +using Titano.Testing; + +namespace Titano.Server.Tests.Storage; + +/// +/// Proxies of originals the server cannot decode (BRIEF §2.3: HEIF) arrive from clients and render nodes through +/// PUT /api/v1/frames/{id}/proxy and become the frame's proxy and thumbnail. +/// +[Category("server")] +public sealed class ProxyUploadTests(TestContext context) +{ + [Test] + public async Task AProxySentForAFrameTheServerCannotDecodeIsCheckedKeptAndThumbnailed() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var (viewer, _, _) = await server.UserAsync("vera", "viewer"); + var upload = await Uploads.UploadAsync(editor, Heif(1), "IMG_0001.HEIC", sessionKey: "roof"); + var frame = Assert.NotNull(await server.Database.Frames.GetAsync(upload.FrameId)); + Assert.Equal(ImageFileFormat.Heif, frame.Format); + + var manifest = await ManifestAsync(editor, upload.SequenceId); + Assert.Equal((true, false), (manifest.Frames.Single().WantsProxy, manifest.Frames.Single().HasProxy)); + Assert.Equal(new ProxySpec(1920, 92), manifest.Proxies); + + using (var garbage = await PutAsync(editor, frame.Id, "not a photo"u8.ToArray())) + await garbage.ExpectErrorAsync(HttpStatusCode.UnprocessableEntity, "invalid_proxy"); + using (var other = await PutAsync(editor, frame.Id, Proxy(frame.CapturedAt!.Value.AddSeconds(1)))) + await other.ExpectErrorAsync(HttpStatusCode.UnprocessableEntity, "proxy_mismatch"); + var proxy = Proxy(frame.CapturedAt!.Value); + using (var refused = await PutAsync(viewer, frame.Id, proxy)) + await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); + using (var missing = await PutAsync(editor, Guid.CreateVersion7(), proxy)) + await missing.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); + using (var stored = await PutAsync(editor, frame.Id, proxy)) + await stored.ExpectAsync(HttpStatusCode.NoContent); + + await Http.BytesEqualAsync(proxy, editor.GetByteArrayAsync($"/api/v1/frames/{frame.Id}/proxy")); + var after = (await ManifestAsync(editor, upload.SequenceId)).Frames.Single(); + Assert.Equal((false, true), (after.WantsProxy, after.HasProxy)); + + // The thumbnail job the proxy queued draws the frame from it. + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (true) + { + using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.Id}/thumb"); + if (thumb.StatusCode == HttpStatusCode.OK) + { + var jpeg = await thumb.Content.ReadAsByteArrayAsync(); + Assert.Equal((320, 240), (JpegDecoder.ReadInfo(jpeg).Width, JpegDecoder.ReadInfo(jpeg).Height)); + break; + } + Assert.True(DateTime.UtcNow < until, $"thumbnail not built: {(int)thumb.StatusCode}"); + await Task.Delay(100); + } + } + + [Test] + public async Task FramesTheServerDecodesAskForNoProxy() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var upload = await Uploads.UploadAsync(editor, Proxy(new DateTime(2026, 9, 28, 18, 2, 11)), "IMG_0001.JPG", sessionKey: "roof"); + + Assert.False((await ManifestAsync(editor, upload.SequenceId)).Frames.Single().WantsProxy); + // Nor takes one: renders and remote views keep the proxy the server makes itself. + using var refused = await PutAsync(editor, upload.FrameId, Proxy(new DateTime(2026, 9, 28, 18, 2, 11))); + await refused.ExpectErrorAsync(HttpStatusCode.UnprocessableEntity, "proxy_not_wanted"); + } + + /// A HEIF file as far as signature sniffing goes (an ftyp box naming heic), with no pixels the server can read. + private static byte[] Heif(int seed) => [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, seed)]; + + /// A 320 × 240 JPEG with in its Exif, as a client makes a proxy. + private static byte[] Proxy(DateTime captured) + { + using var image = new ImageBuffer(320, 240); + for (var c = 0; c < 3; c++) image.Plane(c).Fill(0.3f + c * 0.1f); + var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Titano" }); + return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); + } + + private static Task PutAsync(HttpClient client, Guid frameId, byte[] body) + { + var content = new ByteArrayContent(body); + content.Headers.ContentType = new MediaTypeHeaderValue("image/jpeg"); + return client.PutAsync($"/api/v1/frames/{frameId}/proxy", content); + } + + private static async Task ManifestAsync(HttpClient client, Guid sequenceId) + { + using var response = await client.GetAsync($"/api/v1/sequences/{sequenceId}/manifest"); + return await response.ReadAsync(ProtocolJson.Default.SequenceManifest); + } +} diff --git a/tests/Titano.Server.Tests/Sync/SyncServerTests.cs b/tests/Titano.Server.Tests/Sync/SyncServerTests.cs index 7381c478..effc6a7c 100644 --- a/tests/Titano.Server.Tests/Sync/SyncServerTests.cs +++ b/tests/Titano.Server.Tests/Sync/SyncServerTests.cs @@ -112,16 +112,19 @@ public sealed class SyncServerTests(TestContext context) var (_, _, token) = await server.UserAsync("alberto", "editor"); await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop"); - var photos = Enumerable.Range(20, 3).Select(SyncClient.Photo).ToList(); + var photos = Enumerable.Range(20, 4).Select(SyncClient.Photo).ToList(); var local = await studio.LocalAsync("Aurora", photos); await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); var remoteId = (await studio.EntryAsync("Aurora")).Link!.RemoteId; - // An earlier attempt left half of one original, and a damaged half of another. + // An earlier attempt left half of one original, a damaged half of another, a third complete (stopped while it was + // being hashed) and a fourth longer than the file: the server cannot resume either of the last two (416). var staging = Path.Combine(laptop.Archive.RootPath, ".titano", "staging", "sync-" + remoteId.ToString("N")); Directory.CreateDirectory(staging); await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[0]) + ".jpg.part"), photos[0][..(photos[0].Length / 2)]); await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[1]) + ".jpg.part"), new byte[photos[1].Length / 2]); + await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[2]) + ".jpg.part"), photos[2]); + await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[3]) + ".jpg.part"), [.. photos[3], .. new byte[64]]); await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Aurora", SyncDownloadMode.Originals, laptop.Archive.Id);