diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index cfdc756a..f750e5a8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -58,7 +58,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `Jobs` | `JobEngine` (its number of workers can change while it runs), `IJobHandler`, `IJobHandlerProvider`, `JobContext` (progress, checkpoints, follow-up jobs) | | `Archiving` | `ArchiveService` (create/adopt/open, archive settings written to `archive.json` then the index), `ArchiveIndexer` (scan, rebuild), `ArchiveIngest` (uploads and imports), `ImportService`, `ArchiveVerifier`, `ArchiveImages`, `FrameTiers` (decode tier, `PreviewOnly`/`Undecodable` flags and preview size of frames, written to sidecar and index by the light analysis, deep analysis, renders and image passes), `GprCompanions` (the JPEG of each GoPro GPR, found in the index when an import kept it in another sequence), `ArchiveMaintenance`, `MassRenamer`, `SequenceTrash` (delete into the archive's hidden trash with a dry run, restore; the folder moves back when the index step fails), job handlers | | `Processing` | `ProcessingService`, `SequenceAnalyser`, `SequenceRender`, `ProjectStore`, `ProjectBundles` (a `.titano` file for a sequence, and the sequence a file names), `RenderNodeClient`, job handlers | -| `Sync` | `SyncService` (sync table of one server: compare, link by content, settle conflicts, enqueue transfers), `SyncRules` (status, settings merge and differences, no I/O), `SyncLinkStore` (`sync_links`, `sync_state`), `SyncTransfers` and `SyncJobHandler` (resumable, hash-verified uploads and downloads as jobs; downloads commit through the archive ingest), `SyncAgent` (background comparison, server events, backoff, offline proxies), `ProxyCache` and `RemoteSequence` (server sequences opened with proxies), `ISyncRemote` (the server calls; `TitanoSyncRemote` over `TitanoClient`) | +| `Sync` | `SyncService` (sync table of one server: compare, link by content, settle conflicts, enqueue transfers), `SyncRules` (status, settings merge and differences, no I/O), `SyncPositions` (the settings' frame positions translated by photo to the frames of the copy they go to), `SyncLinkStore` (`sync_links`, `sync_state`), `SyncTransfers` and `SyncJobHandler` (resumable, hash-verified uploads and downloads as jobs; downloads commit through the archive ingest), `SyncAgent` (background comparison, server events, backoff, offline proxies), `ProxyCache` and `RemoteSequence` (server sequences opened with proxies), `ISyncRemote` (the server calls; `TitanoSyncRemote` over `TitanoClient`) | ### 2.3 Titano.App areas diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 84cf4086..0689503c 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -42,6 +42,7 @@ A file of the sequence is open in another program; close it and try again.long Its folder is no longer on disk; nothing is left to move to the trash.long Your account cannot do this.long + The photos of the sequence changed on the server; reload it and apply your edit again.long The file arrived damaged; upload it again.long Open the admin panel over HTTPS to sign in.long Some chunks are still missing.long diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index b0aa9af3..0fb50e00 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -40,6 +40,7 @@ Un file della sequenza è aperto in un altro programma; chiudilo e riprova. La sua cartella non è più sul disco; non resta nulla da spostare nel cestino. Il tuo account non può farlo. + Le foto della sequenza sono cambiate sul server; ricaricala e riapplica la modifica. Il file è arrivato danneggiato; caricalo di nuovo. Apri il pannello di amministrazione in HTTPS per accedere. Mancano ancora alcuni blocchi. diff --git a/src/Titano.Core/Advice/ParameterBindings.cs b/src/Titano.Core/Advice/ParameterBindings.cs index 956dd530..e0339bd4 100644 --- a/src/Titano.Core/Advice/ParameterBindings.cs +++ b/src/Titano.Core/Advice/ParameterBindings.cs @@ -36,7 +36,8 @@ internal static class ParameterBindings { Parameter = AdvisorParameters.Hero, Section = AdvisorSection.Frames, Effective = s => s.HeroFrame, - Set = (s, r) => s.HeroFrame is null ? s with { HeroFrame = (int)r.Value! } : s, + // A hero waiting for its photo (taken from another copy) is the user's choice too. + Set = (s, r) => s.HeroFrame is null && !s.HeroPending ? s with { HeroFrame = (int)r.Value! } : s, }, new() { diff --git a/src/Titano.Core/Project/FramePhotos.cs b/src/Titano.Core/Project/FramePhotos.cs new file mode 100644 index 00000000..880b2169 --- /dev/null +++ b/src/Titano.Core/Project/FramePhotos.cs @@ -0,0 +1,88 @@ +namespace Titano.Core.Project; + +/// +/// A photo as every copy of a sequence knows it (BRIEF §14.5: originals are content-addressed): its SHA-256 once hashed, +/// else its size and capture time (a folder adopted and not verified yet). Frame ids and positions differ between a +/// server sequence and a PC's copy of it; the photo does not. +/// +/// Hex SHA-256; null or empty while the copy has not hashed the file. +public sealed record FramePhoto(string? Sha256, long Size, DateTime? CapturedAt) +{ + public bool IsHashed => !string.IsNullOrEmpty(Sha256); + + /// + /// The same photo: the same SHA-256 when both are hashed, otherwise the same size and capture time (to the + /// millisecond, as databases keep it). + /// + public bool Matches(FramePhoto other) + { + ArgumentNullException.ThrowIfNull(other); + if (IsHashed && other.IsHashed) return string.Equals(Sha256, other.Sha256, StringComparison.OrdinalIgnoreCase); + return Size == other.Size && CapturedAt is { } mine && other.CapturedAt is { } theirs && Millisecond(mine) == Millisecond(theirs); + } + + internal static long Millisecond(DateTime time) => time.Ticks / TimeSpan.TicksPerMillisecond; +} + +/// +/// The photos of one copy's frames by frame index (positions in capture order, BRIEF §5), and where a photo is among +/// them. Immutable once built, so parallel workers may share it. +/// +public sealed class FramePhotos +{ + private readonly FramePhoto?[] _byIndex; + private readonly Dictionary _bySha = new(StringComparer.Ordinal); + private readonly Dictionary<(long Size, long Time), List> _bySizeAndTime = []; + + /// Each frame's index and photo, in any order. + public FramePhotos(IEnumerable<(int Index, FramePhoto Photo)> frames) + { + ArgumentNullException.ThrowIfNull(frames); + var list = frames.Where(f => f.Index >= 0).ToList(); + _byIndex = new FramePhoto?[list.Count == 0 ? 0 : list.Max(f => f.Index) + 1]; + foreach (var (index, photo) in list) _byIndex[index] = photo; + for (var i = 0; i < _byIndex.Length; i++) + { + if (_byIndex[i] is not { } photo) continue; + if (photo.IsHashed) _bySha.TryAdd(photo.Sha256!.ToLowerInvariant(), i); + if (photo.CapturedAt is not { } time) continue; + var key = (photo.Size, FramePhoto.Millisecond(time)); + if (!_bySizeAndTime.TryGetValue(key, out var same)) _bySizeAndTime[key] = same = []; + same.Add(i); + } + } + + public int Count => _byIndex.Length; + + /// The photo at ; null when no frame has that index. + public FramePhoto? this[int index] => (uint)index < (uint)_byIndex.Length ? _byIndex[index] : null; + + /// Some frame is not hashed yet: matching it needs capture times. + public bool HasUnhashed => _byIndex.Any(p => p is { IsHashed: false }); + + /// The index of the first frame holding (); null when none does. + public int? IndexOf(FramePhoto photo) + { + ArgumentNullException.ThrowIfNull(photo); + if (photo.IsHashed && _bySha.TryGetValue(photo.Sha256!.ToLowerInvariant(), out var hashed)) return hashed; + if (photo.CapturedAt is not { } time || !_bySizeAndTime.TryGetValue((photo.Size, FramePhoto.Millisecond(time)), out var candidates)) return null; + foreach (var index in candidates) + { + if (_byIndex[index]!.Matches(photo)) return index; + } + return null; + } + + /// Both lists hold the same photos at the same indices: a position names the same photo on both. + public bool SameAs(FramePhotos other) + { + ArgumentNullException.ThrowIfNull(other); + if (other.Count != Count) return false; + for (var i = 0; i < Count; i++) + { + var (mine, theirs) = (this[i], other[i]); + if (mine is null != theirs is null || (mine is not null && !mine.Matches(theirs!))) return false; + } + return true; + } +} diff --git a/src/Titano.Core/Project/ProjectBundle.cs b/src/Titano.Core/Project/ProjectBundle.cs index bbbf34c8..ac506907 100644 --- a/src/Titano.Core/Project/ProjectBundle.cs +++ b/src/Titano.Core/Project/ProjectBundle.cs @@ -130,6 +130,7 @@ public sealed record ProjectBundle(BundleArchive Archive, BundleSequence Sequenc { ExcludedFrames = current.ExcludedFrames, HeroFrame = current.HeroFrame, + PendingFrames = current.PendingFrames, Camera = Project.Camera with { Keyframes = current.Camera.Keyframes }, }; diff --git a/src/Titano.Core/Project/ProjectSettings.cs b/src/Titano.Core/Project/ProjectSettings.cs index d6c22280..2c9d274f 100644 --- a/src/Titano.Core/Project/ProjectSettings.cs +++ b/src/Titano.Core/Project/ProjectSettings.cs @@ -48,6 +48,30 @@ public sealed record CurvePoint(double X, double Y); /// Virtual-camera keyframe in normalised coordinates. public sealed record CameraKey(int Frame, double CenterX, double CenterY, double Zoom, double Rotation, string Easing = "EaseInOut"); +/// +/// The frame positions of a project that name a photo this copy of the sequence does not hold: settings taken from a +/// copy that holds it (BRIEF §14.5). They become positions again when the photo arrives +/// (). +/// +public sealed record PendingFrame +{ + public string? Sha256 { get; init; } + public long Size { get; init; } + public DateTime? CapturedAt { get; init; } + + /// The photo is excluded from renders. + public bool Excluded { get; init; } + + /// The photo is the hero frame, unless another one is chosen before it arrives. + public bool Hero { get; init; } + + /// Virtual-camera keyframes on the photo; their is set when it arrives. + public IReadOnlyList Keyframes { get; init; } = []; + + [JsonIgnore] + public FramePhoto Photo => new(Sha256, Size, CapturedAt); +} + public sealed record TimingSettings { public Auto Mode { get; init; } = new(); @@ -177,6 +201,9 @@ public sealed record ProjectSettings /// Layout of a project edited as JSON (): indented like . private static readonly JsonSerializerOptions IndentedJson = new() { WriteIndented = true }; + /// Member names as a project edited as JSON reads them (an older or hand-written file may use any casing). + private static readonly JsonNodeOptions CaseInsensitive = new() { PropertyNameCaseInsensitive = true }; + public int Version { get; init; } = CurrentVersion; public TimingSettings Timing { get; init; } = new(); public ExposureSettings Exposure { get; init; } = new(); @@ -191,6 +218,16 @@ public sealed record ProjectSettings /// Index of the hero frame (sequence thumbnail). public int? HeroFrame { get; init; } + /// + /// Exclusions, hero and keyframes naming photos this copy does not hold yet (settings synced from a copy that holds + /// them); null when there are none. They take their place when the photos arrive (). + /// + public IReadOnlyList? PendingFrames { get; init; } + + /// The hero waits for its photo, so advice does not choose one meanwhile. + [JsonIgnore] + public bool HeroPending => PendingFrames?.Any(p => p.Hero) == true; + public string ToJson() => JsonSerializer.Serialize(this, ProjectJson.Default.ProjectSettings); /// @@ -286,6 +323,244 @@ public sealed record ProjectSettings replace(JsonValue.Create(now)); return true; } + + /// + /// Whether a project names frames (exclusions, a hero, camera keyframes or ); settings + /// naming none cross between copies of a sequence as they are. False for text that is not a project object. + /// + public static bool NamesFrames(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return false; + try + { + return JsonNode.Parse(json, CaseInsensitive) is JsonObject project + && (project["excludedFrames"] is JsonArray { Count: > 0 } || project["heroFrame"] is JsonValue || project["pendingFrames"] is JsonArray { Count: > 0 } + || (project["camera"] is JsonObject camera && camera["keyframes"] is JsonArray { Count: > 0 })); + } + catch (Exception ex) when (ex is JsonException or ArgumentException) + { + return false; + } + } + + /// + /// Translates the frame positions of a project written against the frames of another copy of the sequence + /// (; a server sequence and a PC's copy hold the same photos, though not always all of them, + /// BRIEF §14.5) to the frames of , by photo: each exclusion, the hero and each camera keyframe + /// moves to the frame of holding its photo. One whose photo lacks becomes + /// a , and pending frames whose photo holds become positions again; a + /// pending hero only while no other hero is chosen, a pending keyframe only where no keyframe is. With + /// null the positions already name 's frames and only pending frames are + /// placed (photos that arrived). A position naming no frame of names nothing and goes. Edited + /// in place like ; null when nothing changes or is not a + /// project object. + /// + public static string? TranslateFramePositions(string json, FramePhotos? from, FramePhotos to) + { + ArgumentNullException.ThrowIfNull(to); + if (string.IsNullOrWhiteSpace(json)) return null; + try + { + if (JsonNode.Parse(json, CaseInsensitive) is not JsonObject project) return null; + var pending = new List(); + var unreadable = new List(); + if (project["pendingFrames"] is JsonArray stored) + { + foreach (var node in stored) + { + // An entry naming no photo is not this Titano's to place: it stays as it is. + if (node is JsonObject entry && PhotoOf(entry) is { } photo) pending.Add(new Parked(photo, entry.DeepClone().AsObject())); + else unreadable.Add(node?.DeepClone()); + } + } + var changed = from is not null && !from.SameAs(to) && TranslatePositions(project, from, to, pending); + changed |= PlacePending(project, to, pending); + if (!changed) return null; + SortPositions(project); + var left = pending.Where(p => !p.Placed).Select(p => (JsonNode?)p.Entry).Concat(unreadable).ToArray(); + if (left.Length == 0) project.Remove("pendingFrames"); + else project["pendingFrames"] = new JsonArray(left); + return project.ToJsonString(IndentedJson); + } + catch (Exception ex) when (ex is JsonException or ArgumentException or InvalidOperationException) + { + // Not JSON, a member given twice, or a member of an unexpected kind: not a project to edit. + return null; + } + } + + /// Moves every position of from 's frames to 's. + private static bool TranslatePositions(JsonObject project, FramePhotos from, FramePhotos to, List pending) + { + var changed = false; + if (project["excludedFrames"] is JsonArray excluded) + { + var kept = new List(excluded.Count); + foreach (var node in excluded) + { + if (!IsPosition(node, out var at)) kept.Add(node?.DeepClone()); + else if (from[at] is not { } photo) changed = true; + else if (to.IndexOf(photo) is { } now) + { + kept.Add(JsonValue.Create(now)); + changed |= now != at; + } + else + { + Park(pending, photo).Entry["excluded"] = true; + changed = true; + } + } + Replace(excluded, kept); + } + if (project["heroFrame"] is JsonValue hero && hero.TryGetValue(out var heroAt)) + { + var photo = from[heroAt]; + var now = photo is null ? null : to.IndexOf(photo); + if (photo is not null && now is null) Park(pending, photo).Entry["hero"] = true; + if (now != heroAt) + { + project["heroFrame"] = now; + changed = true; + } + } + if (project["camera"] is JsonObject camera && camera["keyframes"] is JsonArray keyframes) + { + var kept = new List(keyframes.Count); + foreach (var node in keyframes) + { + if (node is not JsonObject key || !IsPosition(key["frame"], out var at)) + { + kept.Add(node?.DeepClone()); + continue; + } + var copy = key.DeepClone().AsObject(); + if (from[at] is not { } photo) + { + changed = true; + } + else if (to.IndexOf(photo) is { } now) + { + copy["frame"] = now; + kept.Add(copy); + changed |= now != at; + } + else + { + var parked = Park(pending, photo).Entry; + if (parked["keyframes"] is not JsonArray keys) parked["keyframes"] = keys = new JsonArray(); + keys.Add(copy); + changed = true; + } + } + Replace(keyframes, kept); + } + return changed; + } + + /// Gives the pending frames whose photo holds their positions back. + private static bool PlacePending(JsonObject project, FramePhotos to, List pending) + { + var changed = false; + foreach (var parked in pending) + { + if (to.IndexOf(parked.Photo) is not { } at) continue; + parked.Placed = changed = true; + if (Flag(parked.Entry, "excluded")) + { + if (project["excludedFrames"] is not JsonArray positions) project["excludedFrames"] = positions = new JsonArray(); + if (!positions.Any(n => IsPosition(n, out var p) && p == at)) positions.Add(at); + } + // A hero chosen since the settings were taken is newer than the one that waited. + if (Flag(parked.Entry, "hero") && project["heroFrame"] is null) project["heroFrame"] = at; + if (parked.Entry["keyframes"] is JsonArray { Count: > 0 } keys) + { + if (project["camera"] is not JsonObject camera) project["camera"] = camera = new JsonObject(); + if (camera["keyframes"] is not JsonArray keyframes) camera["keyframes"] = keyframes = new JsonArray(); + foreach (var key in keys.OfType()) + { + // A keyframe set on that frame since is newer. + if (keyframes.Any(n => n is JsonObject k && IsPosition(k["frame"], out var f) && f == at)) continue; + var copy = key.DeepClone().AsObject(); + copy["frame"] = at; + keyframes.Add(copy); + } + } + } + if (project["heroFrame"] is null) return changed; + // A hero is chosen: none waits any more, and an entry left with nothing to place goes. + foreach (var parked in pending.Where(p => !p.Placed && Flag(p.Entry, "hero"))) + { + parked.Entry["hero"] = false; + parked.Placed = !Flag(parked.Entry, "excluded") && parked.Entry["keyframes"] is not JsonArray { Count: > 0 }; + changed = true; + } + return changed; + } + + /// Exclusions in ascending order without repeats and keyframes by frame (values that are no position last, as they were). + private static void SortPositions(JsonObject project) + { + if (project["excludedFrames"] is JsonArray excluded) + { + var positions = excluded.Select(n => IsPosition(n, out var p) ? p : (int?)null).OfType().Distinct().Order().Select(p => (JsonNode?)JsonValue.Create(p)); + Replace(excluded, [.. positions, .. excluded.Where(n => !IsPosition(n, out _)).Select(n => n?.DeepClone())]); + } + if (project["camera"] is JsonObject camera && camera["keyframes"] is JsonArray keyframes) + { + var sorted = keyframes.Select(n => n?.DeepClone()) + .OrderBy(n => n is JsonObject key && IsPosition(key["frame"], out var f) ? f : int.MaxValue).ToList(); + Replace(keyframes, sorted); + } + } + + /// The pending entry of , added when there is none. + private static Parked Park(List pending, FramePhoto photo) + { + if (pending.FirstOrDefault(p => p.Photo.Matches(photo)) is { } known) return known; + var entry = new JsonObject(); + if (photo.IsHashed) entry["sha256"] = photo.Sha256!.ToLowerInvariant(); + entry["size"] = photo.Size; + if (photo.CapturedAt is { } captured) entry["capturedAt"] = captured; + entry["excluded"] = false; + entry["hero"] = false; + entry["keyframes"] = new JsonArray(); + var parked = new Parked(photo, entry); + pending.Add(parked); + return parked; + } + + /// The photo a pending entry names; null when it names none. + private static FramePhoto? PhotoOf(JsonObject entry) + { + var sha = entry["sha256"] is JsonValue s && s.TryGetValue(out var text) ? text : null; + var size = entry["size"] is JsonValue z && z.TryGetValue(out var bytes) ? bytes : -1; + DateTime? captured = entry["capturedAt"] is JsonValue t && t.TryGetValue(out var time) ? time : null; + return string.IsNullOrEmpty(sha) && (captured is null || size < 0) ? null : new FramePhoto(sha, size, captured); + } + + private static bool Flag(JsonObject entry, string name) => entry[name] is JsonValue value && value.TryGetValue(out var set) && set; + + private static bool IsPosition(JsonNode? node, out int position) + { + position = 0; + return node is JsonValue value && value.TryGetValue(out position); + } + + private static void Replace(JsonArray array, IEnumerable nodes) + { + var copy = nodes.ToList(); + array.Clear(); + foreach (var node in copy) array.Add(node); + } + + /// A pending entry being edited: its photo, its JSON, and whether its photo took its place. + private sealed class Parked(FramePhoto photo, JsonObject entry) + { + public FramePhoto Photo { get; } = photo; + public JsonObject Entry { get; } = entry; + public bool Placed { get; set; } + } } [JsonSourceGenerationOptions( diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index a85265da..a8508381 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -97,14 +97,14 @@ public sealed class ArchiveIndexer /// internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) { - (string Before, string Moved)? positions = null; + PositionMove? positions = null; await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) { var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false); if (existing is not null) { positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false); - if (positions is { } project) existing = existing with { SettingsJson = project.Moved }; + if (positions is not null) existing = positions.ApplyTo(existing); } await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false); diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index 750e260a..81fd0afc 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -457,7 +457,8 @@ public sealed class ArchiveIngest : IArchiveIngest /// /// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose - /// flags change) resyncs the whole sequence, which also moves the project's frame positions after an insertion. + /// flags change) resyncs the whole sequence, which also moves the project's frame positions after an insertion. A + /// project waiting for photos taken from another copy's settings places those that arrived either way. /// private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended, bool repaired, CancellationToken cancellationToken) @@ -469,13 +470,20 @@ public sealed class ArchiveIngest : IArchiveIngest await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); return; } + PositionMove? placed = null; await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) { var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false); + if (existing is not null && ProjectPositions.Place(existing, manifest) is { } positions) + { + placed = positions; + existing = positions.ApplyTo(existing); + } await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); await tx.Frames.BulkInsertAsync(appended.Select(f => SequenceMapping.ToFrame(f, manifest.Id, sequence)), cancellationToken).ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); } + if (placed is not null) ProjectPositions.WriteProject(sequence, placed.Before, placed.Moved); await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); } diff --git a/src/Titano.Data/Archiving/ProjectPositions.cs b/src/Titano.Data/Archiving/ProjectPositions.cs index 8aefeab7..2c723d9e 100644 --- a/src/Titano.Data/Archiving/ProjectPositions.cs +++ b/src/Titano.Data/Archiving/ProjectPositions.cs @@ -14,17 +14,19 @@ namespace Titano.Data.Archiving; /// scan found, a sync download, a journalled batch completed after a crash. Positions name the frames in the indexed /// order (what editors, renders and the API read), so the old order comes from the index rows. Only an insertion is /// followed: when an indexed frame is no longer in the sidecar, positions cannot be mapped and stay as they are. The move -/// is part of the frames change, not a settings edit, so the settings version stays and a synced copy receiving the -/// same frames makes the same move. +/// is part of the frames change, not a settings edit, so the settings version stays. Settings synced from another copy +/// arrive translated to this copy's frames (); their positions on +/// photos that were not here yet () take their place when the photos arrive, +/// wherever they land. /// internal static class ProjectPositions { /// - /// The index's copy of the project (the one in force) and the same project with its positions moved to the order of - /// ; null when nothing moves. A project.json the index has no copy of was written by - /// another process against its own frames, so it is left to that writer. + /// The index's copy of the project (the one in force) with its positions moved to the order of + /// and its pending frames placed; null when nothing changes. A project.json the + /// index has no copy of was written by another process against its own frames, so it is left to that writer. /// - public static async Task<(string Before, string Moved)?> MoveAsync(TitanoRepositories db, Sequence indexed, SequenceManifest manifest, + public static async Task MoveAsync(TitanoRepositories db, Sequence indexed, SequenceManifest manifest, CancellationToken cancellationToken) { // The sidecar's frames are the ones indexed: nothing was inserted. @@ -33,13 +35,52 @@ internal static class ProjectPositions var rows = await db.Frames.ListPositionsAsync(indexed.Id, cancellationToken).ConfigureAwait(false); var now = new Dictionary(manifest.Frames.Count); foreach (var frame in manifest.Frames) now[frame.Id] = frame.I; - var moves = new Dictionary(); + Dictionary? moves = []; foreach (var row in rows) { - if (!now.TryGetValue(row.FrameId, out var index)) return null; + if (!now.TryGetValue(row.FrameId, out var index)) + { + moves = null; + break; + } if (index != row.Index) moves[row.Index] = index; } - return ProjectSettings.MoveFramePositions(project, moves) is { } moved ? (project, moved) : null; + return Placed(project, moves is null ? null : ProjectSettings.MoveFramePositions(project, moves), manifest); + } + + /// + /// The index's copy of the project with the pending frames whose photos now holds put in + /// place: frames appended in step with the index, which moves no position. Null when nothing changes. + /// + public static PositionMove? Place(Sequence indexed, SequenceManifest manifest) => + indexed.SettingsJson is { } project ? Placed(project, null, manifest) : null; + + private static PositionMove? Placed(string before, string? moved, SequenceManifest manifest) + { + var current = moved ?? before; + // Most projects wait for no photo: the sidecar's photos are gathered only for those that do. + var placed = current.Contains("pendingFrames", StringComparison.OrdinalIgnoreCase) + ? ProjectSettings.TranslateFramePositions(current, null, new FramePhotos(manifest.Frames.Select(f => (f.I, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))))) + : null; + if ((placed ?? moved) is not { } after) return null; + // A hero that waited for its photo names a frame the Library shows too. + var hero = placed is not null && HeroOf(current) is null && HeroOf(placed) is { } index ? manifest.Frames.FirstOrDefault(f => f.I == index)?.Id : null; + return new PositionMove(before, after, hero); + } + + private static int? HeroOf(string json) + { + try + { + return JsonNode.Parse(json, new JsonNodeOptions { PropertyNameCaseInsensitive = true }) is JsonObject project + && project["heroFrame"] is JsonValue value && value.TryGetValue(out var hero) + ? hero + : null; + } + catch (Exception ex) when (ex is JsonException or ArgumentException) + { + return null; + } } /// @@ -73,3 +114,13 @@ internal static class ProjectPositions } } } + +/// +/// A project whose positions followed its frames: the index's copy before and after, and the frame a hero that waited +/// for its photo now names (the index's hero frame follows it); null when the hero frame stays. +/// +internal sealed record PositionMove(string Before, string Moved, Guid? HeroFrameId) +{ + /// The index row with the moved project. + public Sequence ApplyTo(Sequence row) => row with { SettingsJson = Moved, HeroFrameId = HeroFrameId ?? row.HeroFrameId }; +} diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index a7e78308..75b0be08 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -233,12 +233,22 @@ public sealed class SequenceRepository /// changed the sequence first (BRIEF §14.5). /// public Task EditAsync(Guid id, SequenceEdit edit, int? expectedSettingsVersion, CancellationToken cancellationToken = default) => + EditAsync(id, edit, expectedSettingsVersion, null, cancellationToken); + + /// + /// , also refused when the sequence's frames + /// version is no longer : settings naming frames by position were written + /// against frames that changed since. + /// + public Task EditAsync(Guid id, SequenceEdit edit, int? expectedSettingsVersion, int? expectedFramesVersion, + CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken) .ConfigureAwait(false); if (current is null) return new SequenceEditResult(SequenceEditOutcome.NotFound, null); - if (expectedSettingsVersion is { } expected && expected != current.SettingsVersion) + if ((expectedSettingsVersion is { } expected && expected != current.SettingsVersion) + || (expectedFramesVersion is { } frames && frames != current.FramesVersion)) return new SequenceEditResult(SequenceEditOutcome.Conflict, current); var hero = current.HeroFrameId; if (edit.HeroFrameId == Guid.Empty) hero = null; diff --git a/src/Titano.Data/Sync/SyncPositions.cs b/src/Titano.Data/Sync/SyncPositions.cs new file mode 100644 index 00000000..00f746be --- /dev/null +++ b/src/Titano.Data/Sync/SyncPositions.cs @@ -0,0 +1,57 @@ +using Titano.Core.Project; +using Titano.Protocol; +using SequenceFrames = Titano.Data.Processing.SequenceFrames; + +namespace Titano.Data.Sync; + +/// +/// The frame positions of settings crossing between a local copy and its server sequence (exclusions, hero, camera +/// keyframes; BRIEF §5, §14.5). Each copy numbers its frames in capture order, and the two may hold different photos +/// while originals are still to travel, so settings are translated by photo on the way: the server's are moved to this +/// copy's frames before they are compared or applied, and this copy's to the server's frames as +/// lists them before they are sent. A position on a photo the receiving copy lacks waits +/// there as a pending frame and takes its place when the photo arrives (). +/// +internal sealed class SyncPositions(TitanoDatabase db, ISyncRemote remote, Guid localId, SequenceManifest manifest) +{ + private FramePhotos? _server; + + /// The server's frames version the translation to the server's frames is made against. + public int ServerFrames => manifest.FramesVersion; + + /// Settings of the server sequence with their positions on this copy's frames, as they are now. + public async Task ToLocalAsync(string? settingsJson, CancellationToken cancellationToken) + { + if (!ProjectSettings.NamesFrames(settingsJson)) return settingsJson; + var local = await LocalAsync(cancellationToken).ConfigureAwait(false); + var server = await ServerAsync(local, cancellationToken).ConfigureAwait(false); + return ProjectSettings.TranslateFramePositions(settingsJson!, server, local) ?? settingsJson; + } + + /// Settings of this copy with their positions on the server's frames. + public async Task ToServerAsync(string? settingsJson, CancellationToken cancellationToken) + { + if (!ProjectSettings.NamesFrames(settingsJson)) return settingsJson; + var local = await LocalAsync(cancellationToken).ConfigureAwait(false); + var server = await ServerAsync(local, cancellationToken).ConfigureAwait(false); + return ProjectSettings.TranslateFramePositions(settingsJson!, local, server) ?? settingsJson; + } + + /// This copy's frames, read each time (an import may insert some between a comparison and a write). + private async Task LocalAsync(CancellationToken cancellationToken) => + new((await SequenceFrames.ListAsync(db, localId, cancellationToken).ConfigureAwait(false)) + .Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); + + /// + /// The server's frames from the manifest; their capture times are fetched only when a frame of either side is not + /// hashed, since such a frame is matched by size and capture time. + /// + private async Task ServerAsync(FramePhotos local, CancellationToken cancellationToken) + { + if (_server is not null) return _server; + IReadOnlyDictionary captured = new Dictionary(); + if (local.HasUnhashed || manifest.Frames.Any(f => f.Sha256.Length == 0)) + captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt); + return _server = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, captured.GetValueOrDefault(f.Id))))); + } +} diff --git a/src/Titano.Data/Sync/SyncRemote.cs b/src/Titano.Data/Sync/SyncRemote.cs index 08537a27..5949f116 100644 --- a/src/Titano.Data/Sync/SyncRemote.cs +++ b/src/Titano.Data/Sync/SyncRemote.cs @@ -27,7 +27,10 @@ public interface ISyncRemote : IDisposable async Task FirstFrameAsync(Guid sequenceId, CancellationToken cancellationToken) => (await ListFramesAsync(sequenceId, cancellationToken).ConfigureAwait(false)).FirstOrDefault(); - /// A 409 means the settings version moved on the server. + /// + /// A 409 means the settings version moved on the server, or its frames did since + /// . + /// Task UpdateSequenceAsync(Guid id, UpdateSequenceRequest request, CancellationToken cancellationToken); /// diff --git a/src/Titano.Data/Sync/SyncRules.cs b/src/Titano.Data/Sync/SyncRules.cs index e0852bce..5f83ad05 100644 --- a/src/Titano.Data/Sync/SyncRules.cs +++ b/src/Titano.Data/Sync/SyncRules.cs @@ -26,9 +26,13 @@ public sealed record SyncPayload(string? Title, IReadOnlyList Tags, int public bool SameAs(SyncPayload other) => SameMetadata(other) && SyncRules.SameSettings(SettingsJson, other.SettingsJson); - /// The PATCH that makes the server's copy equal to this one; settings are sent in their normalised form. - public UpdateSequenceRequest ToRequest(int? expectedSettingsVersion) => - new(Title ?? string.Empty, Tags, Rating, Notes ?? string.Empty, SyncRules.NormalizeSettings(SettingsJson), expectedSettingsVersion); + /// + /// The PATCH that makes the server's copy equal to this one; settings are sent in their normalised form, with the + /// server frames version their frame positions name when they were translated to the server's frames. + /// + public UpdateSequenceRequest ToRequest(int? expectedSettingsVersion, int? expectedFramesVersion = null) => + new(Title ?? string.Empty, Tags, Rating, Notes ?? string.Empty, SyncRules.NormalizeSettings(SettingsJson), expectedSettingsVersion, + ExpectedFramesVersion: expectedFramesVersion); } [JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index cc5e6071..3f5becf6 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -8,6 +8,7 @@ using Titano.Core.Diagnostics; using Titano.Core.IO; using Titano.Core.Localization; using Titano.Core.Model; +using Titano.Core.Project; using Titano.Data.Archiving; using Titano.Data.Jobs; using Titano.Data.Processing; @@ -40,7 +41,8 @@ public sealed record SyncOptions /// and, when frame counters or render counts moved, the manifest: originals are matched by SHA-256 and never conflict; /// renders by id. Settings, title, tags, rating and notes travel together under the settings version, so a change on /// one side is sent or taken (automatically or on request) and a change on both sides with different content is a -/// conflict the user settles with , against the versions the user was shown. Uploads and +/// conflict the user settles with , against the versions the user was shown. The settings' +/// frame positions are translated by photo to the frames of the copy they go to (). Uploads and /// downloads run as jobs (): uploads use the chunked, hash-verified, resumable protocol; /// downloaded originals are verified and committed through the archive ingest, so sidecars stay right; proxies go to a /// cache and open as a ; renders travel on request. Thumbnails never travel upward, and @@ -185,7 +187,7 @@ public sealed class SyncService try { var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false); - if (link is not { ConflictRemote: { } theirs, ConflictLocalSettings: { } seenLocal, ConflictRemoteSettings: { } seenRemote } + if (link is not { ConflictRemote: not null, ConflictLocalSettings: { } seenLocal, ConflictRemoteSettings: { } seenRemote } || local.SettingsVersion != seenLocal) { changed = true; @@ -194,7 +196,9 @@ public sealed class SyncService { try { - var updated = await remote.UpdateSequenceAsync(remoteId, SyncPayload.Of(local).ToRequest(seenRemote), cancellationToken).ConfigureAwait(false); + var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false); + var request = await RequestAsync(remote, local, manifest, seenRemote, cancellationToken).ConfigureAwait(false); + var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false); link = Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion); } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict) @@ -204,15 +208,18 @@ public sealed class SyncService } else { + // The server's version as it is now: the one shown, with its positions on the server's current frames. var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); - if (dto.SettingsVersion != seenRemote) + var manifest = dto.SettingsVersion == seenRemote ? await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false) : null; + if (manifest?.SettingsVersion != seenRemote) { changed = true; } else { if (resolution == SyncResolution.SaveBoth) saved = await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false); - if (await ApplyLocallyAsync(local, theirs, cancellationToken).ConfigureAwait(false) is { } applied) + var theirs = SyncPayload.Of(dto, manifest!.SettingsJson); + if (await ApplyLocallyAsync(local, theirs, Positions(remote, local, manifest), cancellationToken).ConfigureAwait(false) is { } applied) { link = Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, seenRemote); } @@ -339,11 +346,11 @@ public sealed class SyncService var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false); for (var attempt = 1; ; attempt++) { - var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); + var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false); try { - var updated = await remote.UpdateSequenceAsync(remoteId, SyncPayload.Of(local).ToRequest(dto.SettingsVersion), cancellationToken) - .ConfigureAwait(false); + var request = await RequestAsync(remote, local, manifest, manifest.SettingsVersion, cancellationToken).ConfigureAwait(false); + var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false); await _links.UpsertAsync(Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion), cancellationToken) .ConfigureAwait(false); return; @@ -370,7 +377,8 @@ public sealed class SyncService var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false); // Edited on this PC meanwhile: nothing is agreed yet, so the next comparison settles both sides' content. - if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false) is not { } applied) return; + if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), Positions(remote, local, manifest), cancellationToken) + .ConfigureAwait(false) is not { } applied) return; await _links.UpsertAsync(Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, manifest.SettingsVersion), cancellationToken) .ConfigureAwait(false); } @@ -543,6 +551,9 @@ public sealed class SyncService var localChanged = link.BaseLocalSettings != local.SettingsVersion; var remoteChanged = link.BaseRemoteSettings != link.RemoteSettingsVersion; if (!localChanged && !remoteChanged) return (link, local); + // Frames inserted here since the copy was listed moved its positions without a new settings version. + if (await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) is { } current && current.SettingsVersion == local.SettingsVersion) + local = current; if (localChanged && remoteChanged) { if (link.ConflictLocalSettings == local.SettingsVersion && link.ConflictRemoteSettings == link.RemoteSettingsVersion) return (link, local); @@ -550,7 +561,8 @@ public sealed class SyncService // The listing and the manifest disagree: the server changed during the comparison; settle it next time. if (manifest.SettingsVersion != dto.SettingsVersion) return (link, local); var mine = SyncPayload.Of(local); - var theirs = SyncPayload.Of(dto, manifest.SettingsJson); + // Compared (and shown) with its positions on this copy's frames: the same photos are the same settings. + var theirs = SyncPayload.Of(dto, await Positions(remote, local, manifest).ToLocalAsync(manifest.SettingsJson, cancellationToken).ConfigureAwait(false)); if (mine.SameAs(theirs)) return (Agreed(link, local.SettingsVersion, manifest.SettingsVersion), local); // Last writer wins only when one side changed; both changed differently is the user's choice (BRIEF §14.5). return (link with @@ -562,10 +574,15 @@ public sealed class SyncService } if (!policy.AutoSyncSettings) return (link, local); if (localChanged) - return (policy.CanEdit ? await PushAsync(remote, link, local, link.BaseRemoteSettings, cancellationToken).ConfigureAwait(false) : link, local); + { + if (!policy.CanEdit) return (link, local); + manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); + return (await PushAsync(remote, link, local, manifest, link.BaseRemoteSettings, cancellationToken).ConfigureAwait(false), local); + } manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); if (manifest.SettingsVersion != dto.SettingsVersion) return (link, local); - if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false) is { } pulled) + if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), Positions(remote, local, manifest), cancellationToken) + .ConfigureAwait(false) is { } pulled) return (Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion), pulled); // The local copy was read when the comparison began and an edit was saved on this PC since: both sides changed, // so the server's version must not replace it unseen (BRIEF §14.5). @@ -574,12 +591,17 @@ public sealed class SyncService return await SettleSettingsAsync(remote, link, dto, fresh, manifest, policy, cancellationToken).ConfigureAwait(false); } - /// Sends this PC's payload unless the server moved past , which shows up as a conflict next. - private static async Task PushAsync(ISyncRemote remote, SyncLink link, Sequence local, int? expected, CancellationToken cancellationToken) + /// + /// Sends this PC's payload unless the server moved past , which shows up as a conflict next, + /// or its frames changed since (the positions are translated again next time). + /// + private async Task PushAsync(ISyncRemote remote, SyncLink link, Sequence local, SequenceManifest manifest, int? expected, + CancellationToken cancellationToken) { try { - var updated = await remote.UpdateSequenceAsync(link.RemoteId, SyncPayload.Of(local).ToRequest(expected), cancellationToken).ConfigureAwait(false); + var request = await RequestAsync(remote, local, manifest, expected, cancellationToken).ConfigureAwait(false); + var updated = await remote.UpdateSequenceAsync(link.RemoteId, request, cancellationToken).ConfigureAwait(false); return Agreed(Observe(link, link.Server, updated), local.SettingsVersion, updated.SettingsVersion); } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict) @@ -588,6 +610,21 @@ public sealed class SyncService } } + /// + /// The PATCH that makes the server's copy equal to , its frame positions moved to the server's + /// frames as lists them; the server refuses it when its frames changed since. + /// + private async Task RequestAsync(ISyncRemote remote, Sequence local, SequenceManifest manifest, int? expected, + CancellationToken cancellationToken) + { + var positions = Positions(remote, local, manifest); + var settings = await positions.ToServerAsync(local.SettingsJson, cancellationToken).ConfigureAwait(false); + var frames = ProjectSettings.NamesFrames(local.SettingsJson) ? positions.ServerFrames : (int?)null; + return (SyncPayload.Of(local) with { SettingsJson = settings }).ToRequest(expected, frames); + } + + private SyncPositions Positions(ISyncRemote remote, Sequence local, SequenceManifest manifest) => new(_db, remote, local.Id, manifest); + /// /// Links local copies and server sequences that hold the same originals but no link joins (an Uploader upload of /// the same card, a link lost with an old index): candidates start at the same second with the same camera, and @@ -648,12 +685,21 @@ public sealed class SyncService private static bool SameMoment(DateTimeOffset a, DateTimeOffset b) => a.UtcTicks / 10 == b.UtcTicks / 10; /// - /// Writes a payload into the local index, project and sidecar in one step (settings version + 1) under the sequence's - /// lease, which the editor's saves take too, and only while the local copy is still at the settings version - /// was read at. Null when an edit was saved on this PC since: nothing is written then. + /// Writes the server's payload into the local index, project and sidecar in one step (settings version + 1) under the + /// sequence's lease, which the editor's saves take too, and only while the local copy is still at the settings version + /// was read at. The settings' frame positions are moved to this copy's frames as they are + /// under the lease (). Null when an edit was saved on this PC since: nothing is written + /// then. A payload the local copy already holds writes nothing, so it needs neither a writable archive nor the lease. /// - private async Task ApplyLocallyAsync(Sequence local, SyncPayload payload, CancellationToken cancellationToken) + private async Task ApplyLocallyAsync(Sequence local, SyncPayload payload, SyncPositions positions, CancellationToken cancellationToken) { + var stored = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidDataException($"Sequence {local.Id} is not indexed."); + if (stored.SettingsVersion != local.SettingsVersion) return null; + // Settled as it is: an archive that is offline or read-only does not keep the link from being agreed. + var mapped = payload with { SettingsJson = await positions.ToLocalAsync(payload.SettingsJson, cancellationToken).ConfigureAwait(false) }; + if (mapped.SameAs(SyncPayload.Of(stored))) return stored; + var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false) ?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed."); var paths = new ArchivePaths(archive.RootPath); @@ -664,6 +710,8 @@ public sealed class SyncService var row = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) ?? throw new InvalidDataException($"Sequence {local.Id} is not indexed."); if (row.SettingsVersion != local.SettingsVersion) return null; + // The frames cannot change under the lease: the positions are moved to them as they are now. + payload = payload with { SettingsJson = await positions.ToLocalAsync(payload.SettingsJson, cancellationToken).ConfigureAwait(false) }; var metadata = !payload.SameMetadata(SyncPayload.Of(row)); var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson); if (!metadata && !settings) return row; diff --git a/src/Titano.Protocol/Sequences.cs b/src/Titano.Protocol/Sequences.cs index 62fc897a..df0bd89f 100644 --- a/src/Titano.Protocol/Sequences.cs +++ b/src/Titano.Protocol/Sequences.cs @@ -37,6 +37,8 @@ public sealed record CreateSequenceRequest(string? Title, string? CameraHint); /// /// PATCH /api/v1/sequences/{id}. enables conflict detection (409). /// names the frame the sequence shows (one of its own; clears it). +/// is the frames version the settings' frame positions were written against: the +/// edit is refused (409 frames_conflict) when frames were added or removed since. /// public sealed record UpdateSequenceRequest( string? Title, @@ -45,7 +47,8 @@ public sealed record UpdateSequenceRequest( string? Notes, string? SettingsJson, int? ExpectedSettingsVersion, - Guid? HeroFrameId = null); + Guid? HeroFrameId = null, + int? ExpectedFramesVersion = null); /// Width of the largest embedded preview before orientation, like ; 0 when unknown. /// Height of that preview before orientation; 0 when unknown. diff --git a/src/Titano.Server/Api/SequenceEndpoints.cs b/src/Titano.Server/Api/SequenceEndpoints.cs index 2071c602..f7a375ba 100644 --- a/src/Titano.Server/Api/SequenceEndpoints.cs +++ b/src/Titano.Server/Api/SequenceEndpoints.cs @@ -140,14 +140,18 @@ public static class SequenceEndpoints HeroFrameId = request.HeroFrameId ?? (request.SettingsJson is { } settings ? await SettingsHeroAsync(db, id, settings, context.RequestAborted) : null), }; - var result = await db.Sequences.EditAsync(id, edit, expected, context.RequestAborted); + var result = await db.Sequences.EditAsync(id, edit, expected, request.ExpectedFramesVersion, context.RequestAborted); switch (result.Outcome) { case SequenceEditOutcome.NotFound: return ApiErrors.NotFound(context, "sequence"); + case SequenceEditOutcome.Conflict when expected is { } settingsVersion && settingsVersion != result.Sequence!.SettingsVersion: + context.Response.Headers.ETag = ETag(result.Sequence); + return ApiErrors.Conflict(context, "settings_conflict", result.Sequence.SettingsVersion.ToString(CultureInfo.InvariantCulture)); case SequenceEditOutcome.Conflict: + // The settings name frames by position and were written against frames that changed since. context.Response.Headers.ETag = ETag(result.Sequence!); - return ApiErrors.Conflict(context, "settings_conflict", result.Sequence!.SettingsVersion.ToString(CultureInfo.InvariantCulture)); + return ApiErrors.Conflict(context, "frames_conflict", result.Sequence!.FramesVersion.ToString(CultureInfo.InvariantCulture)); } var sequence = result.Sequence!; await audit.WriteAsync(caller, context, "sequence_updated", "sequence", id.ToString(), Changes(request)); diff --git a/tests/Titano.Core.Tests/Project/ProjectSettingsTests.cs b/tests/Titano.Core.Tests/Project/ProjectSettingsTests.cs index 2eb939f7..5f9d12e0 100644 --- a/tests/Titano.Core.Tests/Project/ProjectSettingsTests.cs +++ b/tests/Titano.Core.Tests/Project/ProjectSettingsTests.cs @@ -167,6 +167,119 @@ public sealed class ProjectSettingsTests Assert.Null(ProjectSettings.MoveFramePositions("""{ "heroFrame": 1, "HeroFrame": 1 }""", new Dictionary { [1] = 2 })); } + [Test] + public void PositionsCrossToAnotherCopyByPhotoAndWaitForPhotosItLacks() + { + // The server holds six photos; this copy only the last four, and one of its own the server lacks. + var photos = Enumerable.Range(0, 7).Select(Photo).ToList(); + var server = Frames(photos[..6]); + var here = Frames([.. photos[2..6], photos[6]]); + var project = new ProjectSettings + { + ExcludedFrames = [0, 3], + HeroFrame = 1, + Camera = new CameraSettings { Keyframes = [new CameraKey(0, 0.5, 0.5, 1, 0), new CameraKey(4, 0.4, 0.5, 1.5, 0)] }, + Exposure = new ExposureSettings { DeflickerWindow = new Auto().WithManual(41) }, + }; + + var taken = ProjectSettings.FromJson(Assert.NotNull(ProjectSettings.TranslateFramePositions(project.ToJson(), server, here))); + + Assert.SequenceEqual([1], taken.ExcludedFrames); + Assert.Null(taken.HeroFrame); + Assert.SequenceEqual([2], taken.Camera.Keyframes.Select(k => k.Frame)); + Assert.Equal(1.5, taken.Camera.Keyframes[0].Zoom); + Assert.Equal(41, taken.Exposure.DeflickerWindow.Manual); + var pending = Assert.NotNull(taken.PendingFrames); + Assert.Equal(2, pending.Count); + var first = Assert.Contains(pending, p => p.Sha256 == photos[0].Sha256); + Assert.True(first.Excluded && !first.Hero); + Assert.Count(1, first.Keyframes); + Assert.Equal(0.5, first.Keyframes[0].CenterX); + Assert.True(Assert.Contains(pending, p => p.Sha256 == photos[1].Sha256).Hero); + Assert.True(taken.HeroPending); + + // The two photos arrive (inserted before the others): what waited takes its place, and the rest moved with its frames. + var arrived = Frames(photos); + var moved = ProjectSettings.MoveFramePositions(taken.ToJson(), new Dictionary { [0] = 2, [1] = 3, [2] = 4, [3] = 5, [4] = 6 })!; + var placed = ProjectSettings.FromJson(Assert.NotNull(ProjectSettings.TranslateFramePositions(moved, null, arrived))); + Assert.SequenceEqual([0, 3], placed.ExcludedFrames); + Assert.Equal(1, placed.HeroFrame); + Assert.SequenceEqual([0, 4], placed.Camera.Keyframes.Select(k => k.Frame)); + Assert.Null(placed.PendingFrames); + Assert.Null(ProjectSettings.TranslateFramePositions(placed.ToJson(), null, arrived)); + } + + [Test] + public void SettingsSentBackLandOnTheSamePhotos() + { + var photos = Enumerable.Range(0, 5).Select(Photo).ToList(); + var server = Frames(photos[1..]); + var here = Frames(photos); + // Photo 0 is only here: its exclusion waits on the server, and comes back as a position. + var project = new ProjectSettings { ExcludedFrames = [0, 2], HeroFrame = 4 }; + + var sent = Assert.NotNull(ProjectSettings.TranslateFramePositions(project.ToJson(), here, server)); + var back = ProjectSettings.FromJson(Assert.NotNull(ProjectSettings.TranslateFramePositions(sent, server, here))); + + Assert.SequenceEqual([1], ProjectSettings.FromJson(sent).ExcludedFrames); + Assert.Equal(3, ProjectSettings.FromJson(sent).HeroFrame); + Assert.Equal(project.ToJson(), back.ToJson()); + } + + [Test] + public void AHeroOrKeyframeChosenSinceIsNewerThanTheOneThatWaited() + { + var photos = Enumerable.Range(0, 3).Select(Photo).ToList(); + var waiting = new ProjectSettings + { + HeroFrame = 2, + Camera = new CameraSettings { Keyframes = [new CameraKey(0, 0.2, 0.2, 2, 0)] }, + PendingFrames = [new PendingFrame { Sha256 = photos[0].Sha256, Size = photos[0].Size, Hero = true, Keyframes = [new CameraKey(9, 0.7, 0.7, 1, 0)] }], + }; + + var placed = ProjectSettings.FromJson(Assert.NotNull(ProjectSettings.TranslateFramePositions(waiting.ToJson(), null, Frames(photos)))); + + Assert.Equal(2, placed.HeroFrame); + Assert.Count(1, placed.Camera.Keyframes); + Assert.Equal(0.2, placed.Camera.Keyframes[0].CenterX); + Assert.Null(placed.PendingFrames); + } + + [Test] + public void AProjectNamingNoFrameCrossesAsItIs() + { + var json = new ProjectSettings { Exposure = new ExposureSettings { DeflickerWindow = new Auto().WithManual(41) } }.ToJson(); + + Assert.False(ProjectSettings.NamesFrames(json)); + Assert.False(json.Contains("pendingFrames", StringComparison.OrdinalIgnoreCase), "an old project reads and writes as it did"); + Assert.False(ProjectSettings.NamesFrames("not json")); + Assert.True(ProjectSettings.NamesFrames("""{ "HeroFrame": 0 }""")); + Assert.Null(ProjectSettings.TranslateFramePositions(json, Frames([Photo(1)]), Frames([Photo(2)]))); + Assert.Null(ProjectSettings.TranslateFramePositions("[1]", null, Frames([]))); + } + + [Test] + public void PhotosMatchByHashElseBySizeAndCaptureTime() + { + var time = new DateTime(2026, 9, 28, 18, 0, 0, 123); + var hashed = new FramePhoto(new string('a', 64), 100, time); + + Assert.True(hashed.Matches(new FramePhoto(new string('A', 64), 1, null))); + Assert.False(hashed.Matches(new FramePhoto(new string('b', 64), 100, time))); + Assert.True(hashed.Matches(new FramePhoto("", 100, time.AddTicks(9_999)))); + Assert.False(hashed.Matches(new FramePhoto(null, 100, time.AddMilliseconds(1)))); + Assert.False(new FramePhoto(null, 100, null).Matches(new FramePhoto(null, 100, null))); + var frames = Frames([new FramePhoto(null, 100, time), hashed]); + Assert.Equal(1, frames.IndexOf(new FramePhoto(new string('a', 64), 100, null))); + Assert.Equal(0, frames.IndexOf(new FramePhoto(null, 100, time))); + Assert.Null(frames.IndexOf(new FramePhoto(null, 101, time))); + } + + private static FramePhoto Photo(int seed) => + new(Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData([(byte)seed])), 1000 + seed, new DateTime(2026, 9, 28, 18, 0, 0).AddSeconds(10 * seed)); + + private static FramePhotos Frames(IEnumerable photos) => new(photos.Select((p, i) => (i, p))); + [Test] public void BuiltInPresetIdsAreUnique() => Assert.Equal(ExportPresets.BuiltIn.Count, ExportPresets.BuiltIn.Select(p => p.Id).Distinct().Count()); diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs index 548f799c..fa582030 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -276,6 +276,51 @@ public sealed class ArchiveIngestTests(TestContext context) Assert.True(JsonNode.DeepEquals(JsonNode.Parse(row.SettingsJson!), JsonNode.Parse(File.ReadAllText(folder.ProjectPath))), "project.json mirrors the index"); } + [Test] + public async Task PositionsWaitingForTheirPhotosTakeTheirPlaceWhenTheyArrive() + { + await using var fx = await CreateAsync(context); + var results = new List(); + for (var i = 0; i < 3; i++) results.Add(await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1 + i), 30 + i)), fx.Ct)); + var sequenceId = results[0].SequenceId; + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(sequenceId))!.Folder); + // Settings synced from a copy that also holds an earlier and a later photo: what names them waits for them. + var early = fx.Stage(Start, 40); + var late = fx.Stage(Start.AddMinutes(9), 41); + var project = new ProjectSettings + { + ExcludedFrames = [1], + Camera = new CameraSettings { Keyframes = [new CameraKey(0, 0.5, 0.5, 1, 0)] }, + PendingFrames = + [ + new PendingFrame { Sha256 = early.Sha256, Size = early.Size, Excluded = true }, + new PendingFrame { Sha256 = late.Sha256, Size = late.Size, Hero = true, Keyframes = [new CameraKey(0, 0.3, 0.6, 2, 0)] }, + ], + }.ToJson(); + AtomicFile.WriteAllText(folder.ProjectPath, project); + var version = (await fx.Db.Sequences.EditAsync(sequenceId, new SequenceEdit { SettingsJson = project }, null, fx.Ct)).Sequence!.SettingsVersion; + + // The later photo is appended in step with the index; the earlier one is inserted before every other. + var appended = await fx.Service.Ingest.CommitAsync(Request(fx, late), fx.Ct); + var halfway = ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequenceId))!.SettingsJson!); + Assert.Equal(3, halfway.HeroFrame); + Assert.Count(1, halfway.PendingFrames!); + Assert.Equal(early.Sha256, halfway.PendingFrames![0].Sha256); + var inserted = await fx.Service.Ingest.CommitAsync(Request(fx, early), fx.Ct); + + var row = (await fx.Db.Sequences.GetAsync(sequenceId))!; + var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId); + var placed = ProjectSettings.FromJson(row.SettingsJson!); + Assert.SequenceEqual([inserted.FrameId, results[1].FrameId], placed.ExcludedFrames.Select(i => frames[i].Id)); + Assert.Equal(appended.FrameId, frames[placed.HeroFrame!.Value].Id); + Assert.SequenceEqual([results[0].FrameId, appended.FrameId], placed.Camera.Keyframes.Select(k => frames[k.Frame].Id)); + Assert.Equal(0.3, placed.Camera.Keyframes[1].CenterX); + Assert.Null(placed.PendingFrames); + Assert.Equal(appended.FrameId, row.HeroFrameId); + Assert.Equal(version, row.SettingsVersion); + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(row.SettingsJson!), JsonNode.Parse(File.ReadAllText(folder.ProjectPath))), "project.json mirrors the index"); + } + [Test] public async Task PhotosAFolderScanFindsMoveTheProjectPositionsButUnknownMembersStay() { diff --git a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs index 275565b2..c795358e 100644 --- a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs +++ b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs @@ -2,6 +2,7 @@ using System.Net; using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Threading.Channels; +using Titano.Core.Metadata; using Titano.Core.Model; using Titano.Data.Sync; using Titano.Protocol; @@ -41,6 +42,9 @@ internal sealed class FakeSyncRemote : ISyncRemote /// Proxies clients sent, by frame id (for sequences the server cannot decode). public Dictionary SentProxies { get; } = []; + /// Edits the server accepted, in order. + public List Updates { get; } = []; + public FakeSequence Add(string? title, IEnumerable originals, string? settingsJson = null, SequenceStatus status = SequenceStatus.ProxiesBuilt | SequenceStatus.ThumbsBuilt) { @@ -105,7 +109,11 @@ internal sealed class FakeSyncRemote : ISyncRemote public Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken) { - lock (_gate) return Task.FromResult>([.. Find(sequenceId).Frames.Select((f, i) => f.Dto(sequenceId, i))]); + lock (_gate) + { + var sequence = Find(sequenceId); + return Task.FromResult>([.. sequence.Frames.Select(sequence.FrameDto)]); + } } public Task UpdateSequenceAsync(Guid id, UpdateSequenceRequest request, CancellationToken cancellationToken) @@ -115,6 +123,9 @@ internal sealed class FakeSyncRemote : ISyncRemote var s = Find(id); if (request.ExpectedSettingsVersion is { } expected && expected != s.SettingsVersion) throw new TitanoApiException(HttpStatusCode.Conflict, new ApiError("settings_conflict", "changed")); + if (request.ExpectedFramesVersion is { } frames && frames != s.FramesVersion) + throw new TitanoApiException(HttpStatusCode.Conflict, new ApiError("frames_conflict", "changed")); + Updates.Add(request); if (request.Title is not null) s.Title = request.Title.Length == 0 ? null : request.Title; if (request.Tags is not null) s.Tags = [.. request.Tags]; if (request.Rating is { } rating) s.Rating = rating; @@ -135,7 +146,8 @@ internal sealed class FakeSyncRemote : ISyncRemote throw new TitanoApiException(HttpStatusCode.BadRequest, new ApiError("hash_mismatch", "hash")); lock (_gate) { - if (Sequences.Values.SelectMany(s => s.Frames).FirstOrDefault(f => f.Sha256 == sha256) is { } existing) + // Originals it never hashed are not recognised by content. + if (Sequences.Values.Where(s => !s.Unhashed).SelectMany(s => s.Frames).FirstOrDefault(f => f.Sha256 == sha256) is { } existing) return new CompleteUploadResponse(existing.Id, sequenceId ?? Guid.Empty, -1); FakeSequence target; if (sequenceId is { } id) target = Find(id); @@ -244,16 +256,25 @@ internal sealed class FakeSequence /// The server cannot decode these originals (HEIF): its manifest asks for proxies of them. public bool Undecodable { get; set; } + /// The server adopted these originals and never verified them: it lists no SHA-256 for them. + public bool Unhashed { get; set; } + public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow; public DateTime? FirstCapture { get; set; } public string? CameraModel { get; set; } = "Canon EOS R5"; public List Frames { get; } = []; public List Renders { get; } = []; - public FakeFrame AddFrame(byte[] bytes, string? name = null, DateTime? capturedAt = null) + /// Like the server, the capture time comes from the photo's Exif data when none is given. + public FakeFrame AddFrame(byte[] bytes, string? name = null, DateTime? capturedAt = null) => InsertFrame(Frames.Count, bytes, name, capturedAt); + + /// A photo filed at (captured before the frames after it): those move up by one. + public FakeFrame InsertFrame(int index, byte[] bytes, string? name = null, DateTime? capturedAt = null) { - var frame = new FakeFrame(Guid.CreateVersion7(), name ?? $"IMG_{Frames.Count + 1:0000}.JPG", bytes, capturedAt); - Frames.Add(frame); + name ??= $"IMG_{Frames.Count + 1:0000}.JPG"; + using var stream = new MemoryStream(bytes); + var frame = new FakeFrame(Guid.CreateVersion7(), name, bytes, capturedAt ?? MetadataReader.Read(stream, name).CapturedAt); + Frames.Insert(index, frame); FramesVersion++; UpdatedAt = DateTimeOffset.UtcNow; return frame; @@ -263,9 +284,11 @@ internal sealed class FakeSequence Frames.Sum(f => (long)f.Bytes.Length), (long)Status, Renders.Count, null, Tags, Rating, Notes, SettingsVersion, FramesVersion, 0, null, UpdatedAt); public SequenceManifest Manifest(Func proxySent) => new(Id, FramesVersion, SettingsVersion, SettingsJson, Tags, - [.. Frames.Select((f, i) => new ManifestFrame(f.Id, i, f.FileName, f.Bytes.Length, f.Sha256, (Status & SequenceStatus.ProxiesBuilt) != 0 || proxySent(f.Id), - (Status & SequenceStatus.ThumbsBuilt) != 0, WantsProxy: Undecodable && !proxySent(f.Id)))], [.. Renders.Select(r => r.Dto())], - Undecodable ? new ProxySpec(1280, 90) : null); + [.. Frames.Select((f, i) => new ManifestFrame(f.Id, i, f.FileName, f.Bytes.Length, Unhashed ? string.Empty : f.Sha256, + (Status & SequenceStatus.ProxiesBuilt) != 0 || proxySent(f.Id), (Status & SequenceStatus.ThumbsBuilt) != 0, WantsProxy: Undecodable && !proxySent(f.Id)))], + [.. Renders.Select(r => r.Dto())], Undecodable ? new ProxySpec(1280, 90) : null); + + public FrameDto FrameDto(FakeFrame frame) => frame.Dto(Id, Frames.IndexOf(frame), Unhashed); } internal sealed record FakeRender(Guid Id, string FileName, string PresetName, byte[] Bytes) @@ -280,6 +303,6 @@ internal sealed record FakeFrame(Guid Id, string FileName, byte[] Bytes, DateTim /// The original stands in for its proxy: a real JPEG the decoders read. public byte[] Proxy => Bytes; - public FrameDto Dto(Guid sequenceId, int index) => - new(Id, sequenceId, index, FileName, Bytes.Length, Sha256, CapturedAt, null, 1.0 / 250, 8, 100, 24, 1, 48, 32, "jpeg", 0); + public FrameDto Dto(Guid sequenceId, int index, bool unhashed = false) => + new(Id, sequenceId, index, FileName, Bytes.Length, unhashed ? null : Sha256, CapturedAt, null, 1.0 / 250, 8, 100, 24, 1, 48, 32, "jpeg", 0); } diff --git a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs index f3ed537a..b976b40d 100644 --- a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs @@ -1,4 +1,5 @@ using Titano.Core.Model; +using Titano.Core.Project; using Titano.Data.Sync; using Titano.Protocol; using Titano.Protocol.Client; @@ -94,6 +95,85 @@ public sealed class SyncServiceTests(TestContext context) Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } + [Test] + public async Task PositionsSentToTheServerNameItsFramesAndWaitForPhotosItLacks() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(5); + var local = await f.LocalAsync("Harbour", photos); + // The server lacks the earliest photo (not uploaded yet). + var remote = f.Remote.Add("Harbour", photos[1..]); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + + await f.EditLocalSettingsAsync(local.Id, new ProjectSettings { ExcludedFrames = [0, 2], HeroFrame = 4 }.ToJson()); + await f.RefreshAsync(); + + var sent = ProjectSettings.FromJson(remote.SettingsJson!); + Assert.SequenceEqual([Sha(photos[2])], sent.ExcludedFrames.Select(i => remote.Frames[i].Sha256)); + Assert.Equal(Sha(photos[4]), remote.Frames[sent.HeroFrame!.Value].Sha256); + Assert.Count(1, sent.PendingFrames!); + var waiting = sent.PendingFrames![0]; + Assert.True(waiting.Excluded && waiting.Sha256 == Sha(photos[0]), "the exclusion of the photo the server lacks waits for it"); + Assert.Equal(remote.FramesVersion, f.Remote.Updates[^1].ExpectedFramesVersion); + // This PC's project keeps its own positions. + Assert.SequenceEqual([0, 2], ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).ExcludedFrames); + Assert.Equal(SyncStatus.LocalAhead, (await f.EntryAsync("Harbour")).Status); + Assert.Equal(1, (await f.EntryAsync("Harbour")).Link!.MissingRemote); + } + + [Test] + public async Task SettingsSentWhileTheServersFramesChangeAreTranslatedAgain() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(4); + var local = await f.LocalAsync("Harbour", photos[1..]); + var remote = f.Remote.Add("Harbour", photos[1..]); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + await f.EditLocalSettingsAsync(local.Id, new ProjectSettings { ExcludedFrames = [1] }.ToJson()); + + // An earlier photo is filed on the server after the manifest was read, before the settings arrive. + var racing = new RacingRemote(f.Remote, () => remote.InsertFrame(0, photos[0]), beforeUpdate: true); + await f.Sync.RefreshAsync(racing, new SyncPolicy(), f.Ct); + Assert.Empty(f.Remote.Updates); + Assert.Equal(SyncStatus.LocalAhead, (await f.EntryAsync("Harbour")).Status); + + await f.RefreshAsync(); + Assert.SequenceEqual([Sha(photos[2])], ProjectSettings.FromJson(remote.SettingsJson!).ExcludedFrames.Select(i => remote.Frames[i].Sha256)); + Assert.Equal((SyncStatus.ServerAhead, 1), ((await f.EntryAsync("Harbour")).Status, (await f.EntryAsync("Harbour")).Link!.MissingLocal)); + } + + [Test] + public async Task TheSameSettingsFromTheServerAreAgreedWhileTheArchiveIsOffline() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Harbour", photos, SyncFixture.Window41); + var remote = f.Remote.Add("Harbour", photos, SyncFixture.Window41); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); + + // The server's version moves with the same content (another PC sent the same values); the archive's drive is unplugged. + f.Remote.EditSettings(remote.Id, SyncFixture.Window41); + var aside = f.Archive.Paths.ManifestPath + ".unplugged"; + File.Move(f.Archive.Paths.ManifestPath, aside); + try + { + await f.RefreshAsync(); + } + finally + { + File.Move(aside, f.Archive.Paths.ManifestPath); + } + + var entry = await f.EntryAsync("Harbour"); + Assert.Null(entry.Link!.Error); + Assert.Equal(SyncStatus.InSync, entry.Status); + Assert.Equal(local.SettingsVersion, entry.Local!.SettingsVersion); + } + [Test] public async Task AnEditSavedHereDuringTheComparisonIsNotReplacedByTheServers() { @@ -441,6 +521,8 @@ public sealed class SyncServiceTests(TestContext context) Assert.Equal(originals.Id, (await f.EntryAsync("Construction cam")).Transfer!.Id); } + private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes)); + /// A linked sequence whose settings changed differently on both sides since they last agreed (41 here, 33 on the server). private static async Task<(Sequence Local, FakeSequence Remote)> ConflictAsync(SyncFixture f) { diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs index 174aa73e..6760a72f 100644 --- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs @@ -3,6 +3,7 @@ using Titano.Core.Imaging; using Titano.Core.Imaging.Decoders; using Titano.Core.Localization; using Titano.Core.Model; +using Titano.Core.Project; using Titano.Core.Render; using Titano.Data.Sync; using Titano.Testing; @@ -264,6 +265,61 @@ public sealed class SyncTransferTests(TestContext context) Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } + [Test] + public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(6); + var local = await f.LocalAsync("Harbour", photos[1..5]); + var remote = f.Remote.Add("Harbour", photos[1..5]); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + // Another PC sends the server an earlier and a later photo; then an editor there excludes two photos, picks the + // hero and keys the camera, by the server's frames. + remote.InsertFrame(0, photos[0]); + remote.AddFrame(photos[5]); + f.Remote.EditSettings(remote.Id, new ProjectSettings + { + ExcludedFrames = [0, 3], + HeroFrame = 5, + Camera = new CameraSettings { Keyframes = [new CameraKey(2, 0.4, 0.5, 1.5, 0)] }, + }.ToJson()); + + await f.RefreshAsync(); + var taken = await NamedAsync(f, local.Id); + Assert.SequenceEqual([Sha(photos[3])], taken.Excluded); + Assert.Null(taken.Hero); + Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes); + Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them"); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + // The same photos as on the server, and nothing waits any more. + var arrived = await NamedAsync(f, local.Id); + Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], arrived.Excluded); + Assert.Equal(Sha(photos[5]), arrived.Hero); + Assert.SequenceEqual([Sha(photos[2])], arrived.Keyframes); + Assert.Empty(arrived.Pending); + var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; + var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); + Assert.Equal(Sha(photos[5]), frames.Single(fr => fr.Id == row.HeroFrameId).Sha256); + Assert.True(SyncRules.SameSettings(row.SettingsJson, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(row.Folder).ProjectPath, f.Ct)), + "project.json mirrors the index"); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); + + // An edit here later sends the server the same photos, by its frames. + var edited = ProjectSettings.FromJson(row.SettingsJson!) with { Exposure = new ExposureSettings { DeflickerWindow = new Auto().WithManual(41) } }; + await f.EditLocalSettingsAsync(local.Id, edited.ToJson()); + await f.RefreshAsync(); + var server = ProjectSettings.FromJson(remote.SettingsJson!); + Assert.Equal(41, server.Exposure.DeflickerWindow.Manual); + Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], server.ExcludedFrames.Select(i => remote.Frames[i].Sha256)); + Assert.Equal(Sha(photos[5]), remote.Frames[server.HeroFrame!.Value].Sha256); + Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256)); + } + [Test] public async Task TheServersRendersAreFetchedIntoTheLocalExports() { @@ -407,4 +463,13 @@ public sealed class SyncTransferTests(TestContext context) } private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes)); + + /// The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for. + private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId) + { + var project = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(localId, f.Ct))!.SettingsJson!); + var frames = await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct); + return ([.. project.ExcludedFrames.Select(i => frames[i].Sha256!)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null, + [.. project.Camera.Keyframes.Select(k => frames[k.Frame].Sha256!)], [.. (project.PendingFrames ?? []).Select(p => p.Sha256!)]); + } } diff --git a/tests/Titano.Server.Tests/Api/SequenceTests.cs b/tests/Titano.Server.Tests/Api/SequenceTests.cs index ca46a573..3d32d116 100644 --- a/tests/Titano.Server.Tests/Api/SequenceTests.cs +++ b/tests/Titano.Server.Tests/Api/SequenceTests.cs @@ -60,6 +60,32 @@ public sealed class SequenceTests(TestContext context) await invalid.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_settings"); } + [Test] + public async Task SettingsWrittenAgainstFramesThatChangedAreRefused() + { + await using var server = await TestServer.StartAsync(context); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var first = await Uploads.UploadAsync(editor, Uploads.Photo(2000, 41), "a.CR3", sessionKey: "k"); + using var read = await editor.GetAsync($"/api/v1/sequences/{first.SequenceId}"); + var sequence = await read.ReadAsync(ProtocolJson.Default.SequenceDto); + // Another photo arrives after this client read the frames its positions name. + await Uploads.UploadAsync(editor, Uploads.Photo(2000, 42), "b.CR3", sessionKey: "k"); + + using (var stale = await editor.PatchAsJsonAsync($"/api/v1/sequences/{sequence.Id}", + new UpdateSequenceRequest(null, null, null, null, """{"excludedFrames":[0]}""", sequence.SettingsVersion, ExpectedFramesVersion: sequence.FramesVersion), + ProtocolJson.Default.UpdateSequenceRequest)) + { + var error = await stale.ExpectErrorAsync(HttpStatusCode.Conflict, "frames_conflict"); + Assert.Equal((sequence.FramesVersion + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), error.Details); + } + + using var fresh = await editor.PatchAsJsonAsync($"/api/v1/sequences/{sequence.Id}", + new UpdateSequenceRequest(null, null, null, null, """{"excludedFrames":[1]}""", sequence.SettingsVersion, ExpectedFramesVersion: sequence.FramesVersion + 1), + ProtocolJson.Default.UpdateSequenceRequest); + await fresh.ExpectAsync(HttpStatusCode.OK); + Assert.Equal(sequence.SettingsVersion + 1, (await fresh.ReadAsync(ProtocolJson.Default.SequenceDto)).SettingsVersion); + } + [Test] public async Task TheManifestCarriesHashesAndAnEtag() { diff --git a/tests/Titano.Server.Tests/Sync/SyncClient.cs b/tests/Titano.Server.Tests/Sync/SyncClient.cs index 34539ac5..a0ac0202 100644 --- a/tests/Titano.Server.Tests/Sync/SyncClient.cs +++ b/tests/Titano.Server.Tests/Sync/SyncClient.cs @@ -85,6 +85,26 @@ internal sealed class SyncClient : IAsyncDisposable return edited.Sequence!; } + /// One more photo imported into a local sequence, captured at (before others: it is inserted). + public async Task AddPhotoAsync(Guid sequenceId, byte[] photo, DateTime captured) + { + var staging = Path.Combine(Archive.RootPath, ".titano", "staging", "test"); + Directory.CreateDirectory(staging); + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part"); + await File.WriteAllBytesAsync(path, photo, Ct); + return await Archives.Ingest.CommitAsync(new IngestRequest(Archive.Id, sequenceId, null, null, path, $"IMG_{captured:HHmmss}.JPG", photo.Length, Sha(photo), + captured, null), Ct); + } + + /// An edit on this PC, as the editor makes it (project file and index, settings version + 1). + public async Task EditSettingsAsync(Guid sequenceId, string settingsJson) + { + var sequence = (await Db.Sequences.GetAsync(sequenceId, Ct))!; + await File.WriteAllTextAsync(new Titano.Core.Archiving.ArchivePaths(Archive.RootPath).Sequence(sequence.Folder).ProjectPath, settingsJson, Ct); + return (await Db.Sequences.EditAsync(sequenceId, new Titano.Data.Repositories.SequenceEdit { SettingsJson = settingsJson }, sequence.SettingsVersion, Ct)) + .Sequence!; + } + /// A render of made on this PC: its file in the sequence's exports and its row. public async Task RenderAsync(Sequence sequence, string fileName, byte[] bytes) { diff --git a/tests/Titano.Server.Tests/Sync/SyncServerTests.cs b/tests/Titano.Server.Tests/Sync/SyncServerTests.cs index effc6a7c..ae0894d9 100644 --- a/tests/Titano.Server.Tests/Sync/SyncServerTests.cs +++ b/tests/Titano.Server.Tests/Sync/SyncServerTests.cs @@ -249,6 +249,117 @@ public sealed class SyncServerTests(TestContext context) Assert.Equal(0, (await laptop.EntryAsync("Star trails")).Link!.ExportsToGet); } + [Test] + public async Task PositionsEditedOnTheServerStayOnTheirPhotosWhenAnotherPcDownloadsTheInsertedFrames() + { + await using var server = await StartServerAsync(); + var (http, _, token) = await server.UserAsync("alberto", "editor"); + await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); + await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop"); + var photos = Enumerable.Range(60, 5).Select(SyncClient.Photo).ToList(); + // Both PCs hold the last four photos, the third of them excluded. + var local = await studio.LocalAsync("Harbour at dawn", photos[1..], new ProjectSettings { ExcludedFrames = [2] }.ToJson()); + await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); + var remoteId = (await studio.EntryAsync("Harbour at dawn")).Link!.RemoteId; + await ServerIdleAsync(server); + await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); + await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Harbour at dawn", SyncDownloadMode.Originals, + laptop.Archive.Id)).Id); + var copy = (await laptop.EntryAsync("Harbour at dawn")).Local!; + + // The studio files an earlier photo and uploads it: the server inserts it first and moves its own positions. + await studio.AddPhotoAsync(local.Id, photos[0], new DateTime(2026, 9, 28, 17, 59, 50)); + await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); + await ServerIdleAsync(server); + Assert.SequenceEqual([SyncClient.Sha(photos[3])], (await ServerNamedAsync(server, remoteId)).Excluded); + + // An editor on the server excludes the new photo too, by the server's frames, and changes the deflicker window. + var row = (await server.Database.Sequences.GetAsync(remoteId))!; + var current = ProjectSettings.FromJson(row.SettingsJson!); + var edit = current with + { + ExcludedFrames = [0, .. current.ExcludedFrames], + Exposure = current.Exposure with { DeflickerWindow = new Auto().WithManual(33) }, + }; + using (var patch = await http.PatchAsJsonAsync($"/api/v1/sequences/{remoteId}", + new UpdateSequenceRequest(null, null, null, null, edit.ToJson(), row.SettingsVersion), ProtocolJson.Default.UpdateSequenceRequest)) + await patch.ExpectAsync(HttpStatusCode.OK); + + // The laptop takes the edit while it still lacks the new photo, then downloads it. + await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); + var taken = await LocalNamedAsync(laptop, copy.Id); + Assert.Equal(33, taken.Project.Exposure.DeflickerWindow.Manual); + Assert.SequenceEqual([SyncClient.Sha(photos[3])], taken.Excluded); + Assert.True(taken.Project.PendingFrames is [{ Excluded: true } waiting] && waiting.Sha256 == SyncClient.Sha(photos[0]), "the new photo's exclusion waits for it"); + await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Harbour at dawn", SyncDownloadMode.Originals, + null)).Id); + + var onServer = await ServerNamedAsync(server, remoteId); + var here = await LocalNamedAsync(laptop, copy.Id); + Assert.SequenceEqual([SyncClient.Sha(photos[0]), SyncClient.Sha(photos[3])], here.Excluded); + Assert.SequenceEqual(onServer.Excluded, here.Excluded); + Assert.Equal(onServer.Hero, here.Hero); + Assert.Null(here.Project.PendingFrames); + await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); + Assert.Equal(SyncStatus.InSync, (await laptop.EntryAsync("Harbour at dawn")).Status); + } + + [Test] + public async Task PositionsSentBeforeTheirPhotosStayOnThemWhenTheUploadArrives() + { + await using var server = await StartServerAsync(); + var (_, _, token) = await server.UserAsync("alberto", "editor"); + await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); + var photos = Enumerable.Range(70, 5).Select(SyncClient.Photo).ToList(); + var local = await studio.LocalAsync("Pier", photos[1..]); + await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); + var remoteId = (await studio.EntryAsync("Pier")).Link!.RemoteId; + await ServerIdleAsync(server); + await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation); + + // This PC files an earlier photo and, before uploading it, excludes it and a later one and picks the hero. + await studio.AddPhotoAsync(local.Id, photos[0], new DateTime(2026, 9, 28, 17, 59, 50)); + var project = ProjectSettings.FromJson((await studio.Db.Sequences.GetAsync(local.Id))!.SettingsJson ?? "{}"); + await studio.EditSettingsAsync(local.Id, (project with { ExcludedFrames = [0, 3], HeroFrame = 1 }).ToJson()); + await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation); + + var sent = await ServerNamedAsync(server, remoteId); + Assert.SequenceEqual([SyncClient.Sha(photos[3])], sent.Excluded); + Assert.Equal(SyncClient.Sha(photos[1]), sent.Hero); + Assert.True(sent.Project.PendingFrames is [{ Excluded: true } waiting] && waiting.Sha256 == SyncClient.Sha(photos[0]), "the exclusion waits for its photo"); + + await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); + await ServerIdleAsync(server); + + var arrived = await ServerNamedAsync(server, remoteId); + Assert.SequenceEqual([SyncClient.Sha(photos[0]), SyncClient.Sha(photos[3])], arrived.Excluded); + Assert.Equal(SyncClient.Sha(photos[1]), arrived.Hero); + Assert.Null(arrived.Project.PendingFrames); + var heroFrame = (await server.Database.Sequences.GetAsync(remoteId))!.HeroFrameId; + Assert.Equal(SyncClient.Sha(photos[1]), (await server.Database.Frames.ListBySequenceAsync(remoteId)).Single(f => f.Id == heroFrame).Sha256); + var here = await LocalNamedAsync(studio, local.Id); + Assert.SequenceEqual(arrived.Excluded, here.Excluded); + Assert.Equal(arrived.Hero, here.Hero); + await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation); + Assert.Equal(SyncStatus.InSync, (await studio.EntryAsync("Pier")).Status); + } + + /// The server's project and the photos (SHA-256) it excludes and shows as hero. + private static async Task<(ProjectSettings Project, string?[] Excluded, string? Hero)> ServerNamedAsync(TestServer server, Guid remoteId) + { + var project = ProjectSettings.FromJson((await server.Database.Sequences.GetAsync(remoteId))!.SettingsJson ?? "{}"); + var frames = await server.Database.Frames.ListBySequenceAsync(remoteId); + return (project, [.. project.ExcludedFrames.Select(i => frames[i].Sha256)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null); + } + + /// A PC's project and the photos (SHA-256) it excludes and shows as hero. + private static async Task<(ProjectSettings Project, string?[] Excluded, string? Hero)> LocalNamedAsync(SyncClient client, Guid localId) + { + var project = ProjectSettings.FromJson((await client.Db.Sequences.GetAsync(localId))!.SettingsJson ?? "{}"); + var frames = await client.Db.Frames.ListBySequenceAsync(localId); + return (project, [.. project.ExcludedFrames.Select(i => frames[i].Sha256)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null); + } + private Task StartServerAsync() => TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register); /// Waits until the server's engines finished what the commits queued (thumbnails, proxies, analysis, which may edit settings).