From 01e17c99617a3d8fbf99b33c26ff5b64af3705c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 09:40:40 +0000 Subject: [PATCH] fix(archive): keep project frame positions on their frames after inserts The project names frames by position (exclusions, hero, virtual-camera keyframes), and an upload, import, scan or sync download of earlier photos merges them by capture time and renumbers the frames after them; nothing moved the project, so a render excluded the wrong photo and the project's hero no longer matched the Library's. project.json keeps its format: when the index takes a sidecar in which frames were inserted before indexed ones, the index's copy of the project has its positions moved from the indexed order to the new one in the same transaction as the frame rows, and project.json mirrors it unless another writer changed it meanwhile. The JSON is edited in place, so members a newer Titano wrote survive; only an insertion is followed (if an indexed frame is gone, the positions stay). The settings version stays, so a synced copy receiving the same frames makes the same move (CD-2). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Project/ProjectSettings.cs | 51 ++++++++++++- src/Titano.Data/Archiving/ArchiveIndexer.cs | 12 ++- src/Titano.Data/Archiving/ArchiveIngest.cs | 2 +- src/Titano.Data/Archiving/ProjectPositions.cs | 75 +++++++++++++++++++ .../Repositories/FrameRepository.cs | 8 ++ .../Project/ProjectSettingsTests.cs | 46 ++++++++++++ .../Archiving/ArchiveIngestTests.cs | 61 +++++++++++++++ 7 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 src/Titano.Data/Archiving/ProjectPositions.cs diff --git a/src/Titano.Core/Project/ProjectSettings.cs b/src/Titano.Core/Project/ProjectSettings.cs index 9a52011a..d6c22280 100644 --- a/src/Titano.Core/Project/ProjectSettings.cs +++ b/src/Titano.Core/Project/ProjectSettings.cs @@ -174,6 +174,9 @@ public sealed record ProjectSettings { public const int CurrentVersion = 1; + /// Layout of a project edited as JSON (): indented like . + private static readonly JsonSerializerOptions IndentedJson = new() { WriteIndented = true }; + public int Version { get; init; } = CurrentVersion; public TimingSettings Timing { get; init; } = new(); public ExposureSettings Exposure { get; init; } = new(); @@ -182,7 +185,7 @@ public sealed record ProjectSettings public StackSettings Stack { get; init; } = new(); public ExportSettings Export { get; init; } = new(); - /// Frame indices excluded from renders. + /// Frame indices excluded from renders (positions in capture order; see ). public IReadOnlyList ExcludedFrames { get; init; } = []; /// Index of the hero frame (sequence thumbnail). @@ -237,6 +240,52 @@ public sealed record ProjectSettings target[name] = value?.DeepClone(); } } + + /// + /// Moves the frame positions of a project — , and the + /// virtual-camera s — after frames were inserted before them: maps + /// an old position to the new one, and a position it does not list stays. The JSON is edited in place, so members + /// this Titano does not know survive. Null when nothing moves or is not a project object. + /// + public static string? MoveFramePositions(string json, IReadOnlyDictionary moves) + { + ArgumentNullException.ThrowIfNull(moves); + if (moves.Count == 0 || string.IsNullOrWhiteSpace(json)) return null; + try + { + if (JsonNode.Parse(json, new JsonNodeOptions { PropertyNameCaseInsensitive = true }) is not JsonObject project) return null; + var moved = false; + if (project["excludedFrames"] is JsonArray excluded) + { + for (var i = 0; i < excluded.Count; i++) + { + var at = i; + moved |= Move(excluded[at], value => excluded[at] = value, moves); + } + } + if (project["heroFrame"] is { } hero) moved |= Move(hero, value => project["heroFrame"] = value, moves); + if (project["camera"] is JsonObject camera && camera["keyframes"] is JsonArray keyframes) + { + foreach (var key in keyframes.OfType()) + { + if (key["frame"] is { } frame) moved |= Move(frame, value => key["frame"] = value, moves); + } + } + return moved ? project.ToJsonString(IndentedJson) : null; + } + catch (Exception ex) when (ex is JsonException or ArgumentException) + { + // Not JSON, or a member given twice (the case-insensitive member table refuses it): not a project to edit. + return null; + } + } + + private static bool Move(JsonNode? position, Action replace, IReadOnlyDictionary moves) + { + if (position is not JsonValue value || !value.TryGetValue(out var old) || !moves.TryGetValue(old, out var now) || now == old) return false; + replace(JsonValue.Create(now)); + return true; + } } [JsonSourceGenerationOptions( diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index ff13cb34..5561c3d6 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -91,18 +91,28 @@ public sealed class ArchiveIndexer return new ScanSummary(folders.Count, indexed, adopted, removed, failed); } - /// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. + /// + /// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames + /// inserted before indexed ones take the project's frame positions along (). + /// internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) { + (string Before, string Moved)? positions = null; await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) { var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false); + if (existing is not null) + { + positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false); + if (positions is { } project) existing = existing with { SettingsJson = project.Moved }; + } await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false); foreach (var key in manifest.SessionKeys) await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); } + if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved); await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); } diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index c23f1b86..ad100883 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -456,7 +456,7 @@ public sealed class ArchiveIngest : IArchiveIngest /// /// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose - /// flags change) resyncs the whole sequence. + /// flags change) resyncs the whole sequence, which also moves the project's frame positions after an insertion. /// private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended, bool repaired, CancellationToken cancellationToken) diff --git a/src/Titano.Data/Archiving/ProjectPositions.cs b/src/Titano.Data/Archiving/ProjectPositions.cs new file mode 100644 index 00000000..8aefeab7 --- /dev/null +++ b/src/Titano.Data/Archiving/ProjectPositions.cs @@ -0,0 +1,75 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.IO; +using Titano.Core.Model; +using Titano.Core.Project; + +namespace Titano.Data.Archiving; + +/// +/// Keeps a project's frame positions (exclusions, hero, virtual-camera keyframes; BRIEF §5, §7) on their frames when the +/// index takes a sidecar in which frames were inserted before others: an upload or import of earlier photos, photos a +/// scan found, a sync download, a journalled batch completed after a crash. Positions name the frames in the indexed +/// order (what editors, renders and the API read), so the old order comes from the index rows. Only an insertion is +/// followed: when an indexed frame is no longer in the sidecar, positions cannot be mapped and stay as they are. The move +/// is part of the frames change, not a settings edit, so the settings version stays and a synced copy receiving the +/// same frames makes the same move. +/// +internal static class ProjectPositions +{ + /// + /// The index's copy of the project (the one in force) and the same project with its positions moved to the order of + /// ; null when nothing moves. A project.json the index has no copy of was written by + /// another process against its own frames, so it is left to that writer. + /// + public static async Task<(string Before, string Moved)?> MoveAsync(TitanoRepositories db, Sequence indexed, SequenceManifest manifest, + CancellationToken cancellationToken) + { + // The sidecar's frames are the ones indexed: nothing was inserted. + if (indexed.FramesVersion == manifest.Versions.Frames) return null; + if (indexed.SettingsJson is not { } project) return null; + var rows = await db.Frames.ListPositionsAsync(indexed.Id, cancellationToken).ConfigureAwait(false); + var now = new Dictionary(manifest.Frames.Count); + foreach (var frame in manifest.Frames) now[frame.Id] = frame.I; + var moves = new Dictionary(); + foreach (var row in rows) + { + if (!now.TryGetValue(row.FrameId, out var index)) return null; + if (index != row.Index) moves[row.Index] = index; + } + return ProjectSettings.MoveFramePositions(project, moves) is { } moved ? (project, moved) : null; + } + + /// + /// Mirrors moved positions into project.json once the index holds them, unless the file was written since + /// (it then belongs to that writer, whose next save mirrors the index again). A failed write leaves the index right. + /// + public static void WriteProject(SequencePaths sequence, string before, string moved) + { + try + { + if (File.Exists(sequence.ProjectPath) && !SameJson(File.ReadAllText(sequence.ProjectPath), before)) return; + Directory.CreateDirectory(sequence.MetaDirectory); + AtomicFile.WriteAllText(sequence.ProjectPath, moved); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("archive", $"Project of {sequence.RelativeFolder} keeps its old frame positions on disk", ex); + } + } + + /// Same JSON content whatever the layout (an index may store it normalised). + private static bool SameJson(string a, string b) + { + try + { + return JsonNode.DeepEquals(JsonNode.Parse(a), JsonNode.Parse(b)); + } + catch (JsonException) + { + return false; + } + } +} diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs index 7cba5c45..d89aa320 100644 --- a/src/Titano.Data/Repositories/FrameRepository.cs +++ b/src/Titano.Data/Repositories/FrameRepository.cs @@ -10,6 +10,9 @@ public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, str /// A frame and the sequence it belongs to. public readonly record struct FrameRef(Guid FrameId, Guid SequenceId); +/// Where a frame stands in its sequence. +public readonly record struct FramePosition(Guid FrameId, int Index); + /// A frame whose content (SHA-256) also exists elsewhere in the archive. public sealed record DuplicateFrame(string Sha256, Guid FrameId, Guid SequenceId, int Index, string RelativePath); @@ -74,6 +77,11 @@ public sealed class FrameRepository _db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 AND (flags & $2) = 0 ORDER BY sequence_id, idx", [sha256.ToLowerInvariant(), (int)(FrameFlags.Missing | FrameFlags.Corrupt)], Map, cancellationToken); + /// Id and index of every frame of a sequence, in index order. + public Task> ListPositionsAsync(Guid sequenceId, CancellationToken cancellationToken = default) => + _db.ListAsync("SELECT id, idx FROM frames WHERE sequence_id = $1 ORDER BY idx", [sequenceId], r => new FramePosition(r.GetGuid(0), r.GetInt32(1)), + cancellationToken); + /// Writes analysis results for many frames in one transaction; returns the number of frames updated. public Task UpdateLightStatsAsync(IReadOnlyList stats, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => diff --git a/tests/Titano.Core.Tests/Project/ProjectSettingsTests.cs b/tests/Titano.Core.Tests/Project/ProjectSettingsTests.cs index 2536f475..2eb939f7 100644 --- a/tests/Titano.Core.Tests/Project/ProjectSettingsTests.cs +++ b/tests/Titano.Core.Tests/Project/ProjectSettingsTests.cs @@ -121,6 +121,52 @@ public sealed class ProjectSettingsTests public void ProjectMustBeAnObject() => Assert.Throws(() => ProjectSettings.FromJson("[1, 2]")); + [Test] + public void FramePositionsFollowInsertedFramesAndOtherMembersStay() + { + var project = new ProjectSettings + { + ExcludedFrames = [3, 7, 40], + HeroFrame = 2, + Camera = new CameraSettings { Keyframes = [new CameraKey(0, 0.5, 0.5, 1, 0), new CameraKey(7, 0.4, 0.5, 1.5, 0)] }, + Exposure = new ExposureSettings { DeflickerWindow = new Auto().WithManual(41) }, + }; + // Two frames inserted before position 3 and one before position 7; 40 is no frame of the old list and stays. + var moves = new Dictionary { [0] = 0, [1] = 1, [2] = 4, [3] = 5, [4] = 6, [5] = 7, [6] = 8, [7] = 10 }; + + var moved = ProjectSettings.FromJson(Assert.NotNull(ProjectSettings.MoveFramePositions(project.ToJson(), moves))); + + Assert.SequenceEqual([5, 10, 40], moved.ExcludedFrames); + Assert.Equal(4, moved.HeroFrame); + Assert.SequenceEqual([0, 10], moved.Camera.Keyframes.Select(k => k.Frame)); + Assert.Equal((0.4, 1.5), (moved.Camera.Keyframes[1].CenterX, moved.Camera.Keyframes[1].Zoom)); + Assert.Equal(41, moved.Exposure.DeflickerWindow.Manual); + } + + [Test] + public void MovingPositionsEditsTheJsonInPlace() + { + const string project = """{ "ExcludedFrames": [1, 2], "heroFrame": null, "futureOption": { "keep": [1, 2] }, "camera": { "keyframes": [ { "frame": 1.5 } ] } }"""; + + var moved = Assert.NotNull(ProjectSettings.MoveFramePositions(project, new Dictionary { [1] = 3 })); + + var json = System.Text.Json.Nodes.JsonNode.Parse(moved)!; + Assert.SequenceEqual([3, 2], json["ExcludedFrames"]!.AsArray().Select(n => n!.GetValue())); + Assert.Equal("""{"keep":[1,2]}""", json["futureOption"]!.ToJsonString()); + Assert.Null(json["heroFrame"]); + Assert.Equal(1.5, json["camera"]!["keyframes"]![0]!["frame"]!.GetValue()); + } + + [Test] + public void NothingToMoveGivesNoNewProject() + { + Assert.Null(ProjectSettings.MoveFramePositions(new ProjectSettings { ExcludedFrames = [1] }.ToJson(), new Dictionary())); + Assert.Null(ProjectSettings.MoveFramePositions(new ProjectSettings { ExcludedFrames = [1] }.ToJson(), new Dictionary { [2] = 3, [1] = 1 })); + Assert.Null(ProjectSettings.MoveFramePositions("not json", new Dictionary { [1] = 2 })); + Assert.Null(ProjectSettings.MoveFramePositions("[1, 2]", new Dictionary { [1] = 2 })); + Assert.Null(ProjectSettings.MoveFramePositions("""{ "heroFrame": 1, "HeroFrame": 1 }""", new Dictionary { [1] = 2 })); + } + [Test] public void BuiltInPresetIdsAreUnique() => Assert.Equal(ExportPresets.BuiltIn.Count, ExportPresets.BuiltIn.Select(p => p.Id).Distinct().Count()); diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs index 30ad8460..2913ad7e 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -1,6 +1,10 @@ +using System.Text.Json.Nodes; using Titano.Core.Archiving; +using Titano.Core.IO; using Titano.Core.Model; +using Titano.Core.Project; using Titano.Data.Archiving; +using Titano.Data.Repositories; using Titano.Testing; using static Titano.Data.Tests.Archiving.ArchiveFixture; @@ -215,4 +219,61 @@ public sealed class ArchiveIngestTests(TestContext context) // The index can be rebuilt from the sidecar it now has. Assert.Equal(0, (await fx.ScanAsync()).Indexed); } + + [Test] + public async Task FramesInsertedBeforeOthersTakeTheProjectPositionsAlong() + { + await using var fx = await CreateAsync(context); + var results = new List(); + for (var i = 0; i < 5; i++) results.Add(await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1 + i), 10 + i)), fx.Ct)); + var sequenceId = results[0].SequenceId; + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(sequenceId))!.Folder); + var project = new ProjectSettings + { + ExcludedFrames = [3], + HeroFrame = 1, + Camera = new CameraSettings { Keyframes = [new CameraKey(0, 0.5, 0.5, 1, 0), new CameraKey(4, 0.4, 0.5, 1.5, 0)] }, + }.ToJson(); + AtomicFile.WriteAllText(folder.ProjectPath, project); + var edited = await fx.Db.Sequences.EditAsync(sequenceId, new SequenceEdit { SettingsJson = project, HeroFrameId = results[1].FrameId }, null, fx.Ct); + var version = edited.Sequence!.SettingsVersion; + + // Two photos captured before every other one, and one between the third and the fourth, arrive later. + await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 20)), fx.Ct); + await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(30), 21)), fx.Ct); + await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(3.5), 22)), fx.Ct); + + var row = (await fx.Db.Sequences.GetAsync(sequenceId))!; + var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId); + var moved = ProjectSettings.FromJson(row.SettingsJson!); + Assert.SequenceEqual([results[3].FrameId], moved.ExcludedFrames.Select(i => frames[i].Id)); + Assert.Equal(results[1].FrameId, frames[moved.HeroFrame!.Value].Id); + Assert.SequenceEqual([results[0].FrameId, results[4].FrameId], moved.Camera.Keyframes.Select(k => frames[k.Frame].Id)); + Assert.Equal(version, row.SettingsVersion); + Assert.Equal(results[1].FrameId, row.HeroFrameId); + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(row.SettingsJson!), JsonNode.Parse(File.ReadAllText(folder.ProjectPath))), "project.json mirrors the index"); + } + + [Test] + public async Task PhotosAFolderScanFindsMoveTheProjectPositionsButUnknownMembersStay() + { + await using var fx = await CreateAsync(context); + var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct); + await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(2), 2)), fx.Ct); + var row = (await fx.Db.Sequences.GetAsync(late.SequenceId))!; + var folder = fx.Paths.Sequence(row.Folder); + // A project with a member from a newer Titano; only the index copy holds it (the API edited it). + const string project = "{ \"excludedFrames\": [1], \"heroFrame\": 0, \"futureOption\": { \"keep\": true } }"; + await fx.Db.Sequences.EditAsync(late.SequenceId, new SequenceEdit { SettingsJson = project }, null, fx.Ct); + + File.WriteAllBytes(folder.File("originals/early.JPG"), Photos.Jpeg(Start, 3)); + await fx.Service.Indexer.ReconcileAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(late.SequenceId))!, fx.Ct); + + var settings = (await fx.Db.Sequences.GetAsync(late.SequenceId))!.SettingsJson!; + var moved = ProjectSettings.FromJson(settings); + Assert.SequenceEqual([2], moved.ExcludedFrames); + Assert.Equal(1, moved.HeroFrame); + Assert.Contains("futureOption", settings); + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(settings), JsonNode.Parse(File.ReadAllText(folder.ProjectPath))), "project.json mirrors the index"); + } }