fix(archive): keep project frame positions on their frames after inserts
The project names frames by position (exclusions, hero, virtual-camera keyframes), and an upload, import, scan or sync download of earlier photos merges them by capture time and renumbers the frames after them; nothing moved the project, so a render excluded the wrong photo and the project's hero no longer matched the Library's. project.json keeps its format: when the index takes a sidecar in which frames were inserted before indexed ones, the index's copy of the project has its positions moved from the indexed order to the new one in the same transaction as the frame rows, and project.json mirrors it unless another writer changed it meanwhile. The JSON is edited in place, so members a newer Titano wrote survive; only an insertion is followed (if an indexed frame is gone, the positions stay). The settings version stays, so a synced copy receiving the same frames makes the same move (CD-2). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
7 files changed
+252
-3
No files matched your search
@@ -174,6 +174,9 @@ public sealed record ProjectSettings
|
||||
{
|
||||
public const int CurrentVersion = 1;
|
||||
|
||||
/// <summary>Layout of a project edited as JSON (<see cref="MoveFramePositions"/>): indented like <see cref="ToJson"/>.</summary>
|
||||
private static readonly JsonSerializerOptions IndentedJson = new() { WriteIndented = true };
|
||||
|
||||
public int Version { get; init; } = CurrentVersion;
|
||||
public TimingSettings Timing { get; init; } = new();
|
||||
public ExposureSettings Exposure { get; init; } = new();
|
||||
@@ -182,7 +185,7 @@ public sealed record ProjectSettings
|
||||
public StackSettings Stack { get; init; } = new();
|
||||
public ExportSettings Export { get; init; } = new();
|
||||
|
||||
/// <summary>Frame indices excluded from renders.</summary>
|
||||
/// <summary>Frame indices excluded from renders (positions in capture order; see <see cref="MoveFramePositions"/>).</summary>
|
||||
public IReadOnlyList<int> ExcludedFrames { get; init; } = [];
|
||||
|
||||
/// <summary>Index of the hero frame (sequence thumbnail).</summary>
|
||||
@@ -237,6 +240,52 @@ public sealed record ProjectSettings
|
||||
target[name] = value?.DeepClone();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves the frame positions of a project — <see cref="ExcludedFrames"/>, <see cref="HeroFrame"/> and the
|
||||
/// virtual-camera <see cref="CameraKey.Frame"/>s — after frames were inserted before them: <paramref name="moves"/> maps
|
||||
/// an old position to the new one, and a position it does not list stays. The JSON is edited in place, so members
|
||||
/// this Titano does not know survive. Null when nothing moves or <paramref name="json"/> is not a project object.
|
||||
/// </summary>
|
||||
public static string? MoveFramePositions(string json, IReadOnlyDictionary<int, int> moves)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(moves);
|
||||
if (moves.Count == 0 || string.IsNullOrWhiteSpace(json)) return null;
|
||||
try
|
||||
{
|
||||
if (JsonNode.Parse(json, new JsonNodeOptions { PropertyNameCaseInsensitive = true }) is not JsonObject project) return null;
|
||||
var moved = false;
|
||||
if (project["excludedFrames"] is JsonArray excluded)
|
||||
{
|
||||
for (var i = 0; i < excluded.Count; i++)
|
||||
{
|
||||
var at = i;
|
||||
moved |= Move(excluded[at], value => excluded[at] = value, moves);
|
||||
}
|
||||
}
|
||||
if (project["heroFrame"] is { } hero) moved |= Move(hero, value => project["heroFrame"] = value, moves);
|
||||
if (project["camera"] is JsonObject camera && camera["keyframes"] is JsonArray keyframes)
|
||||
{
|
||||
foreach (var key in keyframes.OfType<JsonObject>())
|
||||
{
|
||||
if (key["frame"] is { } frame) moved |= Move(frame, value => key["frame"] = value, moves);
|
||||
}
|
||||
}
|
||||
return moved ? project.ToJsonString(IndentedJson) : null;
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or ArgumentException)
|
||||
{
|
||||
// Not JSON, or a member given twice (the case-insensitive member table refuses it): not a project to edit.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool Move(JsonNode? position, Action<JsonNode> replace, IReadOnlyDictionary<int, int> moves)
|
||||
{
|
||||
if (position is not JsonValue value || !value.TryGetValue<int>(out var old) || !moves.TryGetValue(old, out var now) || now == old) return false;
|
||||
replace(JsonValue.Create(now));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(
|
||||
|
||||
@@ -91,18 +91,28 @@ public sealed class ArchiveIndexer
|
||||
return new ScanSummary(folders.Count, indexed, adopted, removed, failed);
|
||||
}
|
||||
|
||||
/// <summary>Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps.</summary>
|
||||
/// <summary>
|
||||
/// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames
|
||||
/// inserted before indexed ones take the project's frame positions along (<see cref="ProjectPositions"/>).
|
||||
/// </summary>
|
||||
internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
(string Before, string Moved)? positions = null;
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null)
|
||||
{
|
||||
positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false);
|
||||
if (positions is { } project) existing = existing with { SettingsJson = project.Moved };
|
||||
}
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
|
||||
foreach (var key in manifest.SessionKeys)
|
||||
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved);
|
||||
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
|
||||
@@ -456,7 +456,7 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
|
||||
/// <summary>
|
||||
/// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose
|
||||
/// flags change) resyncs the whole sequence.
|
||||
/// flags change) resyncs the whole sequence, which also moves the project's frame positions after an insertion.
|
||||
/// </summary>
|
||||
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List<SidecarFrame> appended,
|
||||
bool repaired, CancellationToken cancellationToken)
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Keeps a project's frame positions (exclusions, hero, virtual-camera keyframes; BRIEF §5, §7) on their frames when the
|
||||
/// index takes a sidecar in which frames were inserted before others: an upload or import of earlier photos, photos a
|
||||
/// scan found, a sync download, a journalled batch completed after a crash. Positions name the frames in the indexed
|
||||
/// order (what editors, renders and the API read), so the old order comes from the index rows. Only an insertion is
|
||||
/// followed: when an indexed frame is no longer in the sidecar, positions cannot be mapped and stay as they are. The move
|
||||
/// is part of the frames change, not a settings edit, so the settings version stays and a synced copy receiving the
|
||||
/// same frames makes the same move.
|
||||
/// </summary>
|
||||
internal static class ProjectPositions
|
||||
{
|
||||
/// <summary>
|
||||
/// The index's copy of the project (the one in force) and the same project with its positions moved to the order of
|
||||
/// <paramref name="manifest"/>; null when nothing moves. A <c>project.json</c> the index has no copy of was written by
|
||||
/// another process against its own frames, so it is left to that writer.
|
||||
/// </summary>
|
||||
public static async Task<(string Before, string Moved)?> MoveAsync(TitanoRepositories db, Sequence indexed, SequenceManifest manifest,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// The sidecar's frames are the ones indexed: nothing was inserted.
|
||||
if (indexed.FramesVersion == manifest.Versions.Frames) return null;
|
||||
if (indexed.SettingsJson is not { } project) return null;
|
||||
var rows = await db.Frames.ListPositionsAsync(indexed.Id, cancellationToken).ConfigureAwait(false);
|
||||
var now = new Dictionary<Guid, int>(manifest.Frames.Count);
|
||||
foreach (var frame in manifest.Frames) now[frame.Id] = frame.I;
|
||||
var moves = new Dictionary<int, int>();
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (!now.TryGetValue(row.FrameId, out var index)) return null;
|
||||
if (index != row.Index) moves[row.Index] = index;
|
||||
}
|
||||
return ProjectSettings.MoveFramePositions(project, moves) is { } moved ? (project, moved) : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Mirrors moved positions into <c>project.json</c> once the index holds them, unless the file was written since
|
||||
/// (it then belongs to that writer, whose next save mirrors the index again). A failed write leaves the index right.
|
||||
/// </summary>
|
||||
public static void WriteProject(SequencePaths sequence, string before, string moved)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(sequence.ProjectPath) && !SameJson(File.ReadAllText(sequence.ProjectPath), before)) return;
|
||||
Directory.CreateDirectory(sequence.MetaDirectory);
|
||||
AtomicFile.WriteAllText(sequence.ProjectPath, moved);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"Project of {sequence.RelativeFolder} keeps its old frame positions on disk", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Same JSON content whatever the layout (an index may store it normalised).</summary>
|
||||
private static bool SameJson(string a, string b)
|
||||
{
|
||||
try
|
||||
{
|
||||
return JsonNode.DeepEquals(JsonNode.Parse(a), JsonNode.Parse(b));
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,9 @@ public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, str
|
||||
/// <summary>A frame and the sequence it belongs to.</summary>
|
||||
public readonly record struct FrameRef(Guid FrameId, Guid SequenceId);
|
||||
|
||||
/// <summary>Where a frame stands in its sequence.</summary>
|
||||
public readonly record struct FramePosition(Guid FrameId, int Index);
|
||||
|
||||
/// <summary>A frame whose content (SHA-256) also exists elsewhere in the archive.</summary>
|
||||
public sealed record DuplicateFrame(string Sha256, Guid FrameId, Guid SequenceId, int Index, string RelativePath);
|
||||
|
||||
@@ -74,6 +77,11 @@ public sealed class FrameRepository
|
||||
_db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 AND (flags & $2) = 0 ORDER BY sequence_id, idx",
|
||||
[sha256.ToLowerInvariant(), (int)(FrameFlags.Missing | FrameFlags.Corrupt)], Map, cancellationToken);
|
||||
|
||||
/// <summary>Id and index of every frame of a sequence, in index order.</summary>
|
||||
public Task<IReadOnlyList<FramePosition>> ListPositionsAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync("SELECT id, idx FROM frames WHERE sequence_id = $1 ORDER BY idx", [sequenceId], r => new FramePosition(r.GetGuid(0), r.GetInt32(1)),
|
||||
cancellationToken);
|
||||
|
||||
/// <summary>Writes analysis results for many frames in one transaction; returns the number of frames updated.</summary>
|
||||
public Task<int> UpdateLightStatsAsync(IReadOnlyList<FrameLightStats> stats, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
|
||||
@@ -121,6 +121,52 @@ public sealed class ProjectSettingsTests
|
||||
public void ProjectMustBeAnObject() =>
|
||||
Assert.Throws<System.Text.Json.JsonException>(() => ProjectSettings.FromJson("[1, 2]"));
|
||||
|
||||
[Test]
|
||||
public void FramePositionsFollowInsertedFramesAndOtherMembersStay()
|
||||
{
|
||||
var project = new ProjectSettings
|
||||
{
|
||||
ExcludedFrames = [3, 7, 40],
|
||||
HeroFrame = 2,
|
||||
Camera = new CameraSettings { Keyframes = [new CameraKey(0, 0.5, 0.5, 1, 0), new CameraKey(7, 0.4, 0.5, 1.5, 0)] },
|
||||
Exposure = new ExposureSettings { DeflickerWindow = new Auto<int>().WithManual(41) },
|
||||
};
|
||||
// Two frames inserted before position 3 and one before position 7; 40 is no frame of the old list and stays.
|
||||
var moves = new Dictionary<int, int> { [0] = 0, [1] = 1, [2] = 4, [3] = 5, [4] = 6, [5] = 7, [6] = 8, [7] = 10 };
|
||||
|
||||
var moved = ProjectSettings.FromJson(Assert.NotNull(ProjectSettings.MoveFramePositions(project.ToJson(), moves)));
|
||||
|
||||
Assert.SequenceEqual([5, 10, 40], moved.ExcludedFrames);
|
||||
Assert.Equal(4, moved.HeroFrame);
|
||||
Assert.SequenceEqual([0, 10], moved.Camera.Keyframes.Select(k => k.Frame));
|
||||
Assert.Equal((0.4, 1.5), (moved.Camera.Keyframes[1].CenterX, moved.Camera.Keyframes[1].Zoom));
|
||||
Assert.Equal(41, moved.Exposure.DeflickerWindow.Manual);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void MovingPositionsEditsTheJsonInPlace()
|
||||
{
|
||||
const string project = """{ "ExcludedFrames": [1, 2], "heroFrame": null, "futureOption": { "keep": [1, 2] }, "camera": { "keyframes": [ { "frame": 1.5 } ] } }""";
|
||||
|
||||
var moved = Assert.NotNull(ProjectSettings.MoveFramePositions(project, new Dictionary<int, int> { [1] = 3 }));
|
||||
|
||||
var json = System.Text.Json.Nodes.JsonNode.Parse(moved)!;
|
||||
Assert.SequenceEqual([3, 2], json["ExcludedFrames"]!.AsArray().Select(n => n!.GetValue<int>()));
|
||||
Assert.Equal("""{"keep":[1,2]}""", json["futureOption"]!.ToJsonString());
|
||||
Assert.Null(json["heroFrame"]);
|
||||
Assert.Equal(1.5, json["camera"]!["keyframes"]![0]!["frame"]!.GetValue<double>());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NothingToMoveGivesNoNewProject()
|
||||
{
|
||||
Assert.Null(ProjectSettings.MoveFramePositions(new ProjectSettings { ExcludedFrames = [1] }.ToJson(), new Dictionary<int, int>()));
|
||||
Assert.Null(ProjectSettings.MoveFramePositions(new ProjectSettings { ExcludedFrames = [1] }.ToJson(), new Dictionary<int, int> { [2] = 3, [1] = 1 }));
|
||||
Assert.Null(ProjectSettings.MoveFramePositions("not json", new Dictionary<int, int> { [1] = 2 }));
|
||||
Assert.Null(ProjectSettings.MoveFramePositions("[1, 2]", new Dictionary<int, int> { [1] = 2 }));
|
||||
Assert.Null(ProjectSettings.MoveFramePositions("""{ "heroFrame": 1, "HeroFrame": 1 }""", new Dictionary<int, int> { [1] = 2 }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void BuiltInPresetIdsAreUnique() =>
|
||||
Assert.Equal(ExportPresets.BuiltIn.Count, ExportPresets.BuiltIn.Select(p => p.Id).Distinct().Count());
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
using System.Text.Json.Nodes;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Repositories;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
|
||||
@@ -215,4 +219,61 @@ public sealed class ArchiveIngestTests(TestContext context)
|
||||
// The index can be rebuilt from the sidecar it now has.
|
||||
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FramesInsertedBeforeOthersTakeTheProjectPositionsAlong()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var results = new List<IngestResult>();
|
||||
for (var i = 0; i < 5; i++) results.Add(await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1 + i), 10 + i)), fx.Ct));
|
||||
var sequenceId = results[0].SequenceId;
|
||||
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(sequenceId))!.Folder);
|
||||
var project = new ProjectSettings
|
||||
{
|
||||
ExcludedFrames = [3],
|
||||
HeroFrame = 1,
|
||||
Camera = new CameraSettings { Keyframes = [new CameraKey(0, 0.5, 0.5, 1, 0), new CameraKey(4, 0.4, 0.5, 1.5, 0)] },
|
||||
}.ToJson();
|
||||
AtomicFile.WriteAllText(folder.ProjectPath, project);
|
||||
var edited = await fx.Db.Sequences.EditAsync(sequenceId, new SequenceEdit { SettingsJson = project, HeroFrameId = results[1].FrameId }, null, fx.Ct);
|
||||
var version = edited.Sequence!.SettingsVersion;
|
||||
|
||||
// Two photos captured before every other one, and one between the third and the fourth, arrive later.
|
||||
await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 20)), fx.Ct);
|
||||
await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(30), 21)), fx.Ct);
|
||||
await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(3.5), 22)), fx.Ct);
|
||||
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequenceId))!;
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId);
|
||||
var moved = ProjectSettings.FromJson(row.SettingsJson!);
|
||||
Assert.SequenceEqual([results[3].FrameId], moved.ExcludedFrames.Select(i => frames[i].Id));
|
||||
Assert.Equal(results[1].FrameId, frames[moved.HeroFrame!.Value].Id);
|
||||
Assert.SequenceEqual([results[0].FrameId, results[4].FrameId], moved.Camera.Keyframes.Select(k => frames[k.Frame].Id));
|
||||
Assert.Equal(version, row.SettingsVersion);
|
||||
Assert.Equal(results[1].FrameId, row.HeroFrameId);
|
||||
Assert.True(JsonNode.DeepEquals(JsonNode.Parse(row.SettingsJson!), JsonNode.Parse(File.ReadAllText(folder.ProjectPath))), "project.json mirrors the index");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task PhotosAFolderScanFindsMoveTheProjectPositionsButUnknownMembersStay()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct);
|
||||
await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(2), 2)), fx.Ct);
|
||||
var row = (await fx.Db.Sequences.GetAsync(late.SequenceId))!;
|
||||
var folder = fx.Paths.Sequence(row.Folder);
|
||||
// A project with a member from a newer Titano; only the index copy holds it (the API edited it).
|
||||
const string project = "{ \"excludedFrames\": [1], \"heroFrame\": 0, \"futureOption\": { \"keep\": true } }";
|
||||
await fx.Db.Sequences.EditAsync(late.SequenceId, new SequenceEdit { SettingsJson = project }, null, fx.Ct);
|
||||
|
||||
File.WriteAllBytes(folder.File("originals/early.JPG"), Photos.Jpeg(Start, 3));
|
||||
await fx.Service.Indexer.ReconcileAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(late.SequenceId))!, fx.Ct);
|
||||
|
||||
var settings = (await fx.Db.Sequences.GetAsync(late.SequenceId))!.SettingsJson!;
|
||||
var moved = ProjectSettings.FromJson(settings);
|
||||
Assert.SequenceEqual([2], moved.ExcludedFrames);
|
||||
Assert.Equal(1, moved.HeroFrame);
|
||||
Assert.Contains("futureOption", settings);
|
||||
Assert.True(JsonNode.DeepEquals(JsonNode.Parse(settings), JsonNode.Parse(File.ReadAllText(folder.ProjectPath))), "project.json mirrors the index");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user