feat: owner review 2 phase 4b: mass conversion after import (#13)
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s

Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #13.
This commit is contained in:
Alby96andClaude Opus 5.5 authored and Alby96 committed 2026-11-03 10:41:50 +01:00
1 parent 7e4a3b481d
commit efa87f68ce
354 files changed
+24530 -843

No files matched your search

@@ -5,6 +5,7 @@ using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Core.Render;
using Chronos.Data.Archiving;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Archiving;
using Chronos.Protocol;
@@ -180,7 +181,7 @@ public sealed class SyncTransferTests(TestContext context)
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted");
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone");
Assert.Contains("deleted on this PC, not downloaded", done.LogTail);
Assert.Contains(Loc.L("Archive.Log.SyncDeletedHere", remote.Frames.Single(fr => fr.Sha256 == Sha(photos[0])).FileName), done.LogTail);
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
}
@@ -207,7 +208,7 @@ public sealed class SyncTransferTests(TestContext context)
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(3, remote.Frames.Count);
Assert.Contains("deleted on the server, not uploaded", done.LogTail);
Assert.Contains(Loc.L("Archive.Log.SyncDeletedThere", "DSC_0004.JPG"), done.LogTail);
// Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good.
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify");
var entry = await f.EntryAsync(local.DisplayTitle);
@@ -793,6 +794,228 @@ public sealed class SyncTransferTests(TestContext context)
Assert.Empty(f.Remote.Sequences);
}
[Test]
public async Task APhotoEachSideConvertedIsNeitherSentNorFetchedAgain()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
// Each side converted its own copy of the first photo, to other bytes (another decoder, another quality).
var (mine, theirs) = (f.NewPhotos(1)[0], f.NewPhotos(1)[0]);
await f.ReplaceLocalAsync(local.Id, 0, mine);
var remote = f.Remote.Add("Comet", [theirs, photos[1]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[0]), Sha(theirs), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
var engine = await f.StartEngineAsync();
await f.Archive.WaitForAsync((await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id, JobState.Done);
await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null)).Id,
JobState.Done);
Assert.SequenceEqual([Sha(theirs), Sha(photos[1])], remote.Frames.Select(r => r.Sha256));
Assert.SequenceEqual([Sha(mine), Sha(photos[1])], (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Select(r => r.Sha256!));
}
[Test]
public async Task CopiesConvertedOnOneSideBeforeTheyWereLinkedAreLinkedByContent()
{
await using var f = await SyncFixture.CreateAsync(context);
// The Uploader sent the card to the server, which converted it before this PC compared the two.
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
var converted = f.NewPhotos(3);
var remote = f.Remote.Add("2026-09-28 Canon", converted);
foreach (var (original, replacement) in photos.Zip(converted))
remote.Removed.Add(new RemovedPhotoDto(Sha(original), Sha(replacement), "replaced", DateTimeOffset.UtcNow));
remote.FirstCapture = local.FirstCapture;
remote.CameraModel = local.CameraModel;
// And the other way round: this PC converted its copy of another card the server holds as it came.
var cards = f.NewPhotos(2);
var mine = await f.LocalAsync("Harbour", cards);
foreach (var (index, replacement) in f.NewPhotos(2).Index()) await f.ReplaceLocalAsync(mine.Id, index, replacement);
mine = (await f.Db.Sequences.GetAsync(mine.Id, f.Ct))!;
var theirs = f.Remote.Add("2026-09-28 Canon (2)", cards);
theirs.FirstCapture = mine.FirstCapture;
theirs.CameraModel = mine.CameraModel;
await f.RefreshAsync();
var comet = await f.EntryAsync("Comet");
Assert.Equal(remote.Id, comet.Link?.RemoteId);
Assert.Equal((0, 0), (comet.Link!.MissingRemote, comet.Link.MissingLocal));
var harbour = await f.EntryAsync("Harbour");
Assert.Equal(theirs.Id, harbour.Link?.RemoteId);
Assert.Equal((0, 0), (harbour.Link!.MissingRemote, harbour.Link.MissingLocal));
}
[Test]
public async Task AFrameMissingHereTakesBackItsPhotoAsTheServerConvertedIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
// The server converted that photo.
var converted = f.NewPhotos(1)[0];
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var behind = await f.EntryAsync("Harbour");
Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The same frame, holding the server's conversion, and no frame beside it.
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256));
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the frame is no longer missing");
Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct)));
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct), "the original is an alias of its conversion here too");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task AFrameThatTookBackItsPhotoInAnotherEncodingIsDrawnAgain()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var engine = await f.StartEngineAsync();
var drawn = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([local.Id], "Harbour"), cancellationToken: f.Ct);
await f.Archive.WaitForAsync(drawn.Id, JobState.Done);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn before");
// The server converted that photo.
var converted = f.NewPhotos(1)[0];
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// Nothing was filed, only a frame swapped, which cleared what covered the sequence: the thumbnails it had are drawn again.
Assert.Equal(lost.Id, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1].Id);
var again = Assert.Contains((await f.Db.Jobs.ListAsync(null, JobKind.Thumbs, 50, null, f.Ct)).Items, j => j.Id != drawn.Id && j.TargetIds.Contains(local.Id));
await f.Archive.WaitForAsync(again.Id, JobState.Done);
Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn again");
}
[Test]
public async Task AFrameTakesBackItsPhotoInAnotherEncodingOnlyWhereTheSequenceIsNow()
{
// The folder the sequence left holds another sequence now, or nothing.
foreach (var taken in new[] { true, false })
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var stranger = taken ? await f.LocalAsync("Stranger", f.NewPhotos(2)) : null;
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
// The server converted that photo.
var converted = f.NewPhotos(1)[0];
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
// Another index moves the folder away, then the other sequence into the folder it left; this index has scanned neither.
var folder = f.Archive.Paths.Sequence(local.Folder).Folder;
var (other, archive) = await f.Archive.OtherWriterAsync();
var moves = stranger is null ? [(local.Id, "moved/{title}")] : new[] { (local.Id, "moved/{title}"), (stranger.Id, "parked/{title}"), (stranger.Id, local.Folder) };
foreach (var (id, template) in moves)
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], template, null, f.Ct), null, f.Ct);
var before = taken ? ArchiveFixture.FilesOf(folder) : null;
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error, taken.ToString());
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched");
else Assert.False(Directory.Exists(folder), "the folder it left is not created again");
Assert.Equal(lost.Sha256, (await f.Db.Frames.GetAsync(lost.Id, f.Ct))!.Sha256);
// Once this index's scan found it there, the converted photo takes the frame's place in the folder the sequence is in now.
await f.Archive.ScanAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256));
Assert.True(frames[1].RelativePath.StartsWith("moved/Harbour/", StringComparison.Ordinal), frames[1].RelativePath);
Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct)));
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "nothing missing");
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched");
else Assert.False(Directory.Exists(folder));
}
}
[Test]
public async Task AConversionMissingHereTakesBackTheOriginalTheServerHolds()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
// This PC converted the second photo, and that file is gone since.
var converted = f.NewPhotos(1)[0];
await f.ReplaceLocalAsync(local.Id, 1, converted);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
await f.RefreshAsync();
Assert.Equal(1, (await f.EntryAsync("Harbour")).Link!.MissingLocal);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(photos[1])), (frames[1].Id, frames[1].Sha256));
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(converted), f.Ct), "the conversion is an alias of the original");
Assert.False(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct));
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task AnUploadSendsTheProxyOfAPhotoThisPcConverted()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server holds the originals and cannot decode them (HEIF, BRIEF §2.3); this PC converted its copy of the first.
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.ReplaceLocalAsync(local.Id, 0, f.NewPhotos(1)[0]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Equal(3, remote.Frames.Count);
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
}
private static async Task<Render> AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes)
{
var render = new Render