feat: owner review 2 phase 4b: mass conversion after import (#13)
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s

Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #13.
This commit is contained in:
Alby96andClaude Opus 5.5 authored and Alby96 committed 2026-11-03 10:41:50 +01:00
1 parent 7e4a3b481d
commit efa87f68ce
354 files changed
+24530 -843

No files matched your search

+122 -31
View File
@@ -26,7 +26,8 @@ namespace Chronos.Data.Sync;
/// which names them and writes the sidecar; the first original creates the local
/// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the
/// server's copies of them are recognised), and originals flagged missing here are restored to their recorded place in
/// the folder the sequence is in (never one it left).
/// the folder the sequence is in (never one it left; one the server holds in another encoding, a conversion on either
/// side, takes the frame's place under its id instead).
/// Proxy downloads fill the proxy cache. Renders travel on request: this PC's to the server (hash-verified chunked
/// upload) and the server's into the local copy's exports. A resumed job skips what is already on the other side, so
/// pausing or a crash loses at most the files in flight.
@@ -111,10 +112,11 @@ internal sealed class SyncTransfers(SyncService sync)
/// <summary>
/// The proxies the server asks for (<see cref="ManifestFrame.WantsProxy"/>) that this PC's decoders can make from its
/// originals (by their archived SHA-256, or the one <paramref name="hashes"/> holds for an original the index has none
/// for; by size and capture time for a frame the server has not hashed, <see cref="SyncRules.ProxyOriginal"/>), made as
/// the server's proxy policy says with the server's capture data, and sent. One the server refuses is
/// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a
/// server busy or out of reach fails the job like an original would.
/// for, compared as the photo either side's conversions make it, <see cref="SyncTombstones"/>: a frame this PC converted
/// makes the proxy of the server's original; by size and capture time for a frame the server has not hashed,
/// <see cref="SyncRules.ProxyOriginal"/>), made as the server's proxy policy says with the server's capture data, and
/// sent. One the server refuses is counted in the job log; a server that makes no proxies, or an account that may not
/// send them, ends the pass; a server busy or out of reach fails the job like an original would.
/// </summary>
private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList<Frame> frames,
IReadOnlyDictionary<Guid, string> hashes)
@@ -123,10 +125,11 @@ internal sealed class SyncTransfers(SyncService sync)
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
var wanted = manifest.Frames.Where(f => f.WantsProxy).ToList();
if (manifest.Proxies is not { } spec || wanted.Count == 0) return;
var tombstones = new SyncTombstones(await sync.Database.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed);
var originals = new Dictionary<string, Frame>(StringComparer.Ordinal);
foreach (var frame in frames)
{
if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame);
if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(tombstones.Canonical(sha)!, frame);
}
var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id);
var photos = SyncRules.Photos(frames);
@@ -144,7 +147,7 @@ internal sealed class SyncTransfers(SyncService sync)
// A frame the server has not hashed yet is this PC's original of its size and capture time.
var original = frame.Sha256.Length == 0
? SyncRules.ProxyOriginal(frame, record?.CapturedAt, frames, photos)
: originals.GetValueOrDefault(frame.Sha256.ToLowerInvariant());
: originals.GetValueOrDefault(tombstones.Canonical(frame.Sha256)!);
var jpeg = original is not null && record is not null
? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName,
token), token).ConfigureAwait(false)
@@ -211,11 +214,22 @@ internal sealed class SyncTransfers(SyncService sync)
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
var (have, restore, photos) = local is null ? ([], [], new FramePhotos([])) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
var (have, restore, photos, missing) = local is null ? ([], [], new FramePhotos([]), []) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
// A photo this PC deleted is not fetched again; one replaced here by a converted file is here as its replacement (owner review 2, R6).
var tombstones = local is null ? SyncTombstones.None
: new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed);
if (!tombstones.IsEmpty) have = [.. have.Select(h => tombstones.Canonical(h)!)];
// A frame flagged missing here whose photo the server holds in another encoding (a conversion on either side, R7): the
// copy fetched takes that frame's place under its id, rather than coming in as a frame of its own beside it.
// One whose own encoding the server holds too is restored from that one instead, and one whose photo another frame
// here holds intact stays as it is, as a restore leaves it.
var byAlias = new ConcurrentDictionary<string, Frame>(StringComparer.Ordinal);
if (!tombstones.IsEmpty)
{
var served = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
foreach (var frame in missing.Where(f => !served.Contains(f.Sha256!.ToLowerInvariant()) && !have.Contains(tombstones.Canonical(f.Sha256)!)))
byAlias.TryAdd(tombstones.Canonical(frame.Sha256)!, frame);
}
// An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and
// capture time is; each one is fetched on its own and hashed here.
var wanted = manifest.Frames
@@ -247,6 +261,7 @@ internal sealed class SyncTransfers(SyncService sync)
var sequenceId = local?.Id;
var refused = 0;
var restores = new ConcurrentBag<(string Staged, Guid FrameId)>();
var aliases = new ConcurrentBag<AliasCopy>();
await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) =>
{
var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false);
@@ -254,13 +269,18 @@ internal sealed class SyncTransfers(SyncService sync)
{
restores.Add((staged, frameId));
}
else if (byAlias.TryRemove(tombstones.Canonical(sha)!, out var lost))
{
// This encoding takes the place of the one missing here, after the transfers as the restores do.
aliases.Add(new AliasCopy(staged, sha, lost, frame.FileName));
}
else if (frame.Sha256.Length == 0 && (tombstones.DeletedLocally(sha) || have.Contains(tombstones.Canonical(sha)!)))
{
// An original the server has not hashed is known by content only now: one this PC deleted (or holds as
// the converted file that replaced it) is not filed again, which would also drop its tombstone.
File.Delete(staged);
Interlocked.Increment(ref refused);
if (tombstones.DeletedLocally(sha)) context.Log($"{frame.FileName}: deleted on this PC, not downloaded");
if (tombstones.DeletedLocally(sha)) context.Log(Loc.L("Archive.Log.SyncDeletedHere", frame.FileName));
}
else
{
@@ -269,30 +289,35 @@ internal sealed class SyncTransfers(SyncService sync)
await progress.StepAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
// Null when the sequence is not in the folder the index has: the copies stay staged for the next attempt.
var restored = restores.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false);
if (restored is not null) TryDeleteDirectory(staging);
// Restored originals are checked against their records, which clears their missing flag.
if (restored > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false);
var restored = restores.IsEmpty && aliases.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false);
var (swapped, moved) = restored is null || aliases.IsEmpty ? (0, restored is null)
: await SwapAliasesAsync(context, archive, local!, paths, aliases, ct).ConfigureAwait(false);
if (!moved) TryDeleteDirectory(staging);
// Restored originals are checked against their records, which clears their missing flag (and reads the tier of a
// frame that took another encoding).
if (restored + swapped > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false);
if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false);
await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false);
if (sequenceId is { } downloaded && wanted.Count > restores.Count + refused)
// Originals filed into the sequence (not restores, nor frames swapped to another encoding, whose passes are queued as
// they swap) need their thumbnails and light analysis.
if (sequenceId is { } downloaded && wanted.Count > restores.Count + aliases.Count + refused)
{
var label = local!.DisplayTitle;
await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false);
await EnqueueFollowUpAsync(context, ProcessingJobs.LightAnalysis([downloaded], label)).ConfigureAwait(false);
await QueuePassesAsync(context, downloaded, [ArchiveJobs.Thumbnails([downloaded], label), ProcessingJobs.LightAnalysis([downloaded], label)])
.ConfigureAwait(false);
}
if (restored is null) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
if (moved) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
}
/// <summary>
/// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet) and the
/// frame each original flagged missing restores. Frames not hashed yet (an adopted folder never verified) are hashed
/// first through the archive's verify, so the server's copies of them are recognised instead of committed again as
/// duplicates.
/// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet), the frame
/// each original flagged missing restores, and the frames flagged missing whose content no intact frame holds. Frames
/// not hashed yet (an adopted folder never verified) are hashed first through the archive's verify, so the server's
/// copies of them are recognised instead of committed again as duplicates.
/// </summary>
private async Task<(HashSet<string> Have, Dictionary<string, Guid> Restore, FramePhotos Photos)> LocalContentAsync(Archive archive, Sequence local,
CancellationToken ct)
private async Task<(HashSet<string> Have, Dictionary<string, Guid> Restore, FramePhotos Photos, IReadOnlyList<Frame> Missing)> LocalContentAsync(
Archive archive, Sequence local, CancellationToken ct)
{
var frames = await SequenceFrames.ListAsync(sync.Database, local.Id, ct).ConfigureAwait(false);
if (frames.Any(f => f.Sha256 is null && (f.Flags & FrameFlags.Missing) == 0))
@@ -303,13 +328,16 @@ internal sealed class SyncTransfers(SyncService sync)
var have = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0 && f.Sha256 is not null).Select(f => f.Sha256!.ToLowerInvariant())
.ToHashSet(StringComparer.Ordinal);
var restore = new Dictionary<string, Guid>(StringComparer.Ordinal);
var missing = new List<Frame>();
foreach (var frame in frames.Where(f => (f.Flags & FrameFlags.Missing) != 0 && f.Sha256 is not null))
{
var sha = frame.Sha256!.ToLowerInvariant();
if (!have.Contains(sha)) restore.TryAdd(sha, frame.Id);
if (have.Contains(sha)) continue;
restore.TryAdd(sha, frame.Id);
missing.Add(frame);
}
var photos = SyncRules.Photos([.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0)]);
return (have, restore, photos);
return (have, restore, photos, missing);
}
private Task VerifyAsync(Archive archive, Guid sequenceId, CancellationToken ct) =>
@@ -344,6 +372,69 @@ internal sealed class SyncTransfers(SyncService sync)
return restored;
}, ct);
/// <summary>
/// Gives each frame flagged missing here whose photo the server holds only in another encoding (a conversion on either
/// side, owner review 2, R7) the verified copy fetched (<see cref="FrameTrash.ReplaceMissingAsync"/>: under the sequence's
/// lease, in its folder as the index has it then, whose sidecar must be the sequence's, <see cref="SequenceWriter.ReadLocked"/>).
/// Run after the transfers, once <see cref="RestoreAsync"/> found the sequence in that folder. A swap clears what covered
/// the sequence, as any replacement does, so the passes it had (<see cref="FrameTrash.FollowUps"/>: the light analysis, a
/// background deep analysis, the thumbnails and proxies) are queued after each one, whatever stops the download later.
/// Returns how many took their frame's place; the copy of a frame that changed meanwhile, or whose file came back, is
/// dropped (logged). Moved, with the copies not swapped yet kept in staging for the next attempt, when the sequence left
/// that folder meanwhile.
/// </summary>
private async Task<(int Swapped, bool Moved)> SwapAliasesAsync(JobContext context, Archive archive, Sequence local, ArchivePaths paths,
IEnumerable<AliasCopy> aliases, CancellationToken ct)
{
var swapped = 0;
// What the sequence had before the first swap cleared it: the passes that make it again.
var had = SequenceStatus.None;
foreach (var alias in aliases)
{
ReplaceResult? result;
try
{
result = await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256,
alias.FileName, ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is InvalidDataException or DirectoryNotFoundException or FileNotFoundException)
{
// Another index moved the sequence away, maybe another one into the folder it left: nothing is filed there.
if (await sync.Database.Sequences.GetAsync(local.Id, ct).ConfigureAwait(false) is { } row && SequenceFolders.Holds(paths.Sequence(row.Folder), row.Id))
throw;
return (swapped, true);
}
if (result is null)
{
File.Delete(alias.Staged);
context.Log(Loc.L("Archive.Log.SyncFrameChanged", alias.FileName));
continue;
}
swapped++;
had |= result.Had;
await QueuePassesAsync(context, local.Id, sync.Archives.FrameTrash.FollowUps(archive, result.Sequence, had)).ConfigureAwait(false);
}
return (swapped, false);
}
/// <summary>
/// Queues <paramref name="requests"/>, passes over the sequence <paramref name="sequenceId"/>, as follow-ups of this job,
/// whatever stops it meanwhile, as a restore from the trash queues its own: the files they cover have changed already, and
/// the next download, which finds nothing missing, would not queue them again. One already queued for the sequence (not
/// started yet, so it covers what changed) is left out.
/// </summary>
private async Task QueuePassesAsync(JobContext context, Guid sequenceId, IEnumerable<ArchiveJobRequest> requests)
{
foreach (var request in requests)
{
if (await sync.Database.Jobs.IsQueuedAsync(request.Kind, sequenceId, CancellationToken.None).ConfigureAwait(false)) continue;
await EnqueueFollowUpAsync(context, request, CancellationToken.None).ConfigureAwait(false);
}
}
/// <summary>A verified copy fetched in another encoding of a frame flagged missing here (<see cref="SwapAliasesAsync"/>).</summary>
private sealed record AliasCopy(string Staged, string Sha256, Frame Lost, string FileName);
/// <summary>Puts a verified copy of an original flagged missing back where its record says; a file that reappeared meanwhile is kept.</summary>
private static void Restore(string staged, string place)
{
@@ -513,7 +604,7 @@ internal sealed class SyncTransfers(SyncService sync)
var cached = files.Count(f => f is not null);
var standIns = Enumerable.Range(0, frames.Count).Count(i => kinds[i] != ProxyCache.Proxy && !NeverProxied(frames[i]));
await sync.SetProxiesAsync(remote.Server, dto, cached, standIns, manifest.FramesVersion, ct).ConfigureAwait(false);
if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server");
if (cached < frames.Count) context.Log(Loc.Plural("Archive.Log.SyncNoPreview", frames.Count - cached));
}
/// <summary>
@@ -563,7 +654,7 @@ internal sealed class SyncTransfers(SyncService sync)
{
if (!File.Exists(path))
{
context.Log($"{frame.FileName}: missing, not uploaded");
context.Log(Loc.L("Archive.Log.SyncMissing", frame.FileName));
return null;
}
var sha = await ChronosClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false);
@@ -573,13 +664,13 @@ internal sealed class SyncTransfers(SyncService sync)
hashes[frame.Id] = sha;
if (tombstones.DeletedRemotely(sha))
{
context.Log($"{frame.FileName}: deleted on the server, not uploaded");
context.Log(Loc.L("Archive.Log.SyncDeletedThere", frame.FileName));
return notSent;
}
}
if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase))
{
context.Log($"{frame.FileName}: changed since it was archived, not uploaded");
context.Log(Loc.L("Archive.Log.SyncChanged", frame.FileName));
return null;
}
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
@@ -717,8 +808,8 @@ internal sealed class SyncTransfers(SyncService sync)
};
/// <summary>Queues a follow-up on this job's engine for the same user; skipped when the engine has no handler for it.</summary>
private static Task<Job?> EnqueueFollowUpAsync(JobContext context, ArchiveJobRequest request) =>
context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority);
private static Task<Job?> EnqueueFollowUpAsync(JobContext context, ArchiveJobRequest request, CancellationToken cancellationToken) =>
context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, cancellationToken);
private static void EnsureSpace(string root, long bytes)
{