fix(archive): settle conversions by content and keep their aliases through trash, undo, sync and ingest

Owner review 2, phase 4b, fix round 1 (trash, undo, recovery, sync and ingest).

- Replace journal: records a phase (moving, written, undoing) and the name each swapped
  frame is filed under. Recovery decides "sidecar written" by what the sidecar lists, not
  by the frames version any writer moves; a failed rollback stays undoing and its failure
  now stops the writer (as a photo deletion's recovery does). The ingest settles restores
  and replacements first, so an upload can no longer complete a swap that never happened.
- Emptying a Replaced entry re-verifies each replacement where it is now: in the sequence
  through later conversions (a second conversion no longer strands the first originals),
  or, when the conversion was deleted, its copy in the trash. The dry run marks such
  entries "conversion deleted" (fingerprinted, shown on the desktop and the panel), and
  the confirmed emptying may then erase them instead of keeping them forever.
- Undo of a conversion takes exactly the entries its job made (the job id is recorded on
  the entry), never entries found by time window.
- Project positions translated by photo use the sidecar's photo of every frame still
  there, so positions on a converted frame stay put when another frame went meanwhile.
- Sync aliases are equivalence classes: an original converted on both sides is one photo;
  link-by-content, proxy counting and proxy upload follow aliases; a download into a
  frame missing here swaps the other encoding into that frame instead of adding a frame.
- The ingest answers a converted photo's original (a card or folder imported again, a
  download) as a duplicate of its conversion and keeps the alias tombstone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 3341a2f845753fc4c5bcba51a3b645f126c95eaf)
This commit is contained in:
Alby96 committed 2026-10-27 22:13:36 +01:00
1 parent 67a6d51cbe
commit d0bbd77c92
31 files changed
+1326 -211

No files matched your search

+6 -4
View File
@@ -192,10 +192,11 @@ public static class SyncRules
/// and not verified) stands for a wanted frame of its size that no hash matched (by size, not name: the server names
/// its copy by its own template); the upload hashes it before making the proxy. A wanted frame the server has not
/// hashed is the local original of its size and capture time (<paramref name="captured"/>, by server frame id), as
/// <see cref="ProxyOriginal"/> finds it for the upload.
/// <see cref="ProxyOriginal"/> finds it for the upload. A wanted frame whose photo this PC holds in another encoding (a
/// conversion on either side, <paramref name="tombstones"/>) is made from that one, as the upload makes it.
/// </summary>
public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection<Frame> local, ImageDecoder decoder,
IReadOnlyDictionary<Guid, DateTime?>? captured = null)
IReadOnlyDictionary<Guid, DateTime?>? captured = null, SyncTombstones? tombstones = null)
{
ArgumentNullException.ThrowIfNull(manifest);
IReadOnlyList<Frame> frames = [.. local];
@@ -207,12 +208,13 @@ public static class SyncRules
{
if (ProxyOriginal(frame, captured?.GetValueOrDefault(frame.Id), frames, photos) is { } original && taken.Add(original.Id) && Readable(original)) count++;
}
var hashed = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal);
var aliases = tombstones ?? SyncTombstones.None;
var hashed = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => aliases.Canonical(f.Sha256)!, StringComparer.Ordinal);
var unhashed = frames.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f) && !taken.Contains(f.Id)).GroupBy(f => f.Size)
.ToDictionary(g => g.Key, g => g.Count());
foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length > 0))
{
var sha = frame.Sha256.ToLowerInvariant();
var sha = aliases.Canonical(frame.Sha256)!;
if (hashed.Contains(sha))
{
if (hashed[sha].Any(Readable)) count++;
+11 -5
View File
@@ -546,7 +546,7 @@ public sealed class SyncService
var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))],
[.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))], tombstones);
// Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3).
var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured) : 0;
var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured, tombstones) : 0;
// Those change without the server's frame counters moving: while some may be made here, the server's next
// change of the sequence brings a new comparison.
var watch = policy.CanEdit && (proxies > 0 || SyncRules.MakesProxiesFor(present, Options.Archive.Decoder));
@@ -714,13 +714,19 @@ public sealed class SyncService
}
}
/// <summary>
/// Whether at least half of the smaller side's originals are the other side's by content: a photo either side replaced by
/// a converted file counts as its replacement (owner review 2, R7), as the comparison of linked pairs counts it, so a copy
/// converted on one side before the two were linked is still found.
/// </summary>
private async Task<bool> SameOriginalsAsync(ISyncRemote remote, Sequence local, SequenceDto dto, CancellationToken cancellationToken)
{
var hashes = (await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false))
.Select(f => f.Sha256?.ToLowerInvariant()).OfType<string>().ToHashSet(StringComparer.Ordinal);
if (hashes.Count == 0) return false;
var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false);
if (!frames.Any(f => !string.IsNullOrEmpty(f.Sha256))) return false;
var manifest = await remote.GetManifestAsync(dto.Id, cancellationToken).ConfigureAwait(false);
var shared = manifest.Frames.Select(f => f.Sha256.ToLowerInvariant()).Distinct(StringComparer.Ordinal).Count(hashes.Contains);
var tombstones = new SyncTombstones(await _db.FrameRemovals.ListBySequenceAsync(local.Id, cancellationToken).ConfigureAwait(false), manifest.Removed);
var hashes = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).Select(f => tombstones.Canonical(f.Sha256)!).ToHashSet(StringComparer.Ordinal);
var shared = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => tombstones.Canonical(f.Sha256)!).Distinct(StringComparer.Ordinal).Count(hashes.Contains);
return shared > 0 && shared * 2 >= Math.Min(hashes.Count, manifest.Frames.Count);
}
+64 -23
View File
@@ -8,33 +8,42 @@ namespace Chronos.Data.Sync;
/// The photos each copy of a linked pair removed (owner review 2, R6; BRIEF §14.5): a photo one side deleted is not
/// offered to it again (no upload of what the server deleted, no download of what this PC deleted; local files are never
/// deleted by sync), and a photo one side replaced by a converted file is the same photo as its replacement on either side
/// (an alias: both are compared, matched and translated as the replacement). No I/O.
/// (R7). Replacements are links between encodings of one photo, from either side and in either direction (a conversion, a
/// conversion of a conversion, its undo): every photo they link is compared, matched and translated as one, however many
/// encodings each side made of it (each side converting its own copy gives two, which are still one photo). No I/O; safe
/// to read from several threads once made.
/// </summary>
public sealed class SyncTombstones
{
/// <summary>Links a replacement chain may have at most (a conversion of a conversion…); longer ones stop there.</summary>
private const int MaxAliasSteps = 8;
private readonly HashSet<string> _deletedLocally;
private readonly HashSet<string> _deletedRemotely;
private readonly Dictionary<string, string> _aliases;
/// <summary>Each SHA-256 a replacement links, and the one its photo is compared as (the ordinal-smallest of them).</summary>
private readonly Dictionary<string, string> _canonical;
public SyncTombstones(IEnumerable<FrameRemoval> local, IEnumerable<RemovedPhotoDto>? remote)
{
ArgumentNullException.ThrowIfNull(local);
_deletedLocally = new HashSet<string>(StringComparer.Ordinal);
_deletedRemotely = new HashSet<string>(StringComparer.Ordinal);
_aliases = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var removal in local) Add(removal.Sha256, removal.ReplacementSha256, removal.Reason == RemovalReason.Replaced, _deletedLocally);
var deletedLocally = new List<string>();
var deletedRemotely = new List<string>();
var links = new List<(string, string)>();
foreach (var removal in local) Add(removal.Sha256, removal.ReplacementSha256, removal.Reason == RemovalReason.Replaced, deletedLocally, links);
foreach (var removal in remote ?? [])
Add(removal.Sha256, removal.Replacement, string.Equals(removal.Reason, Wire(RemovalReason.Replaced), StringComparison.OrdinalIgnoreCase), _deletedRemotely);
{
Add(removal.Sha256, removal.Replacement, string.Equals(removal.Reason, Wire(RemovalReason.Replaced), StringComparison.OrdinalIgnoreCase),
deletedRemotely, links);
}
_canonical = Classes(links);
// A deletion covers every encoding of the photo deleted: kept by the photo's canonical SHA-256.
_deletedLocally = deletedLocally.Select(sha => Canonical(sha)!).ToHashSet(StringComparer.Ordinal);
_deletedRemotely = deletedRemotely.Select(sha => Canonical(sha)!).ToHashSet(StringComparer.Ordinal);
}
/// <summary>No photo removed on either side.</summary>
public static SyncTombstones None { get; } = new([], null);
/// <summary>Whether either side removed or replaced anything.</summary>
public bool IsEmpty => _deletedLocally.Count == 0 && _deletedRemotely.Count == 0 && _aliases.Count == 0;
public bool IsEmpty => _deletedLocally.Count == 0 && _deletedRemotely.Count == 0 && _canonical.Count == 0;
/// <summary>The wire name of a removal reason (<c>deleted</c>, <c>replaced</c>).</summary>
public static string Wire(RemovalReason reason) => reason.ToString().ToLowerInvariant();
@@ -44,38 +53,70 @@ public sealed class SyncTombstones
[.. removals.Select(r => new RemovedPhotoDto(r.Sha256, r.ReplacementSha256, Wire(r.Reason), r.RemovedAt))];
/// <summary>
/// The SHA-256 a photo is compared as: the end of its replacement chain on either side, lower-case; null or empty
/// (not hashed yet) stays as it is.
/// The SHA-256 a photo is compared as, lower-case: of the encodings replacements link it to on either side, the
/// ordinal-smallest, so the same photo compares the same whichever side converted it and in whatever order the links
/// came; its own SHA-256 when nothing links it. Null or empty (not hashed yet) stays as it is.
/// </summary>
public string? Canonical(string? sha256)
{
if (string.IsNullOrEmpty(sha256)) return sha256;
var current = sha256.ToLowerInvariant();
for (var step = 0; step < MaxAliasSteps && _aliases.TryGetValue(current, out var next); step++) current = next;
return current;
var sha = sha256.ToLowerInvariant();
return _canonical.TryGetValue(sha, out var canonical) ? canonical : sha;
}
/// <summary>The photo as it is compared (<see cref="Canonical(string?)"/>).</summary>
public FramePhoto Canonical(FramePhoto photo)
{
ArgumentNullException.ThrowIfNull(photo);
return photo.IsHashed && _aliases.Count > 0 ? photo with { Sha256 = Canonical(photo.Sha256) } : photo;
return photo.IsHashed && _canonical.Count > 0 ? photo with { Sha256 = Canonical(photo.Sha256) } : photo;
}
/// <summary>This PC deleted the photo (or what replaced it): the server's copy is not downloaded.</summary>
/// <summary>This PC deleted the photo (in any of its encodings): the server's copy is not downloaded.</summary>
public bool DeletedLocally(string? sha256) => Deleted(sha256, _deletedLocally);
/// <summary>The server deleted the photo (or what replaced it): this PC's copy is not uploaded.</summary>
/// <summary>The server deleted the photo (in any of its encodings): this PC's copy is not uploaded.</summary>
public bool DeletedRemotely(string? sha256) => Deleted(sha256, _deletedRemotely);
private bool Deleted(string? sha256, HashSet<string> deleted) =>
!string.IsNullOrEmpty(sha256) && deleted.Count > 0 && (deleted.Contains(sha256.ToLowerInvariant()) || deleted.Contains(Canonical(sha256)!));
private bool Deleted(string? sha256, HashSet<string> deleted) => !string.IsNullOrEmpty(sha256) && deleted.Count > 0 && deleted.Contains(Canonical(sha256)!);
private void Add(string sha256, string? replacement, bool replaced, HashSet<string> deleted)
private static void Add(string sha256, string? replacement, bool replaced, List<string> deleted, List<(string, string)> links)
{
if (string.IsNullOrEmpty(sha256)) return;
var sha = sha256.ToLowerInvariant();
if (replaced && !string.IsNullOrEmpty(replacement)) _aliases[sha] = replacement.ToLowerInvariant();
if (replaced && !string.IsNullOrEmpty(replacement)) links.Add((sha, replacement.ToLowerInvariant()));
else deleted.Add(sha);
}
/// <summary>
/// The photos the links join (union-find), each SHA-256 linked to the ordinal-smallest of its photo: a root is always the
/// smallest of what it joins, so the result does not depend on the order of the links.
/// </summary>
private static Dictionary<string, string> Classes(List<(string A, string B)> links)
{
var parent = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var (a, b) in links)
{
if (a == b) continue;
parent.TryAdd(a, a);
parent.TryAdd(b, b);
var (x, y) = (Find(a), Find(b));
if (x == y) continue;
if (string.CompareOrdinal(x, y) < 0) parent[y] = x;
else parent[x] = y;
}
return parent.Keys.ToList().ToDictionary(k => k, Find, StringComparer.Ordinal);
string Find(string sha)
{
var root = sha;
while (parent[root] != root) root = parent[root];
while (parent[sha] != root)
{
var next = parent[sha];
parent[sha] = root;
sha = next;
}
return root;
}
}
}
+87 -22
View File
@@ -26,7 +26,8 @@ namespace Chronos.Data.Sync;
/// which names them and writes the sidecar; the first original creates the local
/// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the
/// server's copies of them are recognised), and originals flagged missing here are restored to their recorded place in
/// the folder the sequence is in (never one it left).
/// the folder the sequence is in (never one it left; one the server holds in another encoding, a conversion on either
/// side, takes the frame's place under its id instead).
/// Proxy downloads fill the proxy cache. Renders travel on request: this PC's to the server (hash-verified chunked
/// upload) and the server's into the local copy's exports. A resumed job skips what is already on the other side, so
/// pausing or a crash loses at most the files in flight.
@@ -111,10 +112,11 @@ internal sealed class SyncTransfers(SyncService sync)
/// <summary>
/// The proxies the server asks for (<see cref="ManifestFrame.WantsProxy"/>) that this PC's decoders can make from its
/// originals (by their archived SHA-256, or the one <paramref name="hashes"/> holds for an original the index has none
/// for; by size and capture time for a frame the server has not hashed, <see cref="SyncRules.ProxyOriginal"/>), made as
/// the server's proxy policy says with the server's capture data, and sent. One the server refuses is
/// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a
/// server busy or out of reach fails the job like an original would.
/// for, compared as the photo either side's conversions make it, <see cref="SyncTombstones"/>: a frame this PC converted
/// makes the proxy of the server's original; by size and capture time for a frame the server has not hashed,
/// <see cref="SyncRules.ProxyOriginal"/>), made as the server's proxy policy says with the server's capture data, and
/// sent. One the server refuses is counted in the job log; a server that makes no proxies, or an account that may not
/// send them, ends the pass; a server busy or out of reach fails the job like an original would.
/// </summary>
private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList<Frame> frames,
IReadOnlyDictionary<Guid, string> hashes)
@@ -123,10 +125,11 @@ internal sealed class SyncTransfers(SyncService sync)
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
var wanted = manifest.Frames.Where(f => f.WantsProxy).ToList();
if (manifest.Proxies is not { } spec || wanted.Count == 0) return;
var tombstones = new SyncTombstones(await sync.Database.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed);
var originals = new Dictionary<string, Frame>(StringComparer.Ordinal);
foreach (var frame in frames)
{
if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame);
if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(tombstones.Canonical(sha)!, frame);
}
var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id);
var photos = SyncRules.Photos(frames);
@@ -144,7 +147,7 @@ internal sealed class SyncTransfers(SyncService sync)
// A frame the server has not hashed yet is this PC's original of its size and capture time.
var original = frame.Sha256.Length == 0
? SyncRules.ProxyOriginal(frame, record?.CapturedAt, frames, photos)
: originals.GetValueOrDefault(frame.Sha256.ToLowerInvariant());
: originals.GetValueOrDefault(tombstones.Canonical(frame.Sha256)!);
var jpeg = original is not null && record is not null
? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName,
token), token).ConfigureAwait(false)
@@ -211,11 +214,22 @@ internal sealed class SyncTransfers(SyncService sync)
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
var (have, restore, photos) = local is null ? ([], [], new FramePhotos([])) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
var (have, restore, photos, missing) = local is null ? ([], [], new FramePhotos([]), []) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
// A photo this PC deleted is not fetched again; one replaced here by a converted file is here as its replacement (owner review 2, R6).
var tombstones = local is null ? SyncTombstones.None
: new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed);
if (!tombstones.IsEmpty) have = [.. have.Select(h => tombstones.Canonical(h)!)];
// A frame flagged missing here whose photo the server holds in another encoding (a conversion on either side, R7): the
// copy fetched takes that frame's place under its id, rather than coming in as a frame of its own beside it.
// One whose own encoding the server holds too is restored from that one instead, and one whose photo another frame
// here holds intact stays as it is, as a restore leaves it.
var byAlias = new ConcurrentDictionary<string, Frame>(StringComparer.Ordinal);
if (!tombstones.IsEmpty)
{
var served = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
foreach (var frame in missing.Where(f => !served.Contains(f.Sha256!.ToLowerInvariant()) && !have.Contains(tombstones.Canonical(f.Sha256)!)))
byAlias.TryAdd(tombstones.Canonical(frame.Sha256)!, frame);
}
// An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and
// capture time is; each one is fetched on its own and hashed here.
var wanted = manifest.Frames
@@ -247,6 +261,7 @@ internal sealed class SyncTransfers(SyncService sync)
var sequenceId = local?.Id;
var refused = 0;
var restores = new ConcurrentBag<(string Staged, Guid FrameId)>();
var aliases = new ConcurrentBag<AliasCopy>();
await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) =>
{
var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false);
@@ -254,6 +269,11 @@ internal sealed class SyncTransfers(SyncService sync)
{
restores.Add((staged, frameId));
}
else if (byAlias.TryRemove(tombstones.Canonical(sha)!, out var lost))
{
// This encoding takes the place of the one missing here, after the transfers as the restores do.
aliases.Add(new AliasCopy(staged, sha, lost, frame.FileName));
}
else if (frame.Sha256.Length == 0 && (tombstones.DeletedLocally(sha) || have.Contains(tombstones.Canonical(sha)!)))
{
// An original the server has not hashed is known by content only now: one this PC deleted (or holds as
@@ -269,30 +289,33 @@ internal sealed class SyncTransfers(SyncService sync)
await progress.StepAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
// Null when the sequence is not in the folder the index has: the copies stay staged for the next attempt.
var restored = restores.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false);
if (restored is not null) TryDeleteDirectory(staging);
// Restored originals are checked against their records, which clears their missing flag.
if (restored > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false);
var restored = restores.IsEmpty && aliases.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false);
var (swapped, moved) = restored is null || aliases.IsEmpty ? (0, restored is null)
: await SwapAliasesAsync(context, archive, local!, paths, aliases, ct).ConfigureAwait(false);
if (!moved) TryDeleteDirectory(staging);
// Restored originals are checked against their records, which clears their missing flag (and reads the tier of a
// frame that took another encoding).
if (restored + swapped > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false);
if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false);
await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false);
if (sequenceId is { } downloaded && wanted.Count > restores.Count + refused)
if (sequenceId is { } downloaded && wanted.Count > restores.Count + aliases.Count + refused)
{
var label = local!.DisplayTitle;
await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false);
await EnqueueFollowUpAsync(context, ProcessingJobs.LightAnalysis([downloaded], label)).ConfigureAwait(false);
}
if (restored is null) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
if (moved) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
}
/// <summary>
/// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet) and the
/// frame each original flagged missing restores. Frames not hashed yet (an adopted folder never verified) are hashed
/// first through the archive's verify, so the server's copies of them are recognised instead of committed again as
/// duplicates.
/// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet), the frame
/// each original flagged missing restores, and the frames flagged missing whose content no intact frame holds. Frames
/// not hashed yet (an adopted folder never verified) are hashed first through the archive's verify, so the server's
/// copies of them are recognised instead of committed again as duplicates.
/// </summary>
private async Task<(HashSet<string> Have, Dictionary<string, Guid> Restore, FramePhotos Photos)> LocalContentAsync(Archive archive, Sequence local,
CancellationToken ct)
private async Task<(HashSet<string> Have, Dictionary<string, Guid> Restore, FramePhotos Photos, IReadOnlyList<Frame> Missing)> LocalContentAsync(
Archive archive, Sequence local, CancellationToken ct)
{
var frames = await SequenceFrames.ListAsync(sync.Database, local.Id, ct).ConfigureAwait(false);
if (frames.Any(f => f.Sha256 is null && (f.Flags & FrameFlags.Missing) == 0))
@@ -303,13 +326,16 @@ internal sealed class SyncTransfers(SyncService sync)
var have = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0 && f.Sha256 is not null).Select(f => f.Sha256!.ToLowerInvariant())
.ToHashSet(StringComparer.Ordinal);
var restore = new Dictionary<string, Guid>(StringComparer.Ordinal);
var missing = new List<Frame>();
foreach (var frame in frames.Where(f => (f.Flags & FrameFlags.Missing) != 0 && f.Sha256 is not null))
{
var sha = frame.Sha256!.ToLowerInvariant();
if (!have.Contains(sha)) restore.TryAdd(sha, frame.Id);
if (have.Contains(sha)) continue;
restore.TryAdd(sha, frame.Id);
missing.Add(frame);
}
var photos = SyncRules.Photos([.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0)]);
return (have, restore, photos);
return (have, restore, photos, missing);
}
private Task VerifyAsync(Archive archive, Guid sequenceId, CancellationToken ct) =>
@@ -344,6 +370,45 @@ internal sealed class SyncTransfers(SyncService sync)
return restored;
}, ct);
/// <summary>
/// Gives each frame flagged missing here whose photo the server holds only in another encoding (a conversion on either
/// side, owner review 2, R7) the verified copy fetched (<see cref="FrameTrash.ReplaceMissingAsync"/>: under the sequence's
/// lease, in its folder as the index has it then, whose sidecar must be the sequence's, <see cref="SequenceWriter.ReadLocked"/>).
/// Run after the transfers, once <see cref="RestoreAsync"/> found the sequence in that folder. Returns how many took their
/// frame's place; the copy of a frame that changed meanwhile, or whose file came back, is dropped (logged). Moved, with the
/// copies not swapped yet kept in staging for the next attempt, when the sequence left that folder meanwhile.
/// </summary>
private async Task<(int Swapped, bool Moved)> SwapAliasesAsync(JobContext context, Archive archive, Sequence local, ArchivePaths paths,
IEnumerable<AliasCopy> aliases, CancellationToken ct)
{
var swapped = 0;
foreach (var alias in aliases)
{
try
{
if (await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256,
alias.FileName, ct).ConfigureAwait(false))
{
swapped++;
continue;
}
}
catch (Exception ex) when (ex is InvalidDataException or DirectoryNotFoundException or FileNotFoundException)
{
// Another index moved the sequence away, maybe another one into the folder it left: nothing is filed there.
if (await sync.Database.Sequences.GetAsync(local.Id, ct).ConfigureAwait(false) is { } row && SequenceFolders.Holds(paths.Sequence(row.Folder), row.Id))
throw;
return (swapped, true);
}
File.Delete(alias.Staged);
context.Log($"{alias.FileName}: its frame here changed meanwhile, not downloaded");
}
return (swapped, false);
}
/// <summary>A verified copy fetched in another encoding of a frame flagged missing here (<see cref="SwapAliasesAsync"/>).</summary>
private sealed record AliasCopy(string Staged, string Sha256, Frame Lost, string FileName);
/// <summary>Puts a verified copy of an original flagged missing back where its record says; a file that reappeared meanwhile is kept.</summary>
private static void Restore(string staged, string place)
{