fix(sync): download and match originals the server has not hashed

A folder the server adopted and never verified lists no SHA-256. A
download of such a sequence failed ("did not arrive intact") because
every unhashed frame collapsed into one wanted entry and the empty hash
was refused, and desktops never sent the proxies the server asked for
those frames (V-9).

- Each unhashed server original is fetched by frame id into staging,
  accepted on its size (two attempts, resumed with Range) and hashed
  here; the archive ingest files it by that hash like any other.
- Server frames without a hash are matched to this PC's originals by
  size and capture time: they are not downloaded twice, not uploaded
  again, counted right in the comparison (each frame on its own), and
  their wanted proxies are counted and sent from the matching original.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 16:21:35 +00:00
1 parent ea6e1a69e0
commit f4103d42a1
5 files changed
+318 -59

No files matched your search

+91 -13
View File
@@ -123,31 +123,93 @@ public static class SyncRules
}
/// <summary>
/// Originals missing on each side by content (BRIEF §14.5: originals never conflict). Local frames not hashed yet
/// (adopted, not verified) are assumed to match server frames that no hash matched.
/// Originals missing on each side by content (BRIEF §14.5: originals never conflict), from their SHA-256 alone (an
/// empty or null one: not hashed yet). See <see cref="CompareOriginals(IReadOnlyCollection{FramePhoto}, IReadOnlyCollection{FramePhoto})"/>.
/// </summary>
public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection<string?> localHashes, IReadOnlyCollection<string> remoteHashes)
public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection<string?> localHashes, IReadOnlyCollection<string> remoteHashes) =>
CompareOriginals([.. localHashes.Select(h => new FramePhoto(h, 0, null))], [.. remoteHashes.Select(h => new FramePhoto(h, 0, null))]);
/// <summary>
/// Originals missing on each side by content (BRIEF §14.5: originals never conflict): the same SHA-256 where both
/// sides hashed them, else the same size and capture time (<see cref="FramePhoto.Matches"/>; a folder adopted on
/// either side and not verified yet). Local frames not hashed yet that nothing matched are assumed to match server
/// frames that nothing matched.
/// </summary>
public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection<FramePhoto> local, IReadOnlyCollection<FramePhoto> remote)
{
var remote = remoteHashes.Select(h => h.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
var hashed = localHashes.Where(h => !string.IsNullOrEmpty(h)).Select(h => h!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
var unhashed = localHashes.Count(string.IsNullOrEmpty);
var matched = hashed.Count(remote.Contains);
var unmatchedRemote = remote.Count - matched;
return (hashed.Count - matched + Math.Max(0, unhashed - unmatchedRemote), Math.Max(0, unmatchedRemote - unhashed));
ArgumentNullException.ThrowIfNull(local);
ArgumentNullException.ThrowIfNull(remote);
var remoteHashes = remote.Where(p => p.IsHashed).Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
var localHashes = local.Where(p => p.IsHashed).Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
// What no hash matched: each content once, each frame not hashed on its own.
var localLeft = Unmatched(local, remoteHashes);
var remoteLeft = Unmatched(remote, localHashes);
var byTime = new Dictionary<(long Size, long Time), List<int>>();
for (var i = 0; i < remoteLeft.Count; i++)
{
if (remoteLeft[i].CapturedAt is not { } time) continue;
var key = (remoteLeft[i].Size, time.Ticks / TimeSpan.TicksPerMillisecond);
if (!byTime.TryGetValue(key, out var list)) byTime[key] = list = [];
list.Add(i);
}
var paired = new bool[remoteLeft.Count];
int localHashedLeft = 0, localUnhashedLeft = 0, remoteMatched = 0;
foreach (var photo in localLeft)
{
var match = -1;
if (photo.CapturedAt is { } time && byTime.TryGetValue((photo.Size, time.Ticks / TimeSpan.TicksPerMillisecond), out var candidates))
{
var at = candidates.FindIndex(i => !paired[i] && remoteLeft[i].Matches(photo));
if (at >= 0) match = candidates[at];
}
if (match >= 0)
{
paired[match] = true;
remoteMatched++;
}
else if (photo.IsHashed)
{
localHashedLeft++;
}
else
{
localUnhashedLeft++;
}
}
var remoteUnmatched = remoteLeft.Count - remoteMatched;
return (localHashedLeft + Math.Max(0, localUnhashedLeft - remoteUnmatched), Math.Max(0, remoteUnmatched - localUnhashedLeft));
static List<FramePhoto> Unmatched(IEnumerable<FramePhoto> side, HashSet<string> other) =>
[
.. side.Where(p => p.IsHashed && !other.Contains(p.Sha256!.ToLowerInvariant())).DistinctBy(p => p.Sha256!.ToLowerInvariant()),
.. side.Where(p => !p.IsHashed),
];
}
/// <summary>
/// Frames whose proxy the server asks for (it cannot decode their originals, BRIEF §2.3) that this PC holds and its
/// <paramref name="decoder"/> reads: an upload makes and sends them. A local original not hashed yet (a folder adopted
/// and not verified) stands for a wanted frame of its size that no hash matched (by size, not name: the server names
/// its copy by its own template); the upload hashes it before making the proxy.
/// its copy by its own template); the upload hashes it before making the proxy. A wanted frame the server has not
/// hashed is the local original of its size and capture time (<paramref name="captured"/>, by server frame id), as
/// <see cref="ProxyOriginal"/> finds it for the upload.
/// </summary>
public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection<Frame> local, ImageDecoder decoder)
public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection<Frame> local, ImageDecoder decoder,
IReadOnlyDictionary<Guid, DateTime?>? captured = null)
{
ArgumentNullException.ThrowIfNull(manifest);
var hashed = local.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal);
var unhashed = local.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f)).GroupBy(f => f.Size).ToDictionary(g => g.Key, g => g.Count());
IReadOnlyList<Frame> frames = [.. local];
var photos = Photos(frames);
var count = 0;
// The originals of frames the server has not hashed first: they are no longer free to stand for others by size.
var taken = new HashSet<Guid>();
foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length == 0))
{
if (ProxyOriginal(frame, captured?.GetValueOrDefault(frame.Id), frames, photos) is { } original && taken.Add(original.Id) && Readable(original)) count++;
}
var hashed = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal);
var unhashed = frames.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f) && !taken.Contains(f.Id)).GroupBy(f => f.Size)
.ToDictionary(g => g.Key, g => g.Count());
foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length > 0))
{
var sha = frame.Sha256.ToLowerInvariant();
@@ -166,6 +228,22 @@ public static class SyncRules
bool Readable(Frame frame) => !ArchiveImages.NeedsProxy(decoder, frame.Format);
}
/// <summary>
/// The local original of a wanted frame the server has not hashed yet (adopted there, not verified): the one of its
/// size and capture time (<paramref name="capturedAt"/>, from the server's frame row); null when none is, or the time
/// is unknown. <paramref name="photos"/> are <see cref="Photos"/> of <paramref name="local"/>.
/// </summary>
public static Frame? ProxyOriginal(ManifestFrame wanted, DateTime? capturedAt, IReadOnlyList<Frame> local, FramePhotos photos)
{
ArgumentNullException.ThrowIfNull(wanted);
ArgumentNullException.ThrowIfNull(local);
ArgumentNullException.ThrowIfNull(photos);
return capturedAt is not null && photos.IndexOf(new FramePhoto(null, wanted.Size, capturedAt)) is { } at ? local[at] : null;
}
/// <summary>The photos of <paramref name="local"/>'s frames, by their place in the list.</summary>
public static FramePhotos Photos(IReadOnlyList<Frame> local) => new(local.Select((f, i) => (i, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
/// <summary>
/// Whether <paramref name="local"/> holds originals a server cannot decode (it asks for their proxies, BRIEF §2.3,
/// §4.2: managed decoders only) that this PC's <paramref name="decoder"/> reads, so it can make those proxies.
+7 -2
View File
@@ -505,9 +505,14 @@ public sealed class SyncService
var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false);
// A frame flagged missing here counts as absent: downloading the originals restores it from the server.
var present = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList();
var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]);
// A frame either side has not hashed yet (a folder adopted, not verified) is told apart by size and capture time.
var captured = manifest.Frames.Any(f => f.Sha256.Length == 0) || present.Any(f => f.Sha256 is null)
? (await remote.ListFramesAsync(link.RemoteId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt)
: null;
var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))],
[.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))]);
// Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3).
var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder) : 0;
var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured) : 0;
// Those change without the server's frame counters moving: while some may be made here, the server's next
// change of the sequence brings a new comparison.
var watch = policy.CanEdit && (proxies > 0 || SyncRules.MakesProxiesFor(present, Options.Archive.Decoder));
+109 -44
View File
@@ -4,6 +4,7 @@ using System.Net;
using Titano.Core.Archiving;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
@@ -11,6 +12,7 @@ using Titano.Data.Processing;
using Titano.Protocol;
using Titano.Protocol.Client;
using SequenceFrames = Titano.Data.Processing.SequenceFrames;
using SequenceManifest = Titano.Protocol.SequenceManifest;
namespace Titano.Data.Sync;
@@ -19,8 +21,9 @@ namespace Titano.Data.Sync;
/// resumable protocol after hashing it again (an original that no longer matches its archived hash is never sent),
/// creating the server sequence first and sending its settings when the upload made it, then the proxies the server asks
/// for (originals it cannot decode, BRIEF §2.3), made here from this PC's originals. Downloads fetch originals the
/// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 and commit
/// them through the archive ingest, which names them and writes the sidecar; the first original creates the local
/// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 (or, for an
/// original the server has not hashed yet, its size, and hash it here) and commit them through the archive ingest,
/// which names them and writes the sidecar; the first original creates the local
/// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the
/// server's copies of them are recognised), and originals flagged missing here are restored to their recorded place.
/// Proxy downloads fill the proxy cache. Renders travel on request: this PC's to the server (hash-verified chunked
@@ -44,10 +47,15 @@ internal sealed class SyncTransfers(SyncService sync)
var (remoteId, created) = await TargetAsync(context, remote, local).ConfigureAwait(false);
var frames = (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList();
var present = remoteId is { } known
? (await remote.GetManifestAsync(known, ct).ConfigureAwait(false)).Frames.Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal)
: [];
var pending = new Queue<Frame>(frames.Where(f => f.Sha256 is null || !present.Contains(f.Sha256.ToLowerInvariant())));
var manifest = remoteId is { } known ? await remote.GetManifestAsync(known, ct).ConfigureAwait(false) : null;
var present = manifest?.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal) ?? [];
// Originals the server holds without a hash (adopted there, not verified) are the ones of their size and capture
// time: the server could not recognise them by content, so they are not sent again.
var adopted = manifest is not null && manifest.Frames.Any(f => f.Sha256.Length == 0)
? await UnhashedAsync(remote, manifest, ct).ConfigureAwait(false)
: new FramePhotos([]);
var pending = new Queue<Frame>(frames.Where(f => (f.Sha256 is null || !present.Contains(f.Sha256.ToLowerInvariant()))
&& adopted.IndexOf(new FramePhoto(f.Sha256, f.Size, f.CapturedAt)) is null));
var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count);
await progress.ReportAsync().ConfigureAwait(false);
var skipped = 0;
@@ -86,10 +94,18 @@ internal sealed class SyncTransfers(SyncService sync)
if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped));
}
/// <summary>The server's frames it has not hashed yet, by size and capture time, at their indices.</summary>
private static async Task<FramePhotos> UnhashedAsync(ISyncRemote remote, SequenceManifest manifest, CancellationToken cancellationToken)
{
var captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
return new FramePhotos(manifest.Frames.Where(f => f.Sha256.Length == 0).Select(f => (f.Index, new FramePhoto(null, f.Size, captured.GetValueOrDefault(f.Id)))));
}
/// <summary>
/// The proxies the server asks for (<see cref="ManifestFrame.WantsProxy"/>) that this PC's decoders can make from its
/// originals (by their archived SHA-256, or the one <paramref name="hashes"/> holds for an original the index has none
/// for), made as the server's proxy policy says with the server's capture data, and sent. One the server refuses is
/// for; by size and capture time for a frame the server has not hashed, <see cref="SyncRules.ProxyOriginal"/>), made as
/// the server's proxy policy says with the server's capture data, and sent. One the server refuses is
/// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a
/// server busy or out of reach fails the job like an original would.
/// </summary>
@@ -106,6 +122,7 @@ internal sealed class SyncTransfers(SyncService sync)
if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame);
}
var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id);
var photos = SyncRules.Photos(frames);
var camera = new SidecarCamera { Make = local.CameraMake, Model = local.CameraModel, Serial = local.CameraSerial };
var decoder = sync.Options.Archive.Decoder;
var progress = new TransferProgress(context, wanted.Count, 0);
@@ -116,7 +133,12 @@ internal sealed class SyncTransfers(SyncService sync)
{
await Parallel.ForEachAsync(wanted, Workers(sync.Options.Uploads, refused.Token), async (frame, token) =>
{
var jpeg = originals.TryGetValue(frame.Sha256.ToLowerInvariant(), out var original) && records.TryGetValue(frame.Id, out var record)
records.TryGetValue(frame.Id, out var record);
// A frame the server has not hashed yet is this PC's original of its size and capture time.
var original = frame.Sha256.Length == 0
? SyncRules.ProxyOriginal(frame, record?.CapturedAt, frames, photos)
: originals.GetValueOrDefault(frame.Sha256.ToLowerInvariant());
var jpeg = original is not null && record is not null
? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName,
token), token).ConfigureAwait(false)
: null;
@@ -182,8 +204,14 @@ internal sealed class SyncTransfers(SyncService sync)
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
var (have, restore) = local is null ? ([], []) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
var wanted = manifest.Frames.Where(f => !have.Contains(f.Sha256.ToLowerInvariant())).DistinctBy(f => f.Sha256.ToLowerInvariant()).ToList();
var (have, restore, photos) = local is null ? ([], [], new FramePhotos([])) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
// An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and
// capture time is; each one is fetched on its own and hashed here.
var wanted = manifest.Frames
.Where(f => f.Sha256.Length == 0
? photos.IndexOf(new FramePhoto(null, f.Size, captured.GetValueOrDefault(f.Id))) is null
: !have.Contains(f.Sha256.ToLowerInvariant()))
.DistinctBy(f => f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
EnsureSpace(paths.Root, wanted.Sum(f => f.Size));
var staging = Path.Combine(paths.StagingDirectory, "sync-" + remoteId.ToString("N"));
@@ -195,7 +223,8 @@ internal sealed class SyncTransfers(SyncService sync)
if (local is null && wanted.Count > 0)
{
// The first original creates the local sequence (grouped by the server id); the others join it.
var first = await CommitAsync(remote, remoteId, archive.Id, null, wanted[0], staging, captured, ct).ConfigureAwait(false);
var (staged, sha) = await FetchOriginalAsync(remote, wanted[0], staging, ct).ConfigureAwait(false);
var first = await CommitAsync(remoteId, archive.Id, null, wanted[0], staged, sha, captured, ct).ConfigureAwait(false);
local = await db.Sequences.GetAsync(first.SequenceId, ct).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
await sync.LinkAsync(remote.Server, remoteId, local, ct).ConfigureAwait(false);
@@ -208,14 +237,15 @@ internal sealed class SyncTransfers(SyncService sync)
var restored = 0;
await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) =>
{
if (restore.TryGetValue(frame.Sha256.ToLowerInvariant(), out var place))
var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false);
if (restore.TryGetValue(sha, out var place))
{
await RestoreAsync(remote, frame, staging, place, token).ConfigureAwait(false);
Restore(staged, place);
Interlocked.Increment(ref restored);
}
else
{
await CommitAsync(remote, remoteId, archive.Id, sequenceId, frame, staging, captured, token).ConfigureAwait(false);
await CommitAsync(remoteId, archive.Id, sequenceId, frame, staged, sha, captured, token).ConfigureAwait(false);
}
await progress.StepAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
@@ -234,11 +264,12 @@ internal sealed class SyncTransfers(SyncService sync)
}
/// <summary>
/// The originals a local copy holds (lower-case SHA-256) and where each original flagged missing belongs. Frames
/// not hashed yet (an adopted folder never verified) are hashed first through the archive's verify, so the
/// server's copies of them are recognised instead of committed again as duplicates.
/// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet) and where
/// each original flagged missing belongs. Frames not hashed yet (an adopted folder never verified) are hashed first
/// through the archive's verify, so the server's copies of them are recognised instead of committed again as duplicates.
/// </summary>
private async Task<(HashSet<string> Have, Dictionary<string, string> Restore)> LocalContentAsync(Archive archive, Sequence local, CancellationToken ct)
private async Task<(HashSet<string> Have, Dictionary<string, string> Restore, FramePhotos Photos)> LocalContentAsync(Archive archive, Sequence local,
CancellationToken ct)
{
var frames = await SequenceFrames.ListAsync(sync.Database, local.Id, ct).ConfigureAwait(false);
if (frames.Any(f => f.Sha256 is null && (f.Flags & FrameFlags.Missing) == 0))
@@ -255,16 +286,16 @@ internal sealed class SyncTransfers(SyncService sync)
var sha = frame.Sha256!.ToLowerInvariant();
if (!have.Contains(sha)) restore.TryAdd(sha, paths.FromRelative(frame.RelativePath));
}
return (have, restore);
var photos = SyncRules.Photos([.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0)]);
return (have, restore, photos);
}
private Task VerifyAsync(Archive archive, Guid sequenceId, CancellationToken ct) =>
sync.Archives.Verifier.VerifyAsync(archive, [sequenceId], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: ct);
/// <summary>Puts a verified copy of an original flagged missing back where its record says; a file that reappeared meanwhile is kept.</summary>
private static async Task RestoreAsync(ISyncRemote remote, ManifestFrame frame, string staging, string place, CancellationToken cancellationToken)
private static void Restore(string staged, string place)
{
var staged = await FetchOriginalAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false);
if (File.Exists(place))
{
File.Delete(staged);
@@ -476,13 +507,12 @@ internal sealed class SyncTransfers(SyncService sync)
?? new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1);
}
/// <summary>Downloads one original into staging and commits it into the local sequence (created from the server id when null).</summary>
private async Task<IngestResult> CommitAsync(ISyncRemote remote, Guid remoteId, Guid archiveId, Guid? sequenceId, ManifestFrame frame, string staging,
/// <summary>Commits one original from staging into the local sequence (created from the server id when null).</summary>
private async Task<IngestResult> CommitAsync(Guid remoteId, Guid archiveId, Guid? sequenceId, ManifestFrame frame, string staged, string sha,
IReadOnlyDictionary<Guid, DateTime?> captured, CancellationToken cancellationToken)
{
var staged = await FetchOriginalAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false);
var request = new IngestRequest(archiveId, sequenceId, null, sequenceId is null ? SyncService.SessionKey(remoteId) : null, staged, frame.FileName,
frame.Size, frame.Sha256.ToLowerInvariant(), captured.GetValueOrDefault(frame.Id), null);
frame.Size, sha, captured.GetValueOrDefault(frame.Id), null);
var result = await sync.Archives.Ingest.CommitAsync(request, cancellationToken).ConfigureAwait(false);
// A duplicate stays in staging (the sequence already holds that content).
if (result.Duplicate) File.Delete(staged);
@@ -490,42 +520,77 @@ internal sealed class SyncTransfers(SyncService sync)
}
/// <summary>
/// A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two
/// attempts). A part as long as the file (a job stopped while hashing it) is only hashed again; a part the server can
/// no longer resume (416) starts over.
/// A verified copy in staging and its lower-case SHA-256: resumed from a partial download, accepted only with the
/// manifest's SHA-256 (two attempts). A part as long as the file (a job stopped while hashing it) is only hashed again;
/// a part the server can no longer resume (416) starts over.
/// </summary>
private static async Task<string> FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging, CancellationToken cancellationToken)
private static async Task<(string Path, string Sha256)> FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging,
CancellationToken cancellationToken)
{
if (frame.Sha256.Length == 0) return await FetchUnhashedAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false);
var sha = frame.Sha256.ToLowerInvariant();
if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName));
var target = Path.Combine(staging, sha + Path.GetExtension(frame.FileName).ToLowerInvariant());
if (File.Exists(target)) return target;
if (File.Exists(target)) return (target, sha);
var part = target + ".part";
for (var attempt = 1; ; attempt++)
{
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size)
{
try
{
await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false);
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1)
{
File.Delete(part);
continue;
}
}
if (!await FetchPartAsync(remote, frame, part, attempt, cancellationToken).ConfigureAwait(false)) continue;
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
{
File.Move(part, target, overwrite: true);
return target;
return (target, sha);
}
File.Delete(part);
if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName));
}
}
/// <summary>
/// An original the server has not hashed yet (a folder it adopted and did not verify): nothing to check it against
/// but its size, so a copy of another size is fetched again once; it is hashed here, and the archive ingest files it
/// by that hash like any other. Kept in staging under the frame's id, as several such originals may share content.
/// </summary>
private static async Task<(string Path, string Sha256)> FetchUnhashedAsync(ISyncRemote remote, ManifestFrame frame, string staging,
CancellationToken cancellationToken)
{
var target = Path.Combine(staging, frame.Id.ToString("N") + Path.GetExtension(frame.FileName).ToLowerInvariant());
var part = target + ".part";
for (var attempt = 1; !File.Exists(target); attempt++)
{
if (!await FetchPartAsync(remote, frame, part, attempt, cancellationToken).ConfigureAwait(false)) continue;
if (new FileInfo(part).Length == frame.Size)
{
File.Move(part, target, overwrite: true);
break;
}
File.Delete(part);
if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName));
}
return (target, await TitanoClient.HashFileAsync(target, cancellationToken).ConfigureAwait(false));
}
/// <summary>
/// Downloads an original into <paramref name="part"/>, resuming after what it holds; a part as long as the file is
/// kept as it is. False when the server can no longer resume it (416) on the first attempt: the part is gone and the
/// next attempt starts over.
/// </summary>
private static async Task<bool> FetchPartAsync(ISyncRemote remote, ManifestFrame frame, string part, int attempt, CancellationToken cancellationToken)
{
if (File.Exists(part) && new FileInfo(part).Length >= frame.Size) return true;
try
{
await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false);
return true;
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1)
{
File.Delete(part);
return false;
}
}
/// <summary>The cached proxy's file name, downloading the first asset kind the server has; null when it has none.</summary>
private static async Task<string?> FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList<string> kinds, string folder, CancellationToken cancellationToken)
{
@@ -1,6 +1,7 @@
using Titano.Core.Imaging;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Sync;
using Titano.Protocol;
using Titano.Testing;
@@ -66,6 +67,41 @@ public sealed class SyncRulesTests
Assert.Equal((1, 0), SyncRules.CompareOriginals([null, null], ["bb"]));
}
[Test]
public void FramesEitherSideHasNotHashedAreMatchedBySizeAndCaptureTime()
{
var time = new DateTime(2026, 9, 28, 18, 0, 0);
FramePhoto Photo(string? sha, long size, int second) => new(sha, size, time.AddSeconds(second));
// A folder the server adopted and never verified: each of its frames counts on its own, matched by size and time.
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa", 100, 0), Photo("bb", 200, 10)], [Photo("", 100, 0), Photo("", 200, 10)]));
Assert.Equal((1, 2), SyncRules.CompareOriginals([Photo("aa", 100, 0), Photo("bb", 200, 10)], [Photo("", 100, 0), Photo("", 300, 20), Photo("", 400, 30)]));
Assert.Equal((0, 3), SyncRules.CompareOriginals(Array.Empty<string?>(), ["", "", ""]));
// Hashed on both sides, the hash decides even when size and time agree.
Assert.Equal((1, 1), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("cc", 100, 0)]));
}
[Test]
public void ProxiesOfFramesTheServerHasNotHashedAreCountedByCaptureTime()
{
var heif = new ImageDecoder([new NativeHeif()]);
var time = new DateTime(2026, 9, 28, 18, 0, 0);
List<Frame> local =
[
Original("aa", 100, ImageFileFormat.Heif) with { CapturedAt = time },
Original("bb", 200, ImageFileFormat.Heif) with { CapturedAt = time.AddSeconds(10) },
];
var first = Wanted(string.Empty, 100, "IMG_0001.HEIC");
var second = Wanted(string.Empty, 200, "IMG_0002.HEIC");
var manifest = new SequenceManifest(Guid.CreateVersion7(), 1, 1, null, [], [first, second], [], new ProxySpec(1920, 90));
var captured = new Dictionary<Guid, DateTime?> { [first.Id] = time, [second.Id] = time.AddSeconds(11) };
Assert.Equal(1, SyncRules.ProxiesToSend(manifest, local, heif, captured));
Assert.Equal(0, SyncRules.ProxiesToSend(manifest, local, heif));
Assert.Same(local[0], SyncRules.ProxyOriginal(first, time, local, SyncRules.Photos(local)));
Assert.Null(SyncRules.ProxyOriginal(second, time.AddSeconds(11), local, SyncRules.Photos(local)));
}
[Test]
public void ProxiesTheServerAsksForAreCountedForOriginalsThisPcReads()
{
@@ -265,6 +265,81 @@ public sealed class SyncTransferTests(TestContext context)
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task OriginalsTheServerHasNotHashedAreDownloadedAndHashedHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// A folder the server adopted and never verified: its manifest lists no SHA-256.
var remote = f.Remote.Add("Adopted on the server", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Adopted on the server");
var frames = await f.Db.Frames.ListBySequenceAsync(entry.Local!.Id, cancellationToken: f.Ct);
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived, hashed here");
// Matched by size and capture time, the copies are the server's originals.
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
// One more photo adopted there: only that one travels.
remote.AddFrame(f.NewPhotos(1)[0]);
await f.RefreshAsync();
Assert.Equal(1, (await f.EntryAsync("Adopted on the server")).Link!.MissingLocal);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(4, (await f.Db.Frames.ListBySequenceAsync(entry.Local.Id, cancellationToken: f.Ct)).Count);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Adopted on the server")).Status);
}
[Test]
public async Task AnUploadDoesNotSendAgainWhatTheServerHoldsWithoutAHash()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Harbour", photos);
// The server adopted the first three photos and never verified them: it cannot recognise them by content.
var remote = f.Remote.Add("Harbour", photos[..3]);
remote.Unhashed = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = (await f.EntryAsync("Harbour")).Link!;
Assert.Equal((1, 0), (ahead.MissingRemote, ahead.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal((4L, 4L), (done.ProgressDone, done.ProgressTotal));
Assert.SequenceEqual(photos.Select(Sha), remote.Frames.Select(r => r.Sha256));
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task ProxiesOfFramesTheServerHasNotHashedAreSentToo()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server adopted the same originals, cannot decode them and has not hashed them.
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
remote.Unhashed = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(3, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Equal((SyncStatus.InSync, 0), ((await f.EntryAsync("Holy grail")).Status, (await f.EntryAsync("Holy grail")).Link!.MissingRemote));
}
[Test]
public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive()
{