diff --git a/src/Titano.Data/Sync/SyncRules.cs b/src/Titano.Data/Sync/SyncRules.cs
index 5f83ad05..8ab8bc42 100644
--- a/src/Titano.Data/Sync/SyncRules.cs
+++ b/src/Titano.Data/Sync/SyncRules.cs
@@ -123,31 +123,93 @@ public static class SyncRules
}
///
- /// Originals missing on each side by content (BRIEF §14.5: originals never conflict). Local frames not hashed yet
- /// (adopted, not verified) are assumed to match server frames that no hash matched.
+ /// Originals missing on each side by content (BRIEF §14.5: originals never conflict), from their SHA-256 alone (an
+ /// empty or null one: not hashed yet). See .
///
- public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection localHashes, IReadOnlyCollection remoteHashes)
+ public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection localHashes, IReadOnlyCollection remoteHashes) =>
+ CompareOriginals([.. localHashes.Select(h => new FramePhoto(h, 0, null))], [.. remoteHashes.Select(h => new FramePhoto(h, 0, null))]);
+
+ ///
+ /// Originals missing on each side by content (BRIEF §14.5: originals never conflict): the same SHA-256 where both
+ /// sides hashed them, else the same size and capture time (; a folder adopted on
+ /// either side and not verified yet). Local frames not hashed yet that nothing matched are assumed to match server
+ /// frames that nothing matched.
+ ///
+ public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection local, IReadOnlyCollection remote)
{
- var remote = remoteHashes.Select(h => h.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
- var hashed = localHashes.Where(h => !string.IsNullOrEmpty(h)).Select(h => h!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
- var unhashed = localHashes.Count(string.IsNullOrEmpty);
- var matched = hashed.Count(remote.Contains);
- var unmatchedRemote = remote.Count - matched;
- return (hashed.Count - matched + Math.Max(0, unhashed - unmatchedRemote), Math.Max(0, unmatchedRemote - unhashed));
+ ArgumentNullException.ThrowIfNull(local);
+ ArgumentNullException.ThrowIfNull(remote);
+ var remoteHashes = remote.Where(p => p.IsHashed).Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
+ var localHashes = local.Where(p => p.IsHashed).Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
+ // What no hash matched: each content once, each frame not hashed on its own.
+ var localLeft = Unmatched(local, remoteHashes);
+ var remoteLeft = Unmatched(remote, localHashes);
+ var byTime = new Dictionary<(long Size, long Time), List>();
+ for (var i = 0; i < remoteLeft.Count; i++)
+ {
+ if (remoteLeft[i].CapturedAt is not { } time) continue;
+ var key = (remoteLeft[i].Size, time.Ticks / TimeSpan.TicksPerMillisecond);
+ if (!byTime.TryGetValue(key, out var list)) byTime[key] = list = [];
+ list.Add(i);
+ }
+ var paired = new bool[remoteLeft.Count];
+ int localHashedLeft = 0, localUnhashedLeft = 0, remoteMatched = 0;
+ foreach (var photo in localLeft)
+ {
+ var match = -1;
+ if (photo.CapturedAt is { } time && byTime.TryGetValue((photo.Size, time.Ticks / TimeSpan.TicksPerMillisecond), out var candidates))
+ {
+ var at = candidates.FindIndex(i => !paired[i] && remoteLeft[i].Matches(photo));
+ if (at >= 0) match = candidates[at];
+ }
+ if (match >= 0)
+ {
+ paired[match] = true;
+ remoteMatched++;
+ }
+ else if (photo.IsHashed)
+ {
+ localHashedLeft++;
+ }
+ else
+ {
+ localUnhashedLeft++;
+ }
+ }
+ var remoteUnmatched = remoteLeft.Count - remoteMatched;
+ return (localHashedLeft + Math.Max(0, localUnhashedLeft - remoteUnmatched), Math.Max(0, remoteUnmatched - localUnhashedLeft));
+
+ static List Unmatched(IEnumerable side, HashSet other) =>
+ [
+ .. side.Where(p => p.IsHashed && !other.Contains(p.Sha256!.ToLowerInvariant())).DistinctBy(p => p.Sha256!.ToLowerInvariant()),
+ .. side.Where(p => !p.IsHashed),
+ ];
}
///
/// Frames whose proxy the server asks for (it cannot decode their originals, BRIEF §2.3) that this PC holds and its
/// reads: an upload makes and sends them. A local original not hashed yet (a folder adopted
/// and not verified) stands for a wanted frame of its size that no hash matched (by size, not name: the server names
- /// its copy by its own template); the upload hashes it before making the proxy.
+ /// its copy by its own template); the upload hashes it before making the proxy. A wanted frame the server has not
+ /// hashed is the local original of its size and capture time (, by server frame id), as
+ /// finds it for the upload.
///
- public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection local, ImageDecoder decoder)
+ public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection local, ImageDecoder decoder,
+ IReadOnlyDictionary? captured = null)
{
ArgumentNullException.ThrowIfNull(manifest);
- var hashed = local.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal);
- var unhashed = local.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f)).GroupBy(f => f.Size).ToDictionary(g => g.Key, g => g.Count());
+ IReadOnlyList frames = [.. local];
+ var photos = Photos(frames);
var count = 0;
+ // The originals of frames the server has not hashed first: they are no longer free to stand for others by size.
+ var taken = new HashSet();
+ foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length == 0))
+ {
+ if (ProxyOriginal(frame, captured?.GetValueOrDefault(frame.Id), frames, photos) is { } original && taken.Add(original.Id) && Readable(original)) count++;
+ }
+ var hashed = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal);
+ var unhashed = frames.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f) && !taken.Contains(f.Id)).GroupBy(f => f.Size)
+ .ToDictionary(g => g.Key, g => g.Count());
foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length > 0))
{
var sha = frame.Sha256.ToLowerInvariant();
@@ -166,6 +228,22 @@ public static class SyncRules
bool Readable(Frame frame) => !ArchiveImages.NeedsProxy(decoder, frame.Format);
}
+ ///
+ /// The local original of a wanted frame the server has not hashed yet (adopted there, not verified): the one of its
+ /// size and capture time (, from the server's frame row); null when none is, or the time
+ /// is unknown. are of .
+ ///
+ public static Frame? ProxyOriginal(ManifestFrame wanted, DateTime? capturedAt, IReadOnlyList local, FramePhotos photos)
+ {
+ ArgumentNullException.ThrowIfNull(wanted);
+ ArgumentNullException.ThrowIfNull(local);
+ ArgumentNullException.ThrowIfNull(photos);
+ return capturedAt is not null && photos.IndexOf(new FramePhoto(null, wanted.Size, capturedAt)) is { } at ? local[at] : null;
+ }
+
+ /// The photos of 's frames, by their place in the list.
+ public static FramePhotos Photos(IReadOnlyList local) => new(local.Select((f, i) => (i, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
+
///
/// Whether holds originals a server cannot decode (it asks for their proxies, BRIEF §2.3,
/// §4.2: managed decoders only) that this PC's reads, so it can make those proxies.
diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs
index 3f5becf6..b32e2a5c 100644
--- a/src/Titano.Data/Sync/SyncService.cs
+++ b/src/Titano.Data/Sync/SyncService.cs
@@ -505,9 +505,14 @@ public sealed class SyncService
var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false);
// A frame flagged missing here counts as absent: downloading the originals restores it from the server.
var present = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList();
- var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]);
+ // A frame either side has not hashed yet (a folder adopted, not verified) is told apart by size and capture time.
+ var captured = manifest.Frames.Any(f => f.Sha256.Length == 0) || present.Any(f => f.Sha256 is null)
+ ? (await remote.ListFramesAsync(link.RemoteId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt)
+ : null;
+ var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))],
+ [.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))]);
// Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3).
- var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder) : 0;
+ var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured) : 0;
// Those change without the server's frame counters moving: while some may be made here, the server's next
// change of the sequence brings a new comparison.
var watch = policy.CanEdit && (proxies > 0 || SyncRules.MakesProxiesFor(present, Options.Archive.Decoder));
diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs
index ae361dbc..e63736f2 100644
--- a/src/Titano.Data/Sync/SyncTransfers.cs
+++ b/src/Titano.Data/Sync/SyncTransfers.cs
@@ -4,6 +4,7 @@ using System.Net;
using Titano.Core.Archiving;
using Titano.Core.Localization;
using Titano.Core.Model;
+using Titano.Core.Project;
using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
@@ -11,6 +12,7 @@ using Titano.Data.Processing;
using Titano.Protocol;
using Titano.Protocol.Client;
using SequenceFrames = Titano.Data.Processing.SequenceFrames;
+using SequenceManifest = Titano.Protocol.SequenceManifest;
namespace Titano.Data.Sync;
@@ -19,8 +21,9 @@ namespace Titano.Data.Sync;
/// resumable protocol after hashing it again (an original that no longer matches its archived hash is never sent),
/// creating the server sequence first and sending its settings when the upload made it, then the proxies the server asks
/// for (originals it cannot decode, BRIEF §2.3), made here from this PC's originals. Downloads fetch originals the
-/// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 and commit
-/// them through the archive ingest, which names them and writes the sidecar; the first original creates the local
+/// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 (or, for an
+/// original the server has not hashed yet, its size, and hash it here) and commit them through the archive ingest,
+/// which names them and writes the sidecar; the first original creates the local
/// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the
/// server's copies of them are recognised), and originals flagged missing here are restored to their recorded place.
/// Proxy downloads fill the proxy cache. Renders travel on request: this PC's to the server (hash-verified chunked
@@ -44,10 +47,15 @@ internal sealed class SyncTransfers(SyncService sync)
var (remoteId, created) = await TargetAsync(context, remote, local).ConfigureAwait(false);
var frames = (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList();
- var present = remoteId is { } known
- ? (await remote.GetManifestAsync(known, ct).ConfigureAwait(false)).Frames.Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal)
- : [];
- var pending = new Queue(frames.Where(f => f.Sha256 is null || !present.Contains(f.Sha256.ToLowerInvariant())));
+ var manifest = remoteId is { } known ? await remote.GetManifestAsync(known, ct).ConfigureAwait(false) : null;
+ var present = manifest?.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal) ?? [];
+ // Originals the server holds without a hash (adopted there, not verified) are the ones of their size and capture
+ // time: the server could not recognise them by content, so they are not sent again.
+ var adopted = manifest is not null && manifest.Frames.Any(f => f.Sha256.Length == 0)
+ ? await UnhashedAsync(remote, manifest, ct).ConfigureAwait(false)
+ : new FramePhotos([]);
+ var pending = new Queue(frames.Where(f => (f.Sha256 is null || !present.Contains(f.Sha256.ToLowerInvariant()))
+ && adopted.IndexOf(new FramePhoto(f.Sha256, f.Size, f.CapturedAt)) is null));
var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count);
await progress.ReportAsync().ConfigureAwait(false);
var skipped = 0;
@@ -86,10 +94,18 @@ internal sealed class SyncTransfers(SyncService sync)
if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped));
}
+ /// The server's frames it has not hashed yet, by size and capture time, at their indices.
+ private static async Task UnhashedAsync(ISyncRemote remote, SequenceManifest manifest, CancellationToken cancellationToken)
+ {
+ var captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
+ return new FramePhotos(manifest.Frames.Where(f => f.Sha256.Length == 0).Select(f => (f.Index, new FramePhoto(null, f.Size, captured.GetValueOrDefault(f.Id)))));
+ }
+
///
/// The proxies the server asks for () that this PC's decoders can make from its
/// originals (by their archived SHA-256, or the one holds for an original the index has none
- /// for), made as the server's proxy policy says with the server's capture data, and sent. One the server refuses is
+ /// for; by size and capture time for a frame the server has not hashed, ), made as
+ /// the server's proxy policy says with the server's capture data, and sent. One the server refuses is
/// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a
/// server busy or out of reach fails the job like an original would.
///
@@ -106,6 +122,7 @@ internal sealed class SyncTransfers(SyncService sync)
if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame);
}
var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id);
+ var photos = SyncRules.Photos(frames);
var camera = new SidecarCamera { Make = local.CameraMake, Model = local.CameraModel, Serial = local.CameraSerial };
var decoder = sync.Options.Archive.Decoder;
var progress = new TransferProgress(context, wanted.Count, 0);
@@ -116,7 +133,12 @@ internal sealed class SyncTransfers(SyncService sync)
{
await Parallel.ForEachAsync(wanted, Workers(sync.Options.Uploads, refused.Token), async (frame, token) =>
{
- var jpeg = originals.TryGetValue(frame.Sha256.ToLowerInvariant(), out var original) && records.TryGetValue(frame.Id, out var record)
+ records.TryGetValue(frame.Id, out var record);
+ // A frame the server has not hashed yet is this PC's original of its size and capture time.
+ var original = frame.Sha256.Length == 0
+ ? SyncRules.ProxyOriginal(frame, record?.CapturedAt, frames, photos)
+ : originals.GetValueOrDefault(frame.Sha256.ToLowerInvariant());
+ var jpeg = original is not null && record is not null
? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName,
token), token).ConfigureAwait(false)
: null;
@@ -182,8 +204,14 @@ internal sealed class SyncTransfers(SyncService sync)
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
- var (have, restore) = local is null ? ([], []) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
- var wanted = manifest.Frames.Where(f => !have.Contains(f.Sha256.ToLowerInvariant())).DistinctBy(f => f.Sha256.ToLowerInvariant()).ToList();
+ var (have, restore, photos) = local is null ? ([], [], new FramePhotos([])) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
+ // An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and
+ // capture time is; each one is fetched on its own and hashed here.
+ var wanted = manifest.Frames
+ .Where(f => f.Sha256.Length == 0
+ ? photos.IndexOf(new FramePhoto(null, f.Size, captured.GetValueOrDefault(f.Id))) is null
+ : !have.Contains(f.Sha256.ToLowerInvariant()))
+ .DistinctBy(f => f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
EnsureSpace(paths.Root, wanted.Sum(f => f.Size));
var staging = Path.Combine(paths.StagingDirectory, "sync-" + remoteId.ToString("N"));
@@ -195,7 +223,8 @@ internal sealed class SyncTransfers(SyncService sync)
if (local is null && wanted.Count > 0)
{
// The first original creates the local sequence (grouped by the server id); the others join it.
- var first = await CommitAsync(remote, remoteId, archive.Id, null, wanted[0], staging, captured, ct).ConfigureAwait(false);
+ var (staged, sha) = await FetchOriginalAsync(remote, wanted[0], staging, ct).ConfigureAwait(false);
+ var first = await CommitAsync(remoteId, archive.Id, null, wanted[0], staged, sha, captured, ct).ConfigureAwait(false);
local = await db.Sequences.GetAsync(first.SequenceId, ct).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
await sync.LinkAsync(remote.Server, remoteId, local, ct).ConfigureAwait(false);
@@ -208,14 +237,15 @@ internal sealed class SyncTransfers(SyncService sync)
var restored = 0;
await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) =>
{
- if (restore.TryGetValue(frame.Sha256.ToLowerInvariant(), out var place))
+ var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false);
+ if (restore.TryGetValue(sha, out var place))
{
- await RestoreAsync(remote, frame, staging, place, token).ConfigureAwait(false);
+ Restore(staged, place);
Interlocked.Increment(ref restored);
}
else
{
- await CommitAsync(remote, remoteId, archive.Id, sequenceId, frame, staging, captured, token).ConfigureAwait(false);
+ await CommitAsync(remoteId, archive.Id, sequenceId, frame, staged, sha, captured, token).ConfigureAwait(false);
}
await progress.StepAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
@@ -234,11 +264,12 @@ internal sealed class SyncTransfers(SyncService sync)
}
///
- /// The originals a local copy holds (lower-case SHA-256) and where each original flagged missing belongs. Frames
- /// not hashed yet (an adopted folder never verified) are hashed first through the archive's verify, so the
- /// server's copies of them are recognised instead of committed again as duplicates.
+ /// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet) and where
+ /// each original flagged missing belongs. Frames not hashed yet (an adopted folder never verified) are hashed first
+ /// through the archive's verify, so the server's copies of them are recognised instead of committed again as duplicates.
///
- private async Task<(HashSet Have, Dictionary Restore)> LocalContentAsync(Archive archive, Sequence local, CancellationToken ct)
+ private async Task<(HashSet Have, Dictionary Restore, FramePhotos Photos)> LocalContentAsync(Archive archive, Sequence local,
+ CancellationToken ct)
{
var frames = await SequenceFrames.ListAsync(sync.Database, local.Id, ct).ConfigureAwait(false);
if (frames.Any(f => f.Sha256 is null && (f.Flags & FrameFlags.Missing) == 0))
@@ -255,16 +286,16 @@ internal sealed class SyncTransfers(SyncService sync)
var sha = frame.Sha256!.ToLowerInvariant();
if (!have.Contains(sha)) restore.TryAdd(sha, paths.FromRelative(frame.RelativePath));
}
- return (have, restore);
+ var photos = SyncRules.Photos([.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0)]);
+ return (have, restore, photos);
}
private Task VerifyAsync(Archive archive, Guid sequenceId, CancellationToken ct) =>
sync.Archives.Verifier.VerifyAsync(archive, [sequenceId], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: ct);
/// Puts a verified copy of an original flagged missing back where its record says; a file that reappeared meanwhile is kept.
- private static async Task RestoreAsync(ISyncRemote remote, ManifestFrame frame, string staging, string place, CancellationToken cancellationToken)
+ private static void Restore(string staged, string place)
{
- var staged = await FetchOriginalAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false);
if (File.Exists(place))
{
File.Delete(staged);
@@ -476,13 +507,12 @@ internal sealed class SyncTransfers(SyncService sync)
?? new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1);
}
- /// Downloads one original into staging and commits it into the local sequence (created from the server id when null).
- private async Task CommitAsync(ISyncRemote remote, Guid remoteId, Guid archiveId, Guid? sequenceId, ManifestFrame frame, string staging,
+ /// Commits one original from staging into the local sequence (created from the server id when null).
+ private async Task CommitAsync(Guid remoteId, Guid archiveId, Guid? sequenceId, ManifestFrame frame, string staged, string sha,
IReadOnlyDictionary captured, CancellationToken cancellationToken)
{
- var staged = await FetchOriginalAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false);
var request = new IngestRequest(archiveId, sequenceId, null, sequenceId is null ? SyncService.SessionKey(remoteId) : null, staged, frame.FileName,
- frame.Size, frame.Sha256.ToLowerInvariant(), captured.GetValueOrDefault(frame.Id), null);
+ frame.Size, sha, captured.GetValueOrDefault(frame.Id), null);
var result = await sync.Archives.Ingest.CommitAsync(request, cancellationToken).ConfigureAwait(false);
// A duplicate stays in staging (the sequence already holds that content).
if (result.Duplicate) File.Delete(staged);
@@ -490,42 +520,77 @@ internal sealed class SyncTransfers(SyncService sync)
}
///
- /// A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two
- /// attempts). A part as long as the file (a job stopped while hashing it) is only hashed again; a part the server can
- /// no longer resume (416) starts over.
+ /// A verified copy in staging and its lower-case SHA-256: resumed from a partial download, accepted only with the
+ /// manifest's SHA-256 (two attempts). A part as long as the file (a job stopped while hashing it) is only hashed again;
+ /// a part the server can no longer resume (416) starts over.
///
- private static async Task FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging, CancellationToken cancellationToken)
+ private static async Task<(string Path, string Sha256)> FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging,
+ CancellationToken cancellationToken)
{
+ if (frame.Sha256.Length == 0) return await FetchUnhashedAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false);
var sha = frame.Sha256.ToLowerInvariant();
if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName));
var target = Path.Combine(staging, sha + Path.GetExtension(frame.FileName).ToLowerInvariant());
- if (File.Exists(target)) return target;
+ if (File.Exists(target)) return (target, sha);
var part = target + ".part";
for (var attempt = 1; ; attempt++)
{
- if (!File.Exists(part) || new FileInfo(part).Length < frame.Size)
- {
- try
- {
- await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
- await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false);
- }
- catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1)
- {
- File.Delete(part);
- continue;
- }
- }
+ if (!await FetchPartAsync(remote, frame, part, attempt, cancellationToken).ConfigureAwait(false)) continue;
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
{
File.Move(part, target, overwrite: true);
- return target;
+ return (target, sha);
}
File.Delete(part);
if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName));
}
}
+ ///
+ /// An original the server has not hashed yet (a folder it adopted and did not verify): nothing to check it against
+ /// but its size, so a copy of another size is fetched again once; it is hashed here, and the archive ingest files it
+ /// by that hash like any other. Kept in staging under the frame's id, as several such originals may share content.
+ ///
+ private static async Task<(string Path, string Sha256)> FetchUnhashedAsync(ISyncRemote remote, ManifestFrame frame, string staging,
+ CancellationToken cancellationToken)
+ {
+ var target = Path.Combine(staging, frame.Id.ToString("N") + Path.GetExtension(frame.FileName).ToLowerInvariant());
+ var part = target + ".part";
+ for (var attempt = 1; !File.Exists(target); attempt++)
+ {
+ if (!await FetchPartAsync(remote, frame, part, attempt, cancellationToken).ConfigureAwait(false)) continue;
+ if (new FileInfo(part).Length == frame.Size)
+ {
+ File.Move(part, target, overwrite: true);
+ break;
+ }
+ File.Delete(part);
+ if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName));
+ }
+ return (target, await TitanoClient.HashFileAsync(target, cancellationToken).ConfigureAwait(false));
+ }
+
+ ///
+ /// Downloads an original into , resuming after what it holds; a part as long as the file is
+ /// kept as it is. False when the server can no longer resume it (416) on the first attempt: the part is gone and the
+ /// next attempt starts over.
+ ///
+ private static async Task FetchPartAsync(ISyncRemote remote, ManifestFrame frame, string part, int attempt, CancellationToken cancellationToken)
+ {
+ if (File.Exists(part) && new FileInfo(part).Length >= frame.Size) return true;
+ try
+ {
+ await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
+ await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false);
+ return true;
+ }
+ catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1)
+ {
+ File.Delete(part);
+ return false;
+ }
+ }
+
/// The cached proxy's file name, downloading the first asset kind the server has; null when it has none.
private static async Task FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList kinds, string folder, CancellationToken cancellationToken)
{
diff --git a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs
index da763383..672194b3 100644
--- a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs
+++ b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs
@@ -1,6 +1,7 @@
using Titano.Core.Imaging;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Model;
+using Titano.Core.Project;
using Titano.Data.Sync;
using Titano.Protocol;
using Titano.Testing;
@@ -66,6 +67,41 @@ public sealed class SyncRulesTests
Assert.Equal((1, 0), SyncRules.CompareOriginals([null, null], ["bb"]));
}
+ [Test]
+ public void FramesEitherSideHasNotHashedAreMatchedBySizeAndCaptureTime()
+ {
+ var time = new DateTime(2026, 9, 28, 18, 0, 0);
+ FramePhoto Photo(string? sha, long size, int second) => new(sha, size, time.AddSeconds(second));
+
+ // A folder the server adopted and never verified: each of its frames counts on its own, matched by size and time.
+ Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa", 100, 0), Photo("bb", 200, 10)], [Photo("", 100, 0), Photo("", 200, 10)]));
+ Assert.Equal((1, 2), SyncRules.CompareOriginals([Photo("aa", 100, 0), Photo("bb", 200, 10)], [Photo("", 100, 0), Photo("", 300, 20), Photo("", 400, 30)]));
+ Assert.Equal((0, 3), SyncRules.CompareOriginals(Array.Empty(), ["", "", ""]));
+ // Hashed on both sides, the hash decides even when size and time agree.
+ Assert.Equal((1, 1), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("cc", 100, 0)]));
+ }
+
+ [Test]
+ public void ProxiesOfFramesTheServerHasNotHashedAreCountedByCaptureTime()
+ {
+ var heif = new ImageDecoder([new NativeHeif()]);
+ var time = new DateTime(2026, 9, 28, 18, 0, 0);
+ List local =
+ [
+ Original("aa", 100, ImageFileFormat.Heif) with { CapturedAt = time },
+ Original("bb", 200, ImageFileFormat.Heif) with { CapturedAt = time.AddSeconds(10) },
+ ];
+ var first = Wanted(string.Empty, 100, "IMG_0001.HEIC");
+ var second = Wanted(string.Empty, 200, "IMG_0002.HEIC");
+ var manifest = new SequenceManifest(Guid.CreateVersion7(), 1, 1, null, [], [first, second], [], new ProxySpec(1920, 90));
+ var captured = new Dictionary { [first.Id] = time, [second.Id] = time.AddSeconds(11) };
+
+ Assert.Equal(1, SyncRules.ProxiesToSend(manifest, local, heif, captured));
+ Assert.Equal(0, SyncRules.ProxiesToSend(manifest, local, heif));
+ Assert.Same(local[0], SyncRules.ProxyOriginal(first, time, local, SyncRules.Photos(local)));
+ Assert.Null(SyncRules.ProxyOriginal(second, time.AddSeconds(11), local, SyncRules.Photos(local)));
+ }
+
[Test]
public void ProxiesTheServerAsksForAreCountedForOriginalsThisPcReads()
{
diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
index 6760a72f..1fd9c015 100644
--- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
+++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
@@ -265,6 +265,81 @@ public sealed class SyncTransferTests(TestContext context)
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
+ [Test]
+ public async Task OriginalsTheServerHasNotHashedAreDownloadedAndHashedHere()
+ {
+ await using var f = await SyncFixture.CreateAsync(context);
+ var photos = f.NewPhotos(3);
+ // A folder the server adopted and never verified: its manifest lists no SHA-256.
+ var remote = f.Remote.Add("Adopted on the server", photos);
+ remote.Unhashed = true;
+ await f.RefreshAsync();
+ var engine = await f.StartEngineAsync();
+
+ var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+
+ var entry = await f.EntryAsync("Adopted on the server");
+ var frames = await f.Db.Frames.ListBySequenceAsync(entry.Local!.Id, cancellationToken: f.Ct);
+ Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived, hashed here");
+ // Matched by size and capture time, the copies are the server's originals.
+ Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
+
+ // One more photo adopted there: only that one travels.
+ remote.AddFrame(f.NewPhotos(1)[0]);
+ await f.RefreshAsync();
+ Assert.Equal(1, (await f.EntryAsync("Adopted on the server")).Link!.MissingLocal);
+ job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+ Assert.Equal(4, (await f.Db.Frames.ListBySequenceAsync(entry.Local.Id, cancellationToken: f.Ct)).Count);
+ Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Adopted on the server")).Status);
+ }
+
+ [Test]
+ public async Task AnUploadDoesNotSendAgainWhatTheServerHoldsWithoutAHash()
+ {
+ await using var f = await SyncFixture.CreateAsync(context);
+ var photos = f.NewPhotos(4);
+ var local = await f.LocalAsync("Harbour", photos);
+ // The server adopted the first three photos and never verified them: it cannot recognise them by content.
+ var remote = f.Remote.Add("Harbour", photos[..3]);
+ remote.Unhashed = true;
+ await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
+ await f.RefreshAsync();
+ var ahead = (await f.EntryAsync("Harbour")).Link!;
+ Assert.Equal((1, 0), (ahead.MissingRemote, ahead.MissingLocal));
+ var engine = await f.StartEngineAsync();
+
+ var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
+ var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
+
+ Assert.Equal((4L, 4L), (done.ProgressDone, done.ProgressTotal));
+ Assert.SequenceEqual(photos.Select(Sha), remote.Frames.Select(r => r.Sha256));
+ Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
+ }
+
+ [Test]
+ public async Task ProxiesOfFramesTheServerHasNotHashedAreSentToo()
+ {
+ await using var f = await SyncFixture.CreateAsync(context);
+ var photos = f.NewPhotos(3);
+ var local = await f.LocalAsync("Holy grail", photos);
+ // The server adopted the same originals, cannot decode them and has not hashed them.
+ var remote = f.Remote.Add("Holy grail", photos);
+ remote.Undecodable = true;
+ remote.Unhashed = true;
+ await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
+ await f.RefreshAsync();
+ Assert.Equal(3, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
+ var engine = await f.StartEngineAsync();
+
+ var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+
+ Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
+ Assert.Equal((SyncStatus.InSync, 0), ((await f.EntryAsync("Holy grail")).Status, (await f.EntryAsync("Holy grail")).Link!.MissingRemote));
+ }
+
[Test]
public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive()
{