diff --git a/src/Titano.Data/Sync/SyncRules.cs b/src/Titano.Data/Sync/SyncRules.cs index 5f83ad05..8ab8bc42 100644 --- a/src/Titano.Data/Sync/SyncRules.cs +++ b/src/Titano.Data/Sync/SyncRules.cs @@ -123,31 +123,93 @@ public static class SyncRules } /// - /// Originals missing on each side by content (BRIEF §14.5: originals never conflict). Local frames not hashed yet - /// (adopted, not verified) are assumed to match server frames that no hash matched. + /// Originals missing on each side by content (BRIEF §14.5: originals never conflict), from their SHA-256 alone (an + /// empty or null one: not hashed yet). See . /// - public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection localHashes, IReadOnlyCollection remoteHashes) + public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection localHashes, IReadOnlyCollection remoteHashes) => + CompareOriginals([.. localHashes.Select(h => new FramePhoto(h, 0, null))], [.. remoteHashes.Select(h => new FramePhoto(h, 0, null))]); + + /// + /// Originals missing on each side by content (BRIEF §14.5: originals never conflict): the same SHA-256 where both + /// sides hashed them, else the same size and capture time (; a folder adopted on + /// either side and not verified yet). Local frames not hashed yet that nothing matched are assumed to match server + /// frames that nothing matched. + /// + public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection local, IReadOnlyCollection remote) { - var remote = remoteHashes.Select(h => h.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); - var hashed = localHashes.Where(h => !string.IsNullOrEmpty(h)).Select(h => h!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); - var unhashed = localHashes.Count(string.IsNullOrEmpty); - var matched = hashed.Count(remote.Contains); - var unmatchedRemote = remote.Count - matched; - return (hashed.Count - matched + Math.Max(0, unhashed - unmatchedRemote), Math.Max(0, unmatchedRemote - unhashed)); + ArgumentNullException.ThrowIfNull(local); + ArgumentNullException.ThrowIfNull(remote); + var remoteHashes = remote.Where(p => p.IsHashed).Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + var localHashes = local.Where(p => p.IsHashed).Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + // What no hash matched: each content once, each frame not hashed on its own. + var localLeft = Unmatched(local, remoteHashes); + var remoteLeft = Unmatched(remote, localHashes); + var byTime = new Dictionary<(long Size, long Time), List>(); + for (var i = 0; i < remoteLeft.Count; i++) + { + if (remoteLeft[i].CapturedAt is not { } time) continue; + var key = (remoteLeft[i].Size, time.Ticks / TimeSpan.TicksPerMillisecond); + if (!byTime.TryGetValue(key, out var list)) byTime[key] = list = []; + list.Add(i); + } + var paired = new bool[remoteLeft.Count]; + int localHashedLeft = 0, localUnhashedLeft = 0, remoteMatched = 0; + foreach (var photo in localLeft) + { + var match = -1; + if (photo.CapturedAt is { } time && byTime.TryGetValue((photo.Size, time.Ticks / TimeSpan.TicksPerMillisecond), out var candidates)) + { + var at = candidates.FindIndex(i => !paired[i] && remoteLeft[i].Matches(photo)); + if (at >= 0) match = candidates[at]; + } + if (match >= 0) + { + paired[match] = true; + remoteMatched++; + } + else if (photo.IsHashed) + { + localHashedLeft++; + } + else + { + localUnhashedLeft++; + } + } + var remoteUnmatched = remoteLeft.Count - remoteMatched; + return (localHashedLeft + Math.Max(0, localUnhashedLeft - remoteUnmatched), Math.Max(0, remoteUnmatched - localUnhashedLeft)); + + static List Unmatched(IEnumerable side, HashSet other) => + [ + .. side.Where(p => p.IsHashed && !other.Contains(p.Sha256!.ToLowerInvariant())).DistinctBy(p => p.Sha256!.ToLowerInvariant()), + .. side.Where(p => !p.IsHashed), + ]; } /// /// Frames whose proxy the server asks for (it cannot decode their originals, BRIEF §2.3) that this PC holds and its /// reads: an upload makes and sends them. A local original not hashed yet (a folder adopted /// and not verified) stands for a wanted frame of its size that no hash matched (by size, not name: the server names - /// its copy by its own template); the upload hashes it before making the proxy. + /// its copy by its own template); the upload hashes it before making the proxy. A wanted frame the server has not + /// hashed is the local original of its size and capture time (, by server frame id), as + /// finds it for the upload. /// - public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection local, ImageDecoder decoder) + public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection local, ImageDecoder decoder, + IReadOnlyDictionary? captured = null) { ArgumentNullException.ThrowIfNull(manifest); - var hashed = local.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal); - var unhashed = local.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f)).GroupBy(f => f.Size).ToDictionary(g => g.Key, g => g.Count()); + IReadOnlyList frames = [.. local]; + var photos = Photos(frames); var count = 0; + // The originals of frames the server has not hashed first: they are no longer free to stand for others by size. + var taken = new HashSet(); + foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length == 0)) + { + if (ProxyOriginal(frame, captured?.GetValueOrDefault(frame.Id), frames, photos) is { } original && taken.Add(original.Id) && Readable(original)) count++; + } + var hashed = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal); + var unhashed = frames.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f) && !taken.Contains(f.Id)).GroupBy(f => f.Size) + .ToDictionary(g => g.Key, g => g.Count()); foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length > 0)) { var sha = frame.Sha256.ToLowerInvariant(); @@ -166,6 +228,22 @@ public static class SyncRules bool Readable(Frame frame) => !ArchiveImages.NeedsProxy(decoder, frame.Format); } + /// + /// The local original of a wanted frame the server has not hashed yet (adopted there, not verified): the one of its + /// size and capture time (, from the server's frame row); null when none is, or the time + /// is unknown. are of . + /// + public static Frame? ProxyOriginal(ManifestFrame wanted, DateTime? capturedAt, IReadOnlyList local, FramePhotos photos) + { + ArgumentNullException.ThrowIfNull(wanted); + ArgumentNullException.ThrowIfNull(local); + ArgumentNullException.ThrowIfNull(photos); + return capturedAt is not null && photos.IndexOf(new FramePhoto(null, wanted.Size, capturedAt)) is { } at ? local[at] : null; + } + + /// The photos of 's frames, by their place in the list. + public static FramePhotos Photos(IReadOnlyList local) => new(local.Select((f, i) => (i, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); + /// /// Whether holds originals a server cannot decode (it asks for their proxies, BRIEF §2.3, /// §4.2: managed decoders only) that this PC's reads, so it can make those proxies. diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index 3f5becf6..b32e2a5c 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -505,9 +505,14 @@ public sealed class SyncService var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false); // A frame flagged missing here counts as absent: downloading the originals restores it from the server. var present = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList(); - var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]); + // A frame either side has not hashed yet (a folder adopted, not verified) is told apart by size and capture time. + var captured = manifest.Frames.Any(f => f.Sha256.Length == 0) || present.Any(f => f.Sha256 is null) + ? (await remote.ListFramesAsync(link.RemoteId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt) + : null; + var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))], + [.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))]); // Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3). - var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder) : 0; + var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured) : 0; // Those change without the server's frame counters moving: while some may be made here, the server's next // change of the sequence brings a new comparison. var watch = policy.CanEdit && (proxies > 0 || SyncRules.MakesProxiesFor(present, Options.Archive.Decoder)); diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs index ae361dbc..e63736f2 100644 --- a/src/Titano.Data/Sync/SyncTransfers.cs +++ b/src/Titano.Data/Sync/SyncTransfers.cs @@ -4,6 +4,7 @@ using System.Net; using Titano.Core.Archiving; using Titano.Core.Localization; using Titano.Core.Model; +using Titano.Core.Project; using Titano.Data.Archiving; using Titano.Data.Archiving.Jobs; using Titano.Data.Jobs; @@ -11,6 +12,7 @@ using Titano.Data.Processing; using Titano.Protocol; using Titano.Protocol.Client; using SequenceFrames = Titano.Data.Processing.SequenceFrames; +using SequenceManifest = Titano.Protocol.SequenceManifest; namespace Titano.Data.Sync; @@ -19,8 +21,9 @@ namespace Titano.Data.Sync; /// resumable protocol after hashing it again (an original that no longer matches its archived hash is never sent), /// creating the server sequence first and sending its settings when the upload made it, then the proxies the server asks /// for (originals it cannot decode, BRIEF §2.3), made here from this PC's originals. Downloads fetch originals the -/// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 and commit -/// them through the archive ingest, which names them and writes the sidecar; the first original creates the local +/// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 (or, for an +/// original the server has not hashed yet, its size, and hash it here) and commit them through the archive ingest, +/// which names them and writes the sidecar; the first original creates the local /// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the /// server's copies of them are recognised), and originals flagged missing here are restored to their recorded place. /// Proxy downloads fill the proxy cache. Renders travel on request: this PC's to the server (hash-verified chunked @@ -44,10 +47,15 @@ internal sealed class SyncTransfers(SyncService sync) var (remoteId, created) = await TargetAsync(context, remote, local).ConfigureAwait(false); var frames = (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList(); - var present = remoteId is { } known - ? (await remote.GetManifestAsync(known, ct).ConfigureAwait(false)).Frames.Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal) - : []; - var pending = new Queue(frames.Where(f => f.Sha256 is null || !present.Contains(f.Sha256.ToLowerInvariant()))); + var manifest = remoteId is { } known ? await remote.GetManifestAsync(known, ct).ConfigureAwait(false) : null; + var present = manifest?.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal) ?? []; + // Originals the server holds without a hash (adopted there, not verified) are the ones of their size and capture + // time: the server could not recognise them by content, so they are not sent again. + var adopted = manifest is not null && manifest.Frames.Any(f => f.Sha256.Length == 0) + ? await UnhashedAsync(remote, manifest, ct).ConfigureAwait(false) + : new FramePhotos([]); + var pending = new Queue(frames.Where(f => (f.Sha256 is null || !present.Contains(f.Sha256.ToLowerInvariant())) + && adopted.IndexOf(new FramePhoto(f.Sha256, f.Size, f.CapturedAt)) is null)); var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count); await progress.ReportAsync().ConfigureAwait(false); var skipped = 0; @@ -86,10 +94,18 @@ internal sealed class SyncTransfers(SyncService sync) if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped)); } + /// The server's frames it has not hashed yet, by size and capture time, at their indices. + private static async Task UnhashedAsync(ISyncRemote remote, SequenceManifest manifest, CancellationToken cancellationToken) + { + var captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt); + return new FramePhotos(manifest.Frames.Where(f => f.Sha256.Length == 0).Select(f => (f.Index, new FramePhoto(null, f.Size, captured.GetValueOrDefault(f.Id))))); + } + /// /// The proxies the server asks for () that this PC's decoders can make from its /// originals (by their archived SHA-256, or the one holds for an original the index has none - /// for), made as the server's proxy policy says with the server's capture data, and sent. One the server refuses is + /// for; by size and capture time for a frame the server has not hashed, ), made as + /// the server's proxy policy says with the server's capture data, and sent. One the server refuses is /// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a /// server busy or out of reach fails the job like an original would. /// @@ -106,6 +122,7 @@ internal sealed class SyncTransfers(SyncService sync) if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame); } var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id); + var photos = SyncRules.Photos(frames); var camera = new SidecarCamera { Make = local.CameraMake, Model = local.CameraModel, Serial = local.CameraSerial }; var decoder = sync.Options.Archive.Decoder; var progress = new TransferProgress(context, wanted.Count, 0); @@ -116,7 +133,12 @@ internal sealed class SyncTransfers(SyncService sync) { await Parallel.ForEachAsync(wanted, Workers(sync.Options.Uploads, refused.Token), async (frame, token) => { - var jpeg = originals.TryGetValue(frame.Sha256.ToLowerInvariant(), out var original) && records.TryGetValue(frame.Id, out var record) + records.TryGetValue(frame.Id, out var record); + // A frame the server has not hashed yet is this PC's original of its size and capture time. + var original = frame.Sha256.Length == 0 + ? SyncRules.ProxyOriginal(frame, record?.CapturedAt, frames, photos) + : originals.GetValueOrDefault(frame.Sha256.ToLowerInvariant()); + var jpeg = original is not null && record is not null ? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName, token), token).ConfigureAwait(false) : null; @@ -182,8 +204,14 @@ internal sealed class SyncTransfers(SyncService sync) var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt); - var (have, restore) = local is null ? ([], []) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false); - var wanted = manifest.Frames.Where(f => !have.Contains(f.Sha256.ToLowerInvariant())).DistinctBy(f => f.Sha256.ToLowerInvariant()).ToList(); + var (have, restore, photos) = local is null ? ([], [], new FramePhotos([])) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false); + // An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and + // capture time is; each one is fetched on its own and hashed here. + var wanted = manifest.Frames + .Where(f => f.Sha256.Length == 0 + ? photos.IndexOf(new FramePhoto(null, f.Size, captured.GetValueOrDefault(f.Id))) is null + : !have.Contains(f.Sha256.ToLowerInvariant())) + .DistinctBy(f => f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList(); EnsureSpace(paths.Root, wanted.Sum(f => f.Size)); var staging = Path.Combine(paths.StagingDirectory, "sync-" + remoteId.ToString("N")); @@ -195,7 +223,8 @@ internal sealed class SyncTransfers(SyncService sync) if (local is null && wanted.Count > 0) { // The first original creates the local sequence (grouped by the server id); the others join it. - var first = await CommitAsync(remote, remoteId, archive.Id, null, wanted[0], staging, captured, ct).ConfigureAwait(false); + var (staged, sha) = await FetchOriginalAsync(remote, wanted[0], staging, ct).ConfigureAwait(false); + var first = await CommitAsync(remoteId, archive.Id, null, wanted[0], staged, sha, captured, ct).ConfigureAwait(false); local = await db.Sequences.GetAsync(first.SequenceId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing")); await sync.LinkAsync(remote.Server, remoteId, local, ct).ConfigureAwait(false); @@ -208,14 +237,15 @@ internal sealed class SyncTransfers(SyncService sync) var restored = 0; await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) => { - if (restore.TryGetValue(frame.Sha256.ToLowerInvariant(), out var place)) + var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false); + if (restore.TryGetValue(sha, out var place)) { - await RestoreAsync(remote, frame, staging, place, token).ConfigureAwait(false); + Restore(staged, place); Interlocked.Increment(ref restored); } else { - await CommitAsync(remote, remoteId, archive.Id, sequenceId, frame, staging, captured, token).ConfigureAwait(false); + await CommitAsync(remoteId, archive.Id, sequenceId, frame, staged, sha, captured, token).ConfigureAwait(false); } await progress.StepAsync().ConfigureAwait(false); }).ConfigureAwait(false); @@ -234,11 +264,12 @@ internal sealed class SyncTransfers(SyncService sync) } /// - /// The originals a local copy holds (lower-case SHA-256) and where each original flagged missing belongs. Frames - /// not hashed yet (an adopted folder never verified) are hashed first through the archive's verify, so the - /// server's copies of them are recognised instead of committed again as duplicates. + /// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet) and where + /// each original flagged missing belongs. Frames not hashed yet (an adopted folder never verified) are hashed first + /// through the archive's verify, so the server's copies of them are recognised instead of committed again as duplicates. /// - private async Task<(HashSet Have, Dictionary Restore)> LocalContentAsync(Archive archive, Sequence local, CancellationToken ct) + private async Task<(HashSet Have, Dictionary Restore, FramePhotos Photos)> LocalContentAsync(Archive archive, Sequence local, + CancellationToken ct) { var frames = await SequenceFrames.ListAsync(sync.Database, local.Id, ct).ConfigureAwait(false); if (frames.Any(f => f.Sha256 is null && (f.Flags & FrameFlags.Missing) == 0)) @@ -255,16 +286,16 @@ internal sealed class SyncTransfers(SyncService sync) var sha = frame.Sha256!.ToLowerInvariant(); if (!have.Contains(sha)) restore.TryAdd(sha, paths.FromRelative(frame.RelativePath)); } - return (have, restore); + var photos = SyncRules.Photos([.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0)]); + return (have, restore, photos); } private Task VerifyAsync(Archive archive, Guid sequenceId, CancellationToken ct) => sync.Archives.Verifier.VerifyAsync(archive, [sequenceId], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: ct); /// Puts a verified copy of an original flagged missing back where its record says; a file that reappeared meanwhile is kept. - private static async Task RestoreAsync(ISyncRemote remote, ManifestFrame frame, string staging, string place, CancellationToken cancellationToken) + private static void Restore(string staged, string place) { - var staged = await FetchOriginalAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false); if (File.Exists(place)) { File.Delete(staged); @@ -476,13 +507,12 @@ internal sealed class SyncTransfers(SyncService sync) ?? new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1); } - /// Downloads one original into staging and commits it into the local sequence (created from the server id when null). - private async Task CommitAsync(ISyncRemote remote, Guid remoteId, Guid archiveId, Guid? sequenceId, ManifestFrame frame, string staging, + /// Commits one original from staging into the local sequence (created from the server id when null). + private async Task CommitAsync(Guid remoteId, Guid archiveId, Guid? sequenceId, ManifestFrame frame, string staged, string sha, IReadOnlyDictionary captured, CancellationToken cancellationToken) { - var staged = await FetchOriginalAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false); var request = new IngestRequest(archiveId, sequenceId, null, sequenceId is null ? SyncService.SessionKey(remoteId) : null, staged, frame.FileName, - frame.Size, frame.Sha256.ToLowerInvariant(), captured.GetValueOrDefault(frame.Id), null); + frame.Size, sha, captured.GetValueOrDefault(frame.Id), null); var result = await sync.Archives.Ingest.CommitAsync(request, cancellationToken).ConfigureAwait(false); // A duplicate stays in staging (the sequence already holds that content). if (result.Duplicate) File.Delete(staged); @@ -490,42 +520,77 @@ internal sealed class SyncTransfers(SyncService sync) } /// - /// A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two - /// attempts). A part as long as the file (a job stopped while hashing it) is only hashed again; a part the server can - /// no longer resume (416) starts over. + /// A verified copy in staging and its lower-case SHA-256: resumed from a partial download, accepted only with the + /// manifest's SHA-256 (two attempts). A part as long as the file (a job stopped while hashing it) is only hashed again; + /// a part the server can no longer resume (416) starts over. /// - private static async Task FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging, CancellationToken cancellationToken) + private static async Task<(string Path, string Sha256)> FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging, + CancellationToken cancellationToken) { + if (frame.Sha256.Length == 0) return await FetchUnhashedAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false); var sha = frame.Sha256.ToLowerInvariant(); if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName)); var target = Path.Combine(staging, sha + Path.GetExtension(frame.FileName).ToLowerInvariant()); - if (File.Exists(target)) return target; + if (File.Exists(target)) return (target, sha); var part = target + ".part"; for (var attempt = 1; ; attempt++) { - if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) - { - try - { - await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) - await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false); - } - catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1) - { - File.Delete(part); - continue; - } - } + if (!await FetchPartAsync(remote, frame, part, attempt, cancellationToken).ConfigureAwait(false)) continue; if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha) { File.Move(part, target, overwrite: true); - return target; + return (target, sha); } File.Delete(part); if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName)); } } + /// + /// An original the server has not hashed yet (a folder it adopted and did not verify): nothing to check it against + /// but its size, so a copy of another size is fetched again once; it is hashed here, and the archive ingest files it + /// by that hash like any other. Kept in staging under the frame's id, as several such originals may share content. + /// + private static async Task<(string Path, string Sha256)> FetchUnhashedAsync(ISyncRemote remote, ManifestFrame frame, string staging, + CancellationToken cancellationToken) + { + var target = Path.Combine(staging, frame.Id.ToString("N") + Path.GetExtension(frame.FileName).ToLowerInvariant()); + var part = target + ".part"; + for (var attempt = 1; !File.Exists(target); attempt++) + { + if (!await FetchPartAsync(remote, frame, part, attempt, cancellationToken).ConfigureAwait(false)) continue; + if (new FileInfo(part).Length == frame.Size) + { + File.Move(part, target, overwrite: true); + break; + } + File.Delete(part); + if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName)); + } + return (target, await TitanoClient.HashFileAsync(target, cancellationToken).ConfigureAwait(false)); + } + + /// + /// Downloads an original into , resuming after what it holds; a part as long as the file is + /// kept as it is. False when the server can no longer resume it (416) on the first attempt: the part is gone and the + /// next attempt starts over. + /// + private static async Task FetchPartAsync(ISyncRemote remote, ManifestFrame frame, string part, int attempt, CancellationToken cancellationToken) + { + if (File.Exists(part) && new FileInfo(part).Length >= frame.Size) return true; + try + { + await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false); + return true; + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1) + { + File.Delete(part); + return false; + } + } + /// The cached proxy's file name, downloading the first asset kind the server has; null when it has none. private static async Task FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList kinds, string folder, CancellationToken cancellationToken) { diff --git a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs index da763383..672194b3 100644 --- a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs @@ -1,6 +1,7 @@ using Titano.Core.Imaging; using Titano.Core.Imaging.Decoders; using Titano.Core.Model; +using Titano.Core.Project; using Titano.Data.Sync; using Titano.Protocol; using Titano.Testing; @@ -66,6 +67,41 @@ public sealed class SyncRulesTests Assert.Equal((1, 0), SyncRules.CompareOriginals([null, null], ["bb"])); } + [Test] + public void FramesEitherSideHasNotHashedAreMatchedBySizeAndCaptureTime() + { + var time = new DateTime(2026, 9, 28, 18, 0, 0); + FramePhoto Photo(string? sha, long size, int second) => new(sha, size, time.AddSeconds(second)); + + // A folder the server adopted and never verified: each of its frames counts on its own, matched by size and time. + Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa", 100, 0), Photo("bb", 200, 10)], [Photo("", 100, 0), Photo("", 200, 10)])); + Assert.Equal((1, 2), SyncRules.CompareOriginals([Photo("aa", 100, 0), Photo("bb", 200, 10)], [Photo("", 100, 0), Photo("", 300, 20), Photo("", 400, 30)])); + Assert.Equal((0, 3), SyncRules.CompareOriginals(Array.Empty(), ["", "", ""])); + // Hashed on both sides, the hash decides even when size and time agree. + Assert.Equal((1, 1), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("cc", 100, 0)])); + } + + [Test] + public void ProxiesOfFramesTheServerHasNotHashedAreCountedByCaptureTime() + { + var heif = new ImageDecoder([new NativeHeif()]); + var time = new DateTime(2026, 9, 28, 18, 0, 0); + List local = + [ + Original("aa", 100, ImageFileFormat.Heif) with { CapturedAt = time }, + Original("bb", 200, ImageFileFormat.Heif) with { CapturedAt = time.AddSeconds(10) }, + ]; + var first = Wanted(string.Empty, 100, "IMG_0001.HEIC"); + var second = Wanted(string.Empty, 200, "IMG_0002.HEIC"); + var manifest = new SequenceManifest(Guid.CreateVersion7(), 1, 1, null, [], [first, second], [], new ProxySpec(1920, 90)); + var captured = new Dictionary { [first.Id] = time, [second.Id] = time.AddSeconds(11) }; + + Assert.Equal(1, SyncRules.ProxiesToSend(manifest, local, heif, captured)); + Assert.Equal(0, SyncRules.ProxiesToSend(manifest, local, heif)); + Assert.Same(local[0], SyncRules.ProxyOriginal(first, time, local, SyncRules.Photos(local))); + Assert.Null(SyncRules.ProxyOriginal(second, time.AddSeconds(11), local, SyncRules.Photos(local))); + } + [Test] public void ProxiesTheServerAsksForAreCountedForOriginalsThisPcReads() { diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs index 6760a72f..1fd9c015 100644 --- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs @@ -265,6 +265,81 @@ public sealed class SyncTransferTests(TestContext context) Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } + [Test] + public async Task OriginalsTheServerHasNotHashedAreDownloadedAndHashedHere() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + // A folder the server adopted and never verified: its manifest lists no SHA-256. + var remote = f.Remote.Add("Adopted on the server", photos); + remote.Unhashed = true; + await f.RefreshAsync(); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + var entry = await f.EntryAsync("Adopted on the server"); + var frames = await f.Db.Frames.ListBySequenceAsync(entry.Local!.Id, cancellationToken: f.Ct); + Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived, hashed here"); + // Matched by size and capture time, the copies are the server's originals. + Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal)); + + // One more photo adopted there: only that one travels. + remote.AddFrame(f.NewPhotos(1)[0]); + await f.RefreshAsync(); + Assert.Equal(1, (await f.EntryAsync("Adopted on the server")).Link!.MissingLocal); + job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + Assert.Equal(4, (await f.Db.Frames.ListBySequenceAsync(entry.Local.Id, cancellationToken: f.Ct)).Count); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Adopted on the server")).Status); + } + + [Test] + public async Task AnUploadDoesNotSendAgainWhatTheServerHoldsWithoutAHash() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(4); + var local = await f.LocalAsync("Harbour", photos); + // The server adopted the first three photos and never verified them: it cannot recognise them by content. + var remote = f.Remote.Add("Harbour", photos[..3]); + remote.Unhashed = true; + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var ahead = (await f.EntryAsync("Harbour")).Link!; + Assert.Equal((1, 0), (ahead.MissingRemote, ahead.MissingLocal)); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); + + Assert.Equal((4L, 4L), (done.ProgressDone, done.ProgressTotal)); + Assert.SequenceEqual(photos.Select(Sha), remote.Frames.Select(r => r.Sha256)); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); + } + + [Test] + public async Task ProxiesOfFramesTheServerHasNotHashedAreSentToo() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Holy grail", photos); + // The server adopted the same originals, cannot decode them and has not hashed them. + var remote = f.Remote.Add("Holy grail", photos); + remote.Undecodable = true; + remote.Unhashed = true; + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + Assert.Equal(3, (await f.EntryAsync("Holy grail")).Link!.MissingRemote); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); + Assert.Equal((SyncStatus.InSync, 0), ((await f.EntryAsync("Holy grail")).Status, (await f.EntryAsync("Holy grail")).Link!.MissingRemote)); + } + [Test] public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive() {