diff --git a/src/Titano.Data/Migrations/postgres/0006_sync_proxy_compare.sql b/src/Titano.Data/Migrations/postgres/0006_sync_proxy_compare.sql new file mode 100644 index 00000000..63764a45 --- /dev/null +++ b/src/Titano.Data/Migrations/postgres/0006_sync_proxy_compare.sql @@ -0,0 +1,5 @@ +-- Client sync state (BRIEF §2.3, §14.5): the server's update time a linked copy's originals were compared at, kept +-- while proxies the server asks for may be made on this PC; they change without the frame counters moving, so the +-- copy is compared again when the server's sequence changed since. Mirrors sqlite/0006_sync_proxy_compare.sql. + +ALTER TABLE sync_links ADD COLUMN diff_remote_updated_at timestamptz; diff --git a/src/Titano.Data/Migrations/sqlite/0006_sync_proxy_compare.sql b/src/Titano.Data/Migrations/sqlite/0006_sync_proxy_compare.sql new file mode 100644 index 00000000..6088e40b --- /dev/null +++ b/src/Titano.Data/Migrations/sqlite/0006_sync_proxy_compare.sql @@ -0,0 +1,5 @@ +-- Client sync state (BRIEF §2.3, §14.5): the server's update time a linked copy's originals were compared at, kept +-- while proxies the server asks for may be made on this PC; they change without the frame counters moving, so the +-- copy is compared again when the server's sequence changed since. Mirrors postgres/0006_sync_proxy_compare.sql. + +ALTER TABLE sync_links ADD COLUMN diff_remote_updated_at TEXT; diff --git a/src/Titano.Data/Sync/SyncLinkStore.cs b/src/Titano.Data/Sync/SyncLinkStore.cs index 1d28e3f4..9c7f0c75 100644 --- a/src/Titano.Data/Sync/SyncLinkStore.cs +++ b/src/Titano.Data/Sync/SyncLinkStore.cs @@ -11,7 +11,8 @@ internal sealed class SyncLinkStore(TitanoDatabase db) "server, remote_id, local_id, title, frame_count, total_bytes, render_count, remote_status, remote_settings_version, " + "remote_frames_version, remote_updated_at, base_local_settings, base_remote_settings, diff_local_frames, diff_remote_frames, " + "diff_local_renders, diff_remote_renders, missing_remote, missing_local, exports_to_get, exports_to_get_bytes, exports_to_send, " + - "exports_to_send_bytes, conflict_local_settings, conflict_remote_settings, conflict_remote, proxies_cached, proxies_frames_version, error, synced_at"; + "exports_to_send_bytes, conflict_local_settings, conflict_remote_settings, conflict_remote, proxies_cached, proxies_frames_version, error, synced_at, " + + "diff_remote_updated_at"; private static readonly string UpsertSql = SqlDialect.Upsert("sync_links", Columns.Split(", "), 2); @@ -36,7 +37,7 @@ internal sealed class SyncLinkStore(TitanoDatabase db) l.DiffLocalRenders, l.DiffRemoteRenders, l.MissingRemote, l.MissingLocal, l.ExportsToGet, l.ExportsToGetBytes, l.ExportsToSend, l.ExportsToSendBytes, l.ConflictLocalSettings, l.ConflictRemoteSettings, l.ConflictRemote is { } remote ? JsonSerializer.Serialize(remote, SyncPayloadJson.Default.SyncPayload) : null, - l.ProxiesCached, l.ProxiesFramesVersion, l.Error, l.SyncedAt, + l.ProxiesCached, l.ProxiesFramesVersion, l.Error, l.SyncedAt, l.DiffRemoteUpdatedAt, ], cancellationToken); public Task DeleteAsync(string server, Guid remoteId, CancellationToken cancellationToken) => @@ -103,5 +104,6 @@ internal sealed class SyncLinkStore(TitanoDatabase db) ProxiesFramesVersion = r.GetInt32OrNull(27), Error = r.GetStringOrNull(28), SyncedAt = r.GetDateTimeOffsetOrNull(29), + DiffRemoteUpdatedAt = r.GetDateTimeOffsetOrNull(30), }; } diff --git a/src/Titano.Data/Sync/SyncModel.cs b/src/Titano.Data/Sync/SyncModel.cs index eb842ff5..a379a54d 100644 --- a/src/Titano.Data/Sync/SyncModel.cs +++ b/src/Titano.Data/Sync/SyncModel.cs @@ -91,6 +91,14 @@ public sealed record SyncLink public int? DiffLocalRenders { get; init; } public int? DiffRemoteRenders { get; init; } + /// + /// The server's update time the originals were compared at, kept while proxies it asks for may be made here (some + /// were counted, or the copy holds originals the server cannot decode and this PC can); null otherwise. The proxies + /// it asks for change without its frame counters moving (one arrives from a render node, its proxy policy changes), + /// and every such change leaves a new update time, so the originals are compared again when it moved. + /// + public DateTimeOffset? DiffRemoteUpdatedAt { get; init; } + /// /// Local originals the server does not have, and proxies it asks for that this PC can make (it cannot decode those /// originals): an upload sends both. diff --git a/src/Titano.Data/Sync/SyncRules.cs b/src/Titano.Data/Sync/SyncRules.cs index ccda9f46..e0852bce 100644 --- a/src/Titano.Data/Sync/SyncRules.cs +++ b/src/Titano.Data/Sync/SyncRules.cs @@ -134,16 +134,41 @@ public static class SyncRules /// /// Frames whose proxy the server asks for (it cannot decode their originals, BRIEF §2.3) that this PC holds and its - /// reads: an upload makes and sends them. + /// reads: an upload makes and sends them. A local original not hashed yet (a folder adopted + /// and not verified) stands for a wanted frame of its size that no hash matched (by size, not name: the server names + /// its copy by its own template); the upload hashes it before making the proxy. /// public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection local, ImageDecoder decoder) { ArgumentNullException.ThrowIfNull(manifest); - var readable = local.Where(f => !string.IsNullOrEmpty(f.Sha256) && !ArchiveImages.NeedsProxy(decoder, f.Format)) - .Select(f => f.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); - return manifest.Frames.Count(f => f.WantsProxy && readable.Contains(f.Sha256.ToLowerInvariant())); + var hashed = local.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal); + var unhashed = local.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f)).GroupBy(f => f.Size).ToDictionary(g => g.Key, g => g.Count()); + var count = 0; + foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length > 0)) + { + var sha = frame.Sha256.ToLowerInvariant(); + if (hashed.Contains(sha)) + { + if (hashed[sha].Any(Readable)) count++; + } + else if (unhashed.TryGetValue(frame.Size, out var left) && left > 0) + { + unhashed[frame.Size] = left - 1; + count++; + } + } + return count; + + bool Readable(Frame frame) => !ArchiveImages.NeedsProxy(decoder, frame.Format); } + /// + /// Whether holds originals a server cannot decode (it asks for their proxies, BRIEF §2.3, + /// §4.2: managed decoders only) that this PC's reads, so it can make those proxies. + /// + public static bool MakesProxiesFor(IEnumerable local, ImageDecoder decoder) => + local.Select(f => f.Format).Distinct().Any(format => ArchiveImages.ServerNeedsProxy(format) && !ArchiveImages.NeedsProxy(decoder, format)); + internal static string Blank(string? text) => text?.Trim() ?? string.Empty; private static JsonNode? Parse(string json) diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index a1e655f7..cc5e6071 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -490,7 +490,8 @@ public sealed class SyncService } SequenceManifest? manifest = null; if (link.DiffLocalFrames != local.FramesVersion || link.DiffRemoteFrames != link.RemoteFramesVersion - || link.DiffLocalRenders != local.RenderedCount || link.DiffRemoteRenders != link.RenderCount) + || link.DiffLocalRenders != local.RenderedCount || link.DiffRemoteRenders != link.RenderCount + || (link.DiffRemoteUpdatedAt is { } compared && !SameMoment(compared, link.RemoteUpdatedAt))) { manifest = await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false); @@ -498,7 +499,10 @@ public sealed class SyncService var present = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList(); var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]); // Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3). - if (policy.CanEdit && manifest.Proxies is not null) missingRemote += SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder); + var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder) : 0; + // Those change without the server's frame counters moving: while some may be made here, the server's next + // change of the sequence brings a new comparison. + var watch = policy.CanEdit && (proxies > 0 || SyncRules.MakesProxiesFor(present, Options.Archive.Decoder)); link = await CompareExportsAsync(link, local, manifest.Renders, cancellationToken).ConfigureAwait(false) with { DiffLocalFrames = local.FramesVersion, @@ -506,7 +510,8 @@ public sealed class SyncService RemoteFramesVersion = manifest.FramesVersion, DiffLocalRenders = local.RenderedCount, DiffRemoteRenders = manifest.Renders.Count, - MissingRemote = missingRemote, + DiffRemoteUpdatedAt = watch ? link.RemoteUpdatedAt : null, + MissingRemote = missingRemote + proxies, MissingLocal = missingLocal, }; } @@ -639,6 +644,9 @@ public sealed class SyncService private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified); + /// The same instant as databases keep it (to the microsecond). + private static bool SameMoment(DateTimeOffset a, DateTimeOffset b) => a.UtcTicks / 10 == b.UtcTicks / 10; + /// /// Writes a payload into the local index, project and sidecar in one step (settings version + 1) under the sequence's /// lease, which the editor's saves take too, and only while the local copy is still at the settings version @@ -871,6 +879,7 @@ public sealed class SyncService DiffRemoteFrames = null, DiffLocalRenders = null, DiffRemoteRenders = null, + DiffRemoteUpdatedAt = null, MissingRemote = 0, MissingLocal = 0, ExportsToGet = 0, diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs index 95e42f10..ae361dbc 100644 --- a/src/Titano.Data/Sync/SyncTransfers.cs +++ b/src/Titano.Data/Sync/SyncTransfers.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Globalization; using System.Net; using Titano.Core.Archiving; @@ -50,12 +51,16 @@ internal sealed class SyncTransfers(SyncService sync) var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count); await progress.ReportAsync().ConfigureAwait(false); var skipped = 0; + // The SHA-256 of each original the index has none for (a folder adopted and not verified), as its upload computes + // it: the proxies the server asks for are matched to this PC's originals by content. Every such original is + // pending, so each one still here is hashed. + var hashes = new ConcurrentDictionary(); // Without a server sequence the originals go out one by one under this sequence's session key until the // server has filed one, which creates the sequence (an original it already holds elsewhere creates nothing). while (remoteId is null && pending.TryDequeue(out var first)) { - var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, ct) + var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, hashes, ct) .ConfigureAwait(false); if (sent is null) skipped++; else if (sent.SequenceId != Guid.Empty) @@ -70,11 +75,11 @@ internal sealed class SyncTransfers(SyncService sync) await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) => { - if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, token).ConfigureAwait(false) is null) + if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, hashes, token).ConfigureAwait(false) is null) Interlocked.Increment(ref skipped); await progress.StepAsync().ConfigureAwait(false); }).ConfigureAwait(false); - await SendProxiesAsync(context, remote, target, local, paths, frames).ConfigureAwait(false); + await SendProxiesAsync(context, remote, target, local, paths, frames, hashes).ConfigureAwait(false); if (created) await sync.PushAllAsync(remote, target, ct).ConfigureAwait(false); await sync.RecompareAsync(remote, target, ct).ConfigureAwait(false); @@ -83,17 +88,23 @@ internal sealed class SyncTransfers(SyncService sync) /// /// The proxies the server asks for () that this PC's decoders can make from its - /// originals, made as the server's proxy policy says with the server's capture data, and sent. One the server refuses - /// is counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a + /// originals (by their archived SHA-256, or the one holds for an original the index has none + /// for), made as the server's proxy policy says with the server's capture data, and sent. One the server refuses is + /// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a /// server busy or out of reach fails the job like an original would. /// - private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList frames) + private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList frames, + IReadOnlyDictionary hashes) { var ct = context.Cancellation; var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); var wanted = manifest.Frames.Where(f => f.WantsProxy).ToList(); if (manifest.Proxies is not { } spec || wanted.Count == 0) return; - var originals = frames.Where(f => f.Sha256 is not null).GroupBy(f => f.Sha256!.ToLowerInvariant()).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + var originals = new Dictionary(StringComparer.Ordinal); + foreach (var frame in frames) + { + if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame); + } var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id); var camera = new SidecarCamera { Make = local.CameraMake, Model = local.CameraModel, Serial = local.CameraSerial }; var decoder = sync.Options.Archive.Decoder; @@ -443,10 +454,11 @@ internal sealed class SyncTransfers(SyncService sync) /// /// Hashes the original again and sends it into (or the sequence of - /// ); null (logged) when it is gone or no longer the archived file. + /// ); null (logged) when it is gone or no longer the archived file. The hash of an + /// original the index has none for goes into . /// private static async Task UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path, - Frame frame, CancellationToken cancellationToken) + Frame frame, ConcurrentDictionary hashes, CancellationToken cancellationToken) { if (!File.Exists(path)) { @@ -454,6 +466,7 @@ internal sealed class SyncTransfers(SyncService sync) return null; } var sha = await TitanoClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false); + if (frame.Sha256 is null) hashes[frame.Id] = sha; if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase)) { context.Log($"{frame.FileName}: changed since it was archived, not uploaded"); diff --git a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs index f2b7fa91..da763383 100644 --- a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs @@ -1,5 +1,8 @@ +using Titano.Core.Imaging; +using Titano.Core.Imaging.Decoders; using Titano.Core.Model; using Titano.Data.Sync; +using Titano.Protocol; using Titano.Testing; namespace Titano.Data.Tests.Sync; @@ -63,6 +66,38 @@ public sealed class SyncRulesTests Assert.Equal((1, 0), SyncRules.CompareOriginals([null, null], ["bb"])); } + [Test] + public void ProxiesTheServerAsksForAreCountedForOriginalsThisPcReads() + { + var heif = new ImageDecoder([new NativeHeif()]); + List local = + [ + Original("aa", 100, ImageFileFormat.Heif), + Original(null, 200, ImageFileFormat.Heif), + Original(null, 300, ImageFileFormat.Heif), + Original("dd", 400, ImageFileFormat.Avif), + ]; + var manifest = new SequenceManifest(Guid.CreateVersion7(), 1, 1, null, [], + [ + Wanted("AA", 100, "2026-10-01_0001.heic"), + // Not hashed here: matched by size, whatever name the server gave its copy. + Wanted("bb", 200, "2026-10-01_0002.heic"), + Wanted("cc", 999, "IMG_0003.HEIC"), + // The server has no hash of it yet: nothing tells which original it is. + Wanted(string.Empty, 300, "IMG_0004.HEIC"), + // This PC cannot decode AVIF. + Wanted("dd", 400, "IMG_0005.AVIF"), + Wanted("ee", 200, "IMG_0006.HEIC") with { WantsProxy = false }, + ], [], new ProxySpec(1920, 90)); + + Assert.Equal(2, SyncRules.ProxiesToSend(manifest, local, heif)); + Assert.Equal(0, SyncRules.ProxiesToSend(manifest, local, DecoderFactory.CreateServer())); + + Assert.True(SyncRules.MakesProxiesFor(local, heif), "HEIF the server cannot decode and this PC can"); + Assert.False(SyncRules.MakesProxiesFor(local, DecoderFactory.CreateServer()), "a PC that reads no more than the server"); + Assert.False(SyncRules.MakesProxiesFor([Original("aa", 100, ImageFileFormat.Jpeg)], heif), "the server decodes JPEG itself"); + } + [Test] public void SettingsAreComparedAsProjectsNotAsText() { @@ -92,4 +127,29 @@ public sealed class SyncRulesTests Assert.True(mine.SameMetadata(mine with { Tags = ["HARBOUR", "sunset"], Notes = "mine " }), "tags ignore case and order, blanks are trimmed"); Assert.Empty(SyncRules.Differences(mine, mine with { Tags = ["harbour", "SUNSET"] })); } + + private static Frame Original(string? sha, long size, ImageFileFormat format) => new() + { + Id = Guid.CreateVersion7(), + SequenceId = Local.Id, + Index = (int)(size / 100), + RelativePath = $"2026/a/{size}", + FileName = $"{size}", + Size = size, + Sha256 = sha, + Format = format, + }; + + private static ManifestFrame Wanted(string sha, long size, string name) => + new(Guid.CreateVersion7(), (int)(size / 100), name, size, sha, HasProxy: false, HasThumb: true, WantsProxy: true); + + /// A provider that decodes HEIF natively (WIC with the HEIF extension on a desktop). + private sealed class NativeHeif : IImageDecoderProvider + { + public string Name => "native-heif"; + + public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null; + + public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null; + } } diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs index 129fd092..174aa73e 100644 --- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs @@ -62,6 +62,59 @@ public sealed class SyncTransferTests(TestContext context) Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status); } + [Test] + public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + // An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds + // under the names it gave them, and cannot decode (HEIF, BRIEF §2.3). + var folder = Path.Combine(f.Archive.Paths.Root, "Adopted"); + Directory.CreateDirectory(folder); + for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct); + await f.Archive.ScanAsync(); + var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Titano.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted"); + Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed"); + var remote = f.Remote.Add("Adopted", photos); + remote.Undecodable = true; + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var ahead = await f.EntryAsync(local.DisplayTitle); + Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); + + Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); + Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status); + } + + [Test] + public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Holy grail", photos); + var remote = f.Remote.Add("Holy grail", photos); + remote.Undecodable = true; + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote); + var calls = f.Remote.ManifestCalls; + await f.RefreshAsync(); + Assert.Equal(calls, f.Remote.ManifestCalls); + + // A render node sends them after a job: the server's frame counters stand still, its sequence changes. + foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy; + remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1); + await f.RefreshAsync(); + + var entry = await f.EntryAsync("Holy grail"); + Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote)); + } + [Test] public async Task AResumedUploadSendsOnlyWhatTheServerLacks() {