diff --git a/src/Titano.Data/Sync/SyncPositions.cs b/src/Titano.Data/Sync/SyncPositions.cs index 00f746be..20c53e2f 100644 --- a/src/Titano.Data/Sync/SyncPositions.cs +++ b/src/Titano.Data/Sync/SyncPositions.cs @@ -1,3 +1,4 @@ +using Titano.Core.Model; using Titano.Core.Project; using Titano.Protocol; using SequenceFrames = Titano.Data.Processing.SequenceFrames; @@ -11,47 +12,113 @@ namespace Titano.Data.Sync; /// copy's frames before they are compared or applied, and this copy's to the server's frames as /// lists them before they are sent. A position on a photo the receiving copy lacks waits /// there as a pending frame and takes its place when the photo arrives (). +/// Positions are only ever translated against the frames they name: this copy's row is read with its frames as one +/// state (frames inserted here move the row's positions in the same step), and a manifest whose frames mix two orders +/// is not translated at all. /// internal sealed class SyncPositions(TitanoDatabase db, ISyncRemote remote, Guid localId, SequenceManifest manifest) { - private FramePhotos? _server; + /// Times this copy's frames are listed again when an insertion lands while they are read. + private const int Attempts = 3; + + /// The manifest's frames by hash, and with their capture times once fetched. + private FramePhotos? _hashed, _timed; /// The server's frames version the translation to the server's frames is made against. public int ServerFrames => manifest.FramesVersion; - /// Settings of the server sequence with their positions on this copy's frames, as they are now. - public async Task ToLocalAsync(string? settingsJson, CancellationToken cancellationToken) + /// + /// The server's settings with their positions on this copy's frames as they are now, with this copy's row at those + /// frames; null when that row is no longer at (an edit was saved here since), or + /// either copy's frames cannot be read as one state now (the next comparison translates again). + /// + public async Task ToLocalAsync(string? settingsJson, int settingsVersion, CancellationToken cancellationToken) { - if (!ProjectSettings.NamesFrames(settingsJson)) return settingsJson; - var local = await LocalAsync(cancellationToken).ConfigureAwait(false); - var server = await ServerAsync(local, cancellationToken).ConfigureAwait(false); - return ProjectSettings.TranslateFramePositions(settingsJson!, server, local) ?? settingsJson; + if (!ProjectSettings.NamesFrames(settingsJson)) + return await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is { } row && row.SettingsVersion == settingsVersion + ? new TranslatedSettings(row, settingsJson) + : null; + if (await LocalAsync(cancellationToken).ConfigureAwait(false) is not { } local || local.Row.SettingsVersion != settingsVersion) return null; + if (await ServerAsync(local.Frames, cancellationToken).ConfigureAwait(false) is not { } server) return null; + return new TranslatedSettings(local.Row, ProjectSettings.TranslateFramePositions(settingsJson!, server, local.Frames) ?? settingsJson); } - /// Settings of this copy with their positions on the server's frames. - public async Task ToServerAsync(string? settingsJson, CancellationToken cancellationToken) - { - if (!ProjectSettings.NamesFrames(settingsJson)) return settingsJson; - var local = await LocalAsync(cancellationToken).ConfigureAwait(false); - var server = await ServerAsync(local, cancellationToken).ConfigureAwait(false); - return ProjectSettings.TranslateFramePositions(settingsJson!, local, server) ?? settingsJson; - } - - /// This copy's frames, read each time (an import may insert some between a comparison and a write). - private async Task LocalAsync(CancellationToken cancellationToken) => - new((await SequenceFrames.ListAsync(db, localId, cancellationToken).ConfigureAwait(false)) - .Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); - /// - /// The server's frames from the manifest; their capture times are fetched only when a frame of either side is not - /// hashed, since such a frame is matched by size and capture time. + /// This copy's settings at with their positions on the server's frames, with the + /// row they were taken from: frames inserted here since the copy was read moved its positions without a new settings + /// version, so the row is read again with its frames. Null when the row is at another settings version (an edit was + /// saved since), or either copy's frames cannot be read as one state now. /// - private async Task ServerAsync(FramePhotos local, CancellationToken cancellationToken) + public async Task ToServerAsync(int settingsVersion, CancellationToken cancellationToken) { - if (_server is not null) return _server; - IReadOnlyDictionary captured = new Dictionary(); - if (local.HasUnhashed || manifest.Frames.Any(f => f.Sha256.Length == 0)) - captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt); - return _server = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, captured.GetValueOrDefault(f.Id))))); + if (await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is not { } row || row.SettingsVersion != settingsVersion) return null; + // Positions cannot appear in a project that names no frame: an insertion only moves those it has. + if (!ProjectSettings.NamesFrames(row.SettingsJson)) return new TranslatedSettings(row, row.SettingsJson); + if (await LocalAsync(cancellationToken).ConfigureAwait(false) is not { } local || local.Row.SettingsVersion != settingsVersion) return null; + if (await ServerAsync(local.Frames, cancellationToken).ConfigureAwait(false) is not { } server) return null; + var settings = local.Row.SettingsJson; + return new TranslatedSettings(local.Row, settings is null ? null : ProjectSettings.TranslateFramePositions(settings, local.Frames, server) ?? settings); } + + /// + /// This copy's row and frames as one state: the frames are listed between two reads of the row that find the same + /// frames version (an insertion renumbers the frames and moves the row's positions in one transaction, and bumps the + /// version). Null when the sequence is gone or an insertion lands during every attempt. + /// + private async Task LocalAsync(CancellationToken cancellationToken) + { + for (var attempt = 0; attempt < Attempts; attempt++) + { + if (await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is not { } before) return null; + var frames = await SequenceFrames.ListAsync(db, localId, cancellationToken).ConfigureAwait(false); + if (await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is not { } after) return null; + if (after.FramesVersion == before.FramesVersion) + return new LocalState(after, new FramePhotos(frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))))); + } + return null; + } + + /// + /// The server's frames from the manifest; null when they do not read as one frame list (an index given twice, or + /// not the indices 0…n-1: pages read across an insertion). Their capture times are fetched unless both copies hold + /// the same hashed photos at the same places: a frame is matched by size and capture time when either side has not + /// hashed it, and a position parked on a server photo always names its capture time, so the same project reads the + /// same whichever side hashed what. + /// + private async Task ServerAsync(FramePhotos local, CancellationToken cancellationToken) + { + if (_hashed is null) + { + if (!OneList(manifest.Frames)) return null; + _hashed = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, null)))); + } + if (_hashed.SameAs(local)) return _hashed; + if (_timed is null) + { + var captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt); + _timed = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, captured.GetValueOrDefault(f.Id))))); + } + return _timed; + } + + /// Each frame once, at the indices 0…n-1 a sequence's frames always take. + private static bool OneList(IReadOnlyList frames) + { + var seen = new bool[frames.Count]; + var ids = new HashSet(); + foreach (var frame in frames) + { + if ((uint)frame.Index >= (uint)seen.Length || seen[frame.Index] || !ids.Add(frame.Id)) return false; + seen[frame.Index] = true; + } + return true; + } + + private sealed record LocalState(Sequence Row, FramePhotos Frames); } + +/// +/// Settings translated by , with this copy's row at the frames their positions were +/// translated to or from (its own settings may have been moved since it was first read). +/// +internal sealed record TranslatedSettings(Sequence Local, string? SettingsJson); diff --git a/src/Titano.Data/Sync/SyncRemote.cs b/src/Titano.Data/Sync/SyncRemote.cs index 5949f116..4593a684 100644 --- a/src/Titano.Data/Sync/SyncRemote.cs +++ b/src/Titano.Data/Sync/SyncRemote.cs @@ -20,7 +20,7 @@ public interface ISyncRemote : IDisposable Task GetManifestAsync(Guid id, CancellationToken cancellationToken); - /// Every frame row of a sequence, in index order. + /// Every frame row of a sequence, each once, in index order. Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken); /// The first frame row of a sequence (its thumbnail stands for a sequence without a hero frame); null when it has none yet. @@ -101,11 +101,13 @@ public sealed class TitanoSyncRemote(TitanoClient client, bool ownsClient) : ISy public async Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken) { var all = new List(); + // Pages follow the frame index: frames inserted while they are read list a frame again, which is kept once. + var seen = new HashSet(); string? cursor = null; do { var page = await client.ListFramesAsync(sequenceId, cursor, FramePage, cancellationToken).ConfigureAwait(false); - all.AddRange(page.Items); + all.AddRange(page.Items.Where(f => seen.Add(f.Id))); cursor = page.NextCursor; } while (cursor is not null); diff --git a/src/Titano.Data/Sync/SyncRules.cs b/src/Titano.Data/Sync/SyncRules.cs index 8ab8bc42..10e31ed6 100644 --- a/src/Titano.Data/Sync/SyncRules.cs +++ b/src/Titano.Data/Sync/SyncRules.cs @@ -132,8 +132,10 @@ public static class SyncRules /// /// Originals missing on each side by content (BRIEF §14.5: originals never conflict): the same SHA-256 where both /// sides hashed them, else the same size and capture time (; a folder adopted on - /// either side and not verified yet). Local frames not hashed yet that nothing matched are assumed to match server - /// frames that nothing matched. + /// either side and not verified yet). A photo is there when any photo of the other side matches it, as downloads and + /// uploads find them (): two copies of one photo in a folder the server adopted are + /// both held by the one original this PC keeps. Local frames not hashed yet that nothing matched are assumed to match + /// server frames that nothing matched. /// public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection local, IReadOnlyCollection remote) { @@ -144,39 +146,15 @@ public static class SyncRules // What no hash matched: each content once, each frame not hashed on its own. var localLeft = Unmatched(local, remoteHashes); var remoteLeft = Unmatched(remote, localHashes); - var byTime = new Dictionary<(long Size, long Time), List>(); - for (var i = 0; i < remoteLeft.Count; i++) + var localPhotos = new FramePhotos(localLeft.Select((p, i) => (i, p))); + var remotePhotos = new FramePhotos(remoteLeft.Select((p, i) => (i, p))); + var remoteUnmatched = remoteLeft.Count(p => localPhotos.IndexOf(p) is null); + int localHashedLeft = 0, localUnhashedLeft = 0; + foreach (var photo in localLeft.Where(p => remotePhotos.IndexOf(p) is null)) { - if (remoteLeft[i].CapturedAt is not { } time) continue; - var key = (remoteLeft[i].Size, time.Ticks / TimeSpan.TicksPerMillisecond); - if (!byTime.TryGetValue(key, out var list)) byTime[key] = list = []; - list.Add(i); + if (photo.IsHashed) localHashedLeft++; + else localUnhashedLeft++; } - var paired = new bool[remoteLeft.Count]; - int localHashedLeft = 0, localUnhashedLeft = 0, remoteMatched = 0; - foreach (var photo in localLeft) - { - var match = -1; - if (photo.CapturedAt is { } time && byTime.TryGetValue((photo.Size, time.Ticks / TimeSpan.TicksPerMillisecond), out var candidates)) - { - var at = candidates.FindIndex(i => !paired[i] && remoteLeft[i].Matches(photo)); - if (at >= 0) match = candidates[at]; - } - if (match >= 0) - { - paired[match] = true; - remoteMatched++; - } - else if (photo.IsHashed) - { - localHashedLeft++; - } - else - { - localUnhashedLeft++; - } - } - var remoteUnmatched = remoteLeft.Count - remoteMatched; return (localHashedLeft + Math.Max(0, localUnhashedLeft - remoteUnmatched), Math.Max(0, remoteUnmatched - localUnhashedLeft)); static List Unmatched(IEnumerable side, HashSet other) => diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index fe8bb97e..647d8085 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -197,9 +197,16 @@ public sealed class SyncService try { var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false); - var request = await RequestAsync(remote, local, manifest, seenRemote, cancellationToken).ConfigureAwait(false); - var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false); - link = Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion); + // Edited here since it was shown, or frames that kept changing while they were read: compared again. + if (await RequestAsync(remote, local, manifest, seenRemote, cancellationToken).ConfigureAwait(false) is not { } request) + { + changed = true; + } + else + { + var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false); + link = Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion); + } } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict) { @@ -349,7 +356,13 @@ public sealed class SyncService var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false); try { - var request = await RequestAsync(remote, local, manifest, manifest.SettingsVersion, cancellationToken).ConfigureAwait(false); + if (await RequestAsync(remote, local, manifest, manifest.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } request) + { + // An edit saved here meanwhile goes instead; frames that keep changing here leave it to the next comparison. + if (attempt >= 3 || await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) is not { } fresh) return; + local = fresh; + continue; + } var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false); await _links.UpsertAsync(Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion), cancellationToken) .ConfigureAwait(false); @@ -569,9 +582,13 @@ public sealed class SyncService manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); // The listing and the manifest disagree: the server changed during the comparison; settle it next time. if (manifest.SettingsVersion != dto.SettingsVersion) return (link, local); + // Compared (and shown) with its positions on this copy's frames, against this copy as it is at those frames: the + // same photos are the same settings. Edited here meanwhile, or frames that kept changing: compared next time. + if (await Positions(remote, local, manifest).ToLocalAsync(manifest.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false) + is not { } translated) return (link, local); + local = translated.Local; var mine = SyncPayload.Of(local); - // Compared (and shown) with its positions on this copy's frames: the same photos are the same settings. - var theirs = SyncPayload.Of(dto, await Positions(remote, local, manifest).ToLocalAsync(manifest.SettingsJson, cancellationToken).ConfigureAwait(false)); + var theirs = SyncPayload.Of(dto, translated.SettingsJson); if (mine.SameAs(theirs)) return (Agreed(link, local.SettingsVersion, manifest.SettingsVersion), local); // Last writer wins only when one side changed; both changed differently is the user's choice (BRIEF §14.5). return (link with @@ -602,14 +619,14 @@ public sealed class SyncService /// /// Sends this PC's payload unless the server moved past , which shows up as a conflict next, - /// or its frames changed since (the positions are translated again next time). + /// or either copy's frames changed since they were read (the positions are translated again next time). /// private async Task PushAsync(ISyncRemote remote, SyncLink link, Sequence local, SequenceManifest manifest, int? expected, CancellationToken cancellationToken) { try { - var request = await RequestAsync(remote, local, manifest, expected, cancellationToken).ConfigureAwait(false); + if (await RequestAsync(remote, local, manifest, expected, cancellationToken).ConfigureAwait(false) is not { } request) return link; var updated = await remote.UpdateSequenceAsync(link.RemoteId, request, cancellationToken).ConfigureAwait(false); return Agreed(Observe(link, link.Server, updated), local.SettingsVersion, updated.SettingsVersion); } @@ -620,16 +637,18 @@ public sealed class SyncService } /// - /// The PATCH that makes the server's copy equal to , its frame positions moved to the server's - /// frames as lists them; the server refuses it when its frames changed since. + /// The PATCH that makes the server's copy equal to this copy at 's settings version (read again + /// with its frames, which may have moved its positions since), its frame positions moved to the server's frames as + /// lists them; the server refuses it when its frames changed since. Null when this copy + /// was edited since was read, or either copy's frames cannot be read as one state now. /// - private async Task RequestAsync(ISyncRemote remote, Sequence local, SequenceManifest manifest, int? expected, + private async Task RequestAsync(ISyncRemote remote, Sequence local, SequenceManifest manifest, int? expected, CancellationToken cancellationToken) { var positions = Positions(remote, local, manifest); - var settings = await positions.ToServerAsync(local.SettingsJson, cancellationToken).ConfigureAwait(false); - var frames = ProjectSettings.NamesFrames(local.SettingsJson) ? positions.ServerFrames : (int?)null; - return (SyncPayload.Of(local) with { SettingsJson = settings }).ToRequest(expected, frames); + if (await positions.ToServerAsync(local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } translated) return null; + var frames = ProjectSettings.NamesFrames(translated.Local.SettingsJson) ? positions.ServerFrames : (int?)null; + return (SyncPayload.Of(translated.Local) with { SettingsJson = translated.SettingsJson }).ToRequest(expected, frames); } private SyncPositions Positions(ISyncRemote remote, Sequence local, SequenceManifest manifest) => new(_db, remote, local.Id, manifest); @@ -695,19 +714,17 @@ public sealed class SyncService /// /// Writes the server's payload into the local index, project and sidecar in one step (settings version + 1) under the - /// sequence's lease, which the editor's saves take too, and only while the local copy is still at the settings version - /// was read at. The settings' frame positions are moved to this copy's frames as they are - /// under the lease (). Null when an edit was saved on this PC since: nothing is written - /// then. A payload the local copy already holds writes nothing, so it needs neither a writable archive nor the lease. + /// sequence's lease, which the editor's saves and the archive's frame commits take too, and only while the local copy + /// is still at the settings version was read at. The settings' frame positions are moved to + /// this copy's frames as they are under the lease (). Null when an edit was saved on this + /// PC since, or the frames of either copy could not be read as one state: nothing is written then. A payload the local + /// copy already holds writes nothing, so it needs neither a writable archive nor the lease. /// private async Task ApplyLocallyAsync(Sequence local, SyncPayload payload, SyncPositions positions, CancellationToken cancellationToken) { - var stored = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) - ?? throw new InvalidDataException($"Sequence {local.Id} is not indexed."); - if (stored.SettingsVersion != local.SettingsVersion) return null; + if (await positions.ToLocalAsync(payload.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } stored) return null; // Settled as it is: an archive that is offline or read-only does not keep the link from being agreed. - var mapped = payload with { SettingsJson = await positions.ToLocalAsync(payload.SettingsJson, cancellationToken).ConfigureAwait(false) }; - if (mapped.SameAs(SyncPayload.Of(stored))) return stored; + if ((payload with { SettingsJson = stored.SettingsJson }).SameAs(SyncPayload.Of(stored.Local))) return stored.Local; var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false) ?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed."); @@ -716,11 +733,10 @@ public sealed class SyncService var folder = paths.Sequence(local.Folder); await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false)) { - var row = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) - ?? throw new InvalidDataException($"Sequence {local.Id} is not indexed."); - if (row.SettingsVersion != local.SettingsVersion) return null; // The frames cannot change under the lease: the positions are moved to them as they are now. - payload = payload with { SettingsJson = await positions.ToLocalAsync(payload.SettingsJson, cancellationToken).ConfigureAwait(false) }; + if (await positions.ToLocalAsync(payload.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } current) return null; + var row = current.Local; + payload = payload with { SettingsJson = current.SettingsJson }; var metadata = !payload.SameMetadata(SyncPayload.Of(row)); var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson); if (!metadata && !settings) return row; diff --git a/src/Titano.Server/Api/SequenceEndpoints.cs b/src/Titano.Server/Api/SequenceEndpoints.cs index f7a375ba..6d6983c3 100644 --- a/src/Titano.Server/Api/SequenceEndpoints.cs +++ b/src/Titano.Server/Api/SequenceEndpoints.cs @@ -27,6 +27,9 @@ public static class SequenceEndpoints private const int MaxNotes = 2000; private const int MaxTags = 50; + /// Listings of a sequence's frames read without a lock before one is read under its row lock (). + private const int SnapshotAttempts = 3; + public static void Map(RouteGroupBuilder api) { // Uploaders list sequences to pick an upload target. @@ -220,14 +223,16 @@ public static class SequenceEndpoints /// /// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5), with the proxies the server asks - /// for (); ETag over the version counters and the number of proxies asked for. + /// for (); ETag over the version counters and the number of proxies asked for. The frames, + /// the frames version and the settings whose positions name those frames are of one frames version (). /// private static async Task ManifestAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive) { var ct = context.RequestAborted; - if (await db.Sequences.GetAsync(id, ct) is not { } sequence) return ApiErrors.NotFound(context, "sequence"); var spec = ServerProxies.Spec(archive.Current); - var frames = await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, ct); + if (await SnapshotAsync(db, id, (on, s, c) => ServerProxies.FramesAsync(on, archive.Current, s, spec, c), ct) is not { } snapshot) + return ApiErrors.NotFound(context, "sequence"); + var (sequence, frames) = snapshot; var etag = string.Create(CultureInfo.InvariantCulture, $"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}-{frames.Count(f => f.WantsProxy)}\""); if (context.Request.Headers.IfNoneMatch == etag) return Results.StatusCode(StatusCodes.Status304NotModified); @@ -238,6 +243,30 @@ public static class SequenceEndpoints [.. renders.Select(Dto.Render)], spec), ProtocolJson.Default.SequenceManifest); } + /// + /// The sequence and its frames as of one frames version: a client pairs the settings' frame positions with these + /// frames. reads them in pages, and a commit of earlier photos renumbers them and moves + /// the settings' positions at once, so a listing read across one would mix two orders. It is read again while the + /// frames version moved during it, and then inside a transaction holding the sequence's row, which every commit of + /// frames locks first (on SQLite, the write lock: the listing runs on that transaction's own connection). Null when + /// the sequence is gone. + /// + internal static async Task<(Sequence Sequence, List Frames)?> SnapshotAsync(TitanoDatabase db, Guid id, + Func>> listFrames, CancellationToken cancellationToken) + { + for (var attempt = 0; attempt < SnapshotAttempts; attempt++) + { + if (await db.Sequences.GetAsync(id, cancellationToken) is not { } sequence) return null; + var frames = await listFrames(db, sequence, cancellationToken); + if (await db.Sequences.GetAsync(id, cancellationToken) is not { } after) return null; + if (after.FramesVersion == sequence.FramesVersion) return (sequence, frames); + } + // Commits keep landing (an upload of earlier photos): the frames are listed while none can. + await using var tx = await db.BeginAsync(cancellationToken); + if (await tx.Sequences.GetForUpdateAsync(id, cancellationToken) is not { } locked) return null; + return (locked, await listFrames(tx, locked, cancellationToken)); + } + private static async Task FramesAsync(HttpContext context, Guid id, string? cursor, int? limit, TitanoDatabase db) { var ct = context.RequestAborted; diff --git a/src/Titano.Server/Storage/ServerProxies.cs b/src/Titano.Server/Storage/ServerProxies.cs index 34e73cc3..308d340b 100644 --- a/src/Titano.Server/Storage/ServerProxies.cs +++ b/src/Titano.Server/Storage/ServerProxies.cs @@ -38,7 +38,7 @@ public static class ServerProxies /// built them all or one was sent for the frame; it is wanted for a frame the server cannot decode that has none /// (and is not missing), unless is null. /// - public static async Task> FramesAsync(TitanoDatabase db, Archive archive, Sequence sequence, ProxySpec? spec, + public static async Task> FramesAsync(TitanoRepositories db, Archive archive, Sequence sequence, ProxySpec? spec, CancellationToken cancellationToken) { var proxies = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt); diff --git a/tests/Titano.Data.Tests/Sync/SyncFixture.cs b/tests/Titano.Data.Tests/Sync/SyncFixture.cs index 0e10ea5a..bafc9206 100644 --- a/tests/Titano.Data.Tests/Sync/SyncFixture.cs +++ b/tests/Titano.Data.Tests/Sync/SyncFixture.cs @@ -74,6 +74,18 @@ internal sealed class SyncFixture : IAsyncDisposable return sequence; } + /// Files one more original into a local sequence through the archive ingest (an earlier photo lands before the others). + public async Task AddLocalAsync(Guid sequenceId, byte[] original) + { + var staging = Path.Combine(Archive.Paths.StagingDirectory, "test"); + Directory.CreateDirectory(staging); + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part"); + await File.WriteAllBytesAsync(path, original, Ct); + var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(original)); + await Archive.Service.Ingest.CommitAsync(new IngestRequest(Archive.Archive.Id, sequenceId, null, null, path, $"IMG_{sha[..8]}.JPG", original.Length, sha, + null, null), Ct); + } + /// An edit on this PC, as the editor makes it (project file and index, settings version + 1). public async Task EditLocalSettingsAsync(Guid id, string settingsJson) { diff --git a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs index 672194b3..d57dea1f 100644 --- a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs @@ -79,6 +79,11 @@ public sealed class SyncRulesTests Assert.Equal((0, 3), SyncRules.CompareOriginals(Array.Empty(), ["", "", ""])); // Hashed on both sides, the hash decides even when size and time agree. Assert.Equal((1, 1), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("cc", 100, 0)])); + // A photo the adopted folder holds twice ('IMG_0001.JPG' and 'IMG_0001 (1).JPG') is here in this PC's one copy, + // as the download finds it: nothing is left to fetch. + Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("", 100, 0), Photo("", 100, 0)])); + Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo(null, 100, 0)], [Photo("", 100, 0), Photo("", 100, 0)])); + Assert.Equal((0, 1), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("", 100, 0), Photo("", 100, 0), Photo("", 100, 10)])); } [Test] diff --git a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs index b976b40d..9d090650 100644 --- a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs @@ -144,6 +144,53 @@ public sealed class SyncServiceTests(TestContext context) Assert.Equal((SyncStatus.ServerAhead, 1), ((await f.EntryAsync("Harbour")).Status, (await f.EntryAsync("Harbour")).Link!.MissingLocal)); } + [Test] + public async Task SettingsSentWhileAnImportHereMovesTheirPositionsStayOnTheirPhotos() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(4); + var local = await f.LocalAsync("Harbour", photos[1..]); + var remote = f.Remote.Add("Harbour", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + await f.EditLocalSettingsAsync(local.Id, new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson()); + + // The earliest photo is imported here after the comparison read this copy, before its settings are translated. + var racing = new RacingRemote(f.Remote, () => f.AddLocalAsync(local.Id, photos[0]).GetAwaiter().GetResult(), beforeManifest: true); + await f.Sync.RefreshAsync(racing, new SyncPolicy(), f.Ct); + + var mine = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!); + Assert.SequenceEqual([2], mine.ExcludedFrames); + Assert.Equal(3, mine.HeroFrame); + var sent = ProjectSettings.FromJson(remote.SettingsJson!); + Assert.SequenceEqual([Sha(photos[2])], sent.ExcludedFrames.Select(i => remote.Frames[i].Sha256)); + Assert.Equal(Sha(photos[3]), remote.Frames[sent.HeroFrame!.Value].Sha256); + Assert.Null(sent.PendingFrames); + } + + [Test] + public async Task SettingsComparedWhileAnImportHereMovesTheirPositionsAreAgreed() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(4); + var local = await f.LocalAsync("Harbour", photos[1..]); + var remote = f.Remote.Add("Harbour", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + // Both sides exclude the same photo, each by its own frames. + await f.EditLocalSettingsAsync(local.Id, new ProjectSettings { ExcludedFrames = [1] }.ToJson()); + f.Remote.EditSettings(remote.Id, new ProjectSettings { ExcludedFrames = [2] }.ToJson()); + + var racing = new RacingRemote(f.Remote, () => f.AddLocalAsync(local.Id, photos[0]).GetAwaiter().GetResult(), beforeManifest: true); + await f.Sync.RefreshAsync(racing, new SyncPolicy(), f.Ct); + + var entry = await f.EntryAsync("Harbour"); + Assert.Null(entry.Link!.ConflictRemote); + Assert.Equal(entry.Local!.SettingsVersion, entry.Link.BaseLocalSettings); + Assert.Equal(remote.SettingsVersion, entry.Link.BaseRemoteSettings); + Assert.SequenceEqual([2], ProjectSettings.FromJson(entry.Local.SettingsJson!).ExcludedFrames); + } + [Test] public async Task TheSameSettingsFromTheServerAreAgreedWhileTheArchiveIsOffline() { diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs index 921834ba..61984077 100644 --- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs @@ -366,6 +366,9 @@ public sealed class SyncTransferTests(TestContext context) Assert.Null(taken.Hero); Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes); Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them"); + // Each waits with its capture time too, whichever side hashed what: the same project reads the same either way. + var waiting = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).PendingFrames!; + Assert.True(waiting.All(p => p.CapturedAt is not null), "a photo waits with its capture time"); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); diff --git a/tests/Titano.Server.Tests/Api/SequenceTests.cs b/tests/Titano.Server.Tests/Api/SequenceTests.cs index 3d32d116..31226433 100644 --- a/tests/Titano.Server.Tests/Api/SequenceTests.cs +++ b/tests/Titano.Server.Tests/Api/SequenceTests.cs @@ -1,7 +1,9 @@ using System.Net; using System.Net.Http.Headers; using System.Net.Http.Json; +using Titano.Data; using Titano.Protocol; +using Titano.Server.Api; using Titano.Testing; namespace Titano.Server.Tests.Api; @@ -114,6 +116,59 @@ public sealed class SequenceTests(TestContext context) Assert.Count(3, (await changed.ReadAsync(ProtocolJson.Default.SequenceManifest)).Frames); } + [Test] + public async Task TheManifestListsTheFramesOfOneFramesVersion() + { + await using var server = await TestServer.StartAsync(context); + await FramesOfOneVersionAsync(server); + } + + /// The manifest's frames are read again while frames are committed, then under the row lock (also run on PostgreSQL). + internal static async Task FramesOfOneVersionAsync(TestServer server) + { + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var first = await Uploads.UploadAsync(editor, Uploads.Photo(2000, 51), "a.CR3", sessionKey: "k"); + var db = server.Database; + var id = first.SequenceId; + var start = (await db.Sequences.GetAsync(id))!.FramesVersion; + + // Frames are committed while the first listing is read: it is read again. + var listings = new List(); + var snapshot = await SequenceEndpoints.SnapshotAsync(db, id, async (_, sequence, ct) => + { + listings.Add(sequence.FramesVersion); + if (listings.Count == 1) await db.Sequences.RefreshStatsAsync(id, ct); + return []; + }, CancellationToken.None); + Assert.SequenceEqual([start, start + 1], listings); + Assert.Equal(start + 1, Assert.NotNull(snapshot).Sequence.FramesVersion); + + // Commits keep landing: the last listing is read while none can. + Task? waiting = null; + listings.Clear(); + var locked = await SequenceEndpoints.SnapshotAsync(db, id, async (on, sequence, ct) => + { + listings.Add(sequence.FramesVersion); + if (listings.Count <= 3) + { + await db.Sequences.RefreshStatsAsync(id, ct); + return []; + } + // Listed on the transaction that holds the row (on SQLite, the only connection that may while it writes). + Assert.True(on is TitanoTransaction, "the frames are listed inside the locking transaction"); + Assert.Count(1, await on.Frames.ListBySequenceAsync(id, cancellationToken: ct)); + waiting = db.Sequences.RefreshStatsAsync(id, ct); + await Task.WhenAny(waiting, Task.Delay(300, ct)); + Assert.False(waiting.IsCompleted, "a commit of frames waits until the frames are listed"); + return []; + }, CancellationToken.None); + await Assert.NotNull(waiting); + Assert.SequenceEqual([start + 1, start + 2, start + 3, start + 4], listings); + Assert.Equal(start + 4, Assert.NotNull(locked).Sequence.FramesVersion); + Assert.Equal(start + 5, (await db.Sequences.GetAsync(id))!.FramesVersion); + Assert.Null(await SequenceEndpoints.SnapshotAsync(db, Guid.NewGuid(), (_, _, _) => Task.FromResult(new List()), CancellationToken.None)); + } + [Test] public async Task ListingFiltersAndPagesWithACursor() { diff --git a/tests/Titano.Server.Tests/Postgres/PostgresServerTests.cs b/tests/Titano.Server.Tests/Postgres/PostgresServerTests.cs index 511be137..53b9f73e 100644 --- a/tests/Titano.Server.Tests/Postgres/PostgresServerTests.cs +++ b/tests/Titano.Server.Tests/Postgres/PostgresServerTests.cs @@ -54,6 +54,19 @@ public sealed class PostgresServerTests(TestContext context) Assert.Equal("ok", (await health.ReadAsync(ProtocolJson.Default.HealthStatus)).Checks["database"]); } + [Test] + public async Task TheManifestListsTheFramesOfOneFramesVersionOnPostgres() + { + await using var schema = await Schema.CreateAsync(context); + await using var server = await TestServer.StartAsync(context, o => + { + o.Database = DatabaseKind.Postgres; + o.DbConnectionString = schema.ConnectionString; + }); + Assert.Equal(Dialect.Postgres, server.Database.Dialect); + await Api.SequenceTests.FramesOfOneVersionAsync(server); + } + /// A throw-away schema on the test server, selected through Search Path. private sealed class Schema : IAsyncDisposable {