diff --git a/src/Titano.Data/Sync/SyncPositions.cs b/src/Titano.Data/Sync/SyncPositions.cs
index 00f746be..20c53e2f 100644
--- a/src/Titano.Data/Sync/SyncPositions.cs
+++ b/src/Titano.Data/Sync/SyncPositions.cs
@@ -1,3 +1,4 @@
+using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Protocol;
using SequenceFrames = Titano.Data.Processing.SequenceFrames;
@@ -11,47 +12,113 @@ namespace Titano.Data.Sync;
/// copy's frames before they are compared or applied, and this copy's to the server's frames as
/// lists them before they are sent. A position on a photo the receiving copy lacks waits
/// there as a pending frame and takes its place when the photo arrives ().
+/// Positions are only ever translated against the frames they name: this copy's row is read with its frames as one
+/// state (frames inserted here move the row's positions in the same step), and a manifest whose frames mix two orders
+/// is not translated at all.
///
internal sealed class SyncPositions(TitanoDatabase db, ISyncRemote remote, Guid localId, SequenceManifest manifest)
{
- private FramePhotos? _server;
+ /// Times this copy's frames are listed again when an insertion lands while they are read.
+ private const int Attempts = 3;
+
+ /// The manifest's frames by hash, and with their capture times once fetched.
+ private FramePhotos? _hashed, _timed;
/// The server's frames version the translation to the server's frames is made against.
public int ServerFrames => manifest.FramesVersion;
- /// Settings of the server sequence with their positions on this copy's frames, as they are now.
- public async Task ToLocalAsync(string? settingsJson, CancellationToken cancellationToken)
+ ///
+ /// The server's settings with their positions on this copy's frames as they are now, with this copy's row at those
+ /// frames; null when that row is no longer at (an edit was saved here since), or
+ /// either copy's frames cannot be read as one state now (the next comparison translates again).
+ ///
+ public async Task ToLocalAsync(string? settingsJson, int settingsVersion, CancellationToken cancellationToken)
{
- if (!ProjectSettings.NamesFrames(settingsJson)) return settingsJson;
- var local = await LocalAsync(cancellationToken).ConfigureAwait(false);
- var server = await ServerAsync(local, cancellationToken).ConfigureAwait(false);
- return ProjectSettings.TranslateFramePositions(settingsJson!, server, local) ?? settingsJson;
+ if (!ProjectSettings.NamesFrames(settingsJson))
+ return await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is { } row && row.SettingsVersion == settingsVersion
+ ? new TranslatedSettings(row, settingsJson)
+ : null;
+ if (await LocalAsync(cancellationToken).ConfigureAwait(false) is not { } local || local.Row.SettingsVersion != settingsVersion) return null;
+ if (await ServerAsync(local.Frames, cancellationToken).ConfigureAwait(false) is not { } server) return null;
+ return new TranslatedSettings(local.Row, ProjectSettings.TranslateFramePositions(settingsJson!, server, local.Frames) ?? settingsJson);
}
- /// Settings of this copy with their positions on the server's frames.
- public async Task ToServerAsync(string? settingsJson, CancellationToken cancellationToken)
- {
- if (!ProjectSettings.NamesFrames(settingsJson)) return settingsJson;
- var local = await LocalAsync(cancellationToken).ConfigureAwait(false);
- var server = await ServerAsync(local, cancellationToken).ConfigureAwait(false);
- return ProjectSettings.TranslateFramePositions(settingsJson!, local, server) ?? settingsJson;
- }
-
- /// This copy's frames, read each time (an import may insert some between a comparison and a write).
- private async Task LocalAsync(CancellationToken cancellationToken) =>
- new((await SequenceFrames.ListAsync(db, localId, cancellationToken).ConfigureAwait(false))
- .Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
-
///
- /// The server's frames from the manifest; their capture times are fetched only when a frame of either side is not
- /// hashed, since such a frame is matched by size and capture time.
+ /// This copy's settings at with their positions on the server's frames, with the
+ /// row they were taken from: frames inserted here since the copy was read moved its positions without a new settings
+ /// version, so the row is read again with its frames. Null when the row is at another settings version (an edit was
+ /// saved since), or either copy's frames cannot be read as one state now.
///
- private async Task ServerAsync(FramePhotos local, CancellationToken cancellationToken)
+ public async Task ToServerAsync(int settingsVersion, CancellationToken cancellationToken)
{
- if (_server is not null) return _server;
- IReadOnlyDictionary captured = new Dictionary();
- if (local.HasUnhashed || manifest.Frames.Any(f => f.Sha256.Length == 0))
- captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
- return _server = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, captured.GetValueOrDefault(f.Id)))));
+ if (await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is not { } row || row.SettingsVersion != settingsVersion) return null;
+ // Positions cannot appear in a project that names no frame: an insertion only moves those it has.
+ if (!ProjectSettings.NamesFrames(row.SettingsJson)) return new TranslatedSettings(row, row.SettingsJson);
+ if (await LocalAsync(cancellationToken).ConfigureAwait(false) is not { } local || local.Row.SettingsVersion != settingsVersion) return null;
+ if (await ServerAsync(local.Frames, cancellationToken).ConfigureAwait(false) is not { } server) return null;
+ var settings = local.Row.SettingsJson;
+ return new TranslatedSettings(local.Row, settings is null ? null : ProjectSettings.TranslateFramePositions(settings, local.Frames, server) ?? settings);
}
+
+ ///
+ /// This copy's row and frames as one state: the frames are listed between two reads of the row that find the same
+ /// frames version (an insertion renumbers the frames and moves the row's positions in one transaction, and bumps the
+ /// version). Null when the sequence is gone or an insertion lands during every attempt.
+ ///
+ private async Task LocalAsync(CancellationToken cancellationToken)
+ {
+ for (var attempt = 0; attempt < Attempts; attempt++)
+ {
+ if (await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is not { } before) return null;
+ var frames = await SequenceFrames.ListAsync(db, localId, cancellationToken).ConfigureAwait(false);
+ if (await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is not { } after) return null;
+ if (after.FramesVersion == before.FramesVersion)
+ return new LocalState(after, new FramePhotos(frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))));
+ }
+ return null;
+ }
+
+ ///
+ /// The server's frames from the manifest; null when they do not read as one frame list (an index given twice, or
+ /// not the indices 0…n-1: pages read across an insertion). Their capture times are fetched unless both copies hold
+ /// the same hashed photos at the same places: a frame is matched by size and capture time when either side has not
+ /// hashed it, and a position parked on a server photo always names its capture time, so the same project reads the
+ /// same whichever side hashed what.
+ ///
+ private async Task ServerAsync(FramePhotos local, CancellationToken cancellationToken)
+ {
+ if (_hashed is null)
+ {
+ if (!OneList(manifest.Frames)) return null;
+ _hashed = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, null))));
+ }
+ if (_hashed.SameAs(local)) return _hashed;
+ if (_timed is null)
+ {
+ var captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
+ _timed = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, captured.GetValueOrDefault(f.Id)))));
+ }
+ return _timed;
+ }
+
+ /// Each frame once, at the indices 0…n-1 a sequence's frames always take.
+ private static bool OneList(IReadOnlyList frames)
+ {
+ var seen = new bool[frames.Count];
+ var ids = new HashSet();
+ foreach (var frame in frames)
+ {
+ if ((uint)frame.Index >= (uint)seen.Length || seen[frame.Index] || !ids.Add(frame.Id)) return false;
+ seen[frame.Index] = true;
+ }
+ return true;
+ }
+
+ private sealed record LocalState(Sequence Row, FramePhotos Frames);
}
+
+///
+/// Settings translated by , with this copy's row at the frames their positions were
+/// translated to or from (its own settings may have been moved since it was first read).
+///
+internal sealed record TranslatedSettings(Sequence Local, string? SettingsJson);
diff --git a/src/Titano.Data/Sync/SyncRemote.cs b/src/Titano.Data/Sync/SyncRemote.cs
index 5949f116..4593a684 100644
--- a/src/Titano.Data/Sync/SyncRemote.cs
+++ b/src/Titano.Data/Sync/SyncRemote.cs
@@ -20,7 +20,7 @@ public interface ISyncRemote : IDisposable
Task GetManifestAsync(Guid id, CancellationToken cancellationToken);
- /// Every frame row of a sequence, in index order.
+ /// Every frame row of a sequence, each once, in index order.
Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken);
/// The first frame row of a sequence (its thumbnail stands for a sequence without a hero frame); null when it has none yet.
@@ -101,11 +101,13 @@ public sealed class TitanoSyncRemote(TitanoClient client, bool ownsClient) : ISy
public async Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken)
{
var all = new List();
+ // Pages follow the frame index: frames inserted while they are read list a frame again, which is kept once.
+ var seen = new HashSet();
string? cursor = null;
do
{
var page = await client.ListFramesAsync(sequenceId, cursor, FramePage, cancellationToken).ConfigureAwait(false);
- all.AddRange(page.Items);
+ all.AddRange(page.Items.Where(f => seen.Add(f.Id)));
cursor = page.NextCursor;
}
while (cursor is not null);
diff --git a/src/Titano.Data/Sync/SyncRules.cs b/src/Titano.Data/Sync/SyncRules.cs
index 8ab8bc42..10e31ed6 100644
--- a/src/Titano.Data/Sync/SyncRules.cs
+++ b/src/Titano.Data/Sync/SyncRules.cs
@@ -132,8 +132,10 @@ public static class SyncRules
///
/// Originals missing on each side by content (BRIEF §14.5: originals never conflict): the same SHA-256 where both
/// sides hashed them, else the same size and capture time (; a folder adopted on
- /// either side and not verified yet). Local frames not hashed yet that nothing matched are assumed to match server
- /// frames that nothing matched.
+ /// either side and not verified yet). A photo is there when any photo of the other side matches it, as downloads and
+ /// uploads find them (): two copies of one photo in a folder the server adopted are
+ /// both held by the one original this PC keeps. Local frames not hashed yet that nothing matched are assumed to match
+ /// server frames that nothing matched.
///
public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection local, IReadOnlyCollection remote)
{
@@ -144,39 +146,15 @@ public static class SyncRules
// What no hash matched: each content once, each frame not hashed on its own.
var localLeft = Unmatched(local, remoteHashes);
var remoteLeft = Unmatched(remote, localHashes);
- var byTime = new Dictionary<(long Size, long Time), List>();
- for (var i = 0; i < remoteLeft.Count; i++)
+ var localPhotos = new FramePhotos(localLeft.Select((p, i) => (i, p)));
+ var remotePhotos = new FramePhotos(remoteLeft.Select((p, i) => (i, p)));
+ var remoteUnmatched = remoteLeft.Count(p => localPhotos.IndexOf(p) is null);
+ int localHashedLeft = 0, localUnhashedLeft = 0;
+ foreach (var photo in localLeft.Where(p => remotePhotos.IndexOf(p) is null))
{
- if (remoteLeft[i].CapturedAt is not { } time) continue;
- var key = (remoteLeft[i].Size, time.Ticks / TimeSpan.TicksPerMillisecond);
- if (!byTime.TryGetValue(key, out var list)) byTime[key] = list = [];
- list.Add(i);
+ if (photo.IsHashed) localHashedLeft++;
+ else localUnhashedLeft++;
}
- var paired = new bool[remoteLeft.Count];
- int localHashedLeft = 0, localUnhashedLeft = 0, remoteMatched = 0;
- foreach (var photo in localLeft)
- {
- var match = -1;
- if (photo.CapturedAt is { } time && byTime.TryGetValue((photo.Size, time.Ticks / TimeSpan.TicksPerMillisecond), out var candidates))
- {
- var at = candidates.FindIndex(i => !paired[i] && remoteLeft[i].Matches(photo));
- if (at >= 0) match = candidates[at];
- }
- if (match >= 0)
- {
- paired[match] = true;
- remoteMatched++;
- }
- else if (photo.IsHashed)
- {
- localHashedLeft++;
- }
- else
- {
- localUnhashedLeft++;
- }
- }
- var remoteUnmatched = remoteLeft.Count - remoteMatched;
return (localHashedLeft + Math.Max(0, localUnhashedLeft - remoteUnmatched), Math.Max(0, remoteUnmatched - localUnhashedLeft));
static List Unmatched(IEnumerable side, HashSet other) =>
diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs
index fe8bb97e..647d8085 100644
--- a/src/Titano.Data/Sync/SyncService.cs
+++ b/src/Titano.Data/Sync/SyncService.cs
@@ -197,9 +197,16 @@ public sealed class SyncService
try
{
var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false);
- var request = await RequestAsync(remote, local, manifest, seenRemote, cancellationToken).ConfigureAwait(false);
- var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false);
- link = Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion);
+ // Edited here since it was shown, or frames that kept changing while they were read: compared again.
+ if (await RequestAsync(remote, local, manifest, seenRemote, cancellationToken).ConfigureAwait(false) is not { } request)
+ {
+ changed = true;
+ }
+ else
+ {
+ var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false);
+ link = Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion);
+ }
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict)
{
@@ -349,7 +356,13 @@ public sealed class SyncService
var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false);
try
{
- var request = await RequestAsync(remote, local, manifest, manifest.SettingsVersion, cancellationToken).ConfigureAwait(false);
+ if (await RequestAsync(remote, local, manifest, manifest.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } request)
+ {
+ // An edit saved here meanwhile goes instead; frames that keep changing here leave it to the next comparison.
+ if (attempt >= 3 || await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) is not { } fresh) return;
+ local = fresh;
+ continue;
+ }
var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false);
await _links.UpsertAsync(Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion), cancellationToken)
.ConfigureAwait(false);
@@ -569,9 +582,13 @@ public sealed class SyncService
manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
// The listing and the manifest disagree: the server changed during the comparison; settle it next time.
if (manifest.SettingsVersion != dto.SettingsVersion) return (link, local);
+ // Compared (and shown) with its positions on this copy's frames, against this copy as it is at those frames: the
+ // same photos are the same settings. Edited here meanwhile, or frames that kept changing: compared next time.
+ if (await Positions(remote, local, manifest).ToLocalAsync(manifest.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false)
+ is not { } translated) return (link, local);
+ local = translated.Local;
var mine = SyncPayload.Of(local);
- // Compared (and shown) with its positions on this copy's frames: the same photos are the same settings.
- var theirs = SyncPayload.Of(dto, await Positions(remote, local, manifest).ToLocalAsync(manifest.SettingsJson, cancellationToken).ConfigureAwait(false));
+ var theirs = SyncPayload.Of(dto, translated.SettingsJson);
if (mine.SameAs(theirs)) return (Agreed(link, local.SettingsVersion, manifest.SettingsVersion), local);
// Last writer wins only when one side changed; both changed differently is the user's choice (BRIEF §14.5).
return (link with
@@ -602,14 +619,14 @@ public sealed class SyncService
///
/// Sends this PC's payload unless the server moved past , which shows up as a conflict next,
- /// or its frames changed since (the positions are translated again next time).
+ /// or either copy's frames changed since they were read (the positions are translated again next time).
///
private async Task PushAsync(ISyncRemote remote, SyncLink link, Sequence local, SequenceManifest manifest, int? expected,
CancellationToken cancellationToken)
{
try
{
- var request = await RequestAsync(remote, local, manifest, expected, cancellationToken).ConfigureAwait(false);
+ if (await RequestAsync(remote, local, manifest, expected, cancellationToken).ConfigureAwait(false) is not { } request) return link;
var updated = await remote.UpdateSequenceAsync(link.RemoteId, request, cancellationToken).ConfigureAwait(false);
return Agreed(Observe(link, link.Server, updated), local.SettingsVersion, updated.SettingsVersion);
}
@@ -620,16 +637,18 @@ public sealed class SyncService
}
///
- /// The PATCH that makes the server's copy equal to , its frame positions moved to the server's
- /// frames as lists them; the server refuses it when its frames changed since.
+ /// The PATCH that makes the server's copy equal to this copy at 's settings version (read again
+ /// with its frames, which may have moved its positions since), its frame positions moved to the server's frames as
+ /// lists them; the server refuses it when its frames changed since. Null when this copy
+ /// was edited since was read, or either copy's frames cannot be read as one state now.
///
- private async Task RequestAsync(ISyncRemote remote, Sequence local, SequenceManifest manifest, int? expected,
+ private async Task RequestAsync(ISyncRemote remote, Sequence local, SequenceManifest manifest, int? expected,
CancellationToken cancellationToken)
{
var positions = Positions(remote, local, manifest);
- var settings = await positions.ToServerAsync(local.SettingsJson, cancellationToken).ConfigureAwait(false);
- var frames = ProjectSettings.NamesFrames(local.SettingsJson) ? positions.ServerFrames : (int?)null;
- return (SyncPayload.Of(local) with { SettingsJson = settings }).ToRequest(expected, frames);
+ if (await positions.ToServerAsync(local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } translated) return null;
+ var frames = ProjectSettings.NamesFrames(translated.Local.SettingsJson) ? positions.ServerFrames : (int?)null;
+ return (SyncPayload.Of(translated.Local) with { SettingsJson = translated.SettingsJson }).ToRequest(expected, frames);
}
private SyncPositions Positions(ISyncRemote remote, Sequence local, SequenceManifest manifest) => new(_db, remote, local.Id, manifest);
@@ -695,19 +714,17 @@ public sealed class SyncService
///
/// Writes the server's payload into the local index, project and sidecar in one step (settings version + 1) under the
- /// sequence's lease, which the editor's saves take too, and only while the local copy is still at the settings version
- /// was read at. The settings' frame positions are moved to this copy's frames as they are
- /// under the lease (). Null when an edit was saved on this PC since: nothing is written
- /// then. A payload the local copy already holds writes nothing, so it needs neither a writable archive nor the lease.
+ /// sequence's lease, which the editor's saves and the archive's frame commits take too, and only while the local copy
+ /// is still at the settings version was read at. The settings' frame positions are moved to
+ /// this copy's frames as they are under the lease (). Null when an edit was saved on this
+ /// PC since, or the frames of either copy could not be read as one state: nothing is written then. A payload the local
+ /// copy already holds writes nothing, so it needs neither a writable archive nor the lease.
///
private async Task ApplyLocallyAsync(Sequence local, SyncPayload payload, SyncPositions positions, CancellationToken cancellationToken)
{
- var stored = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false)
- ?? throw new InvalidDataException($"Sequence {local.Id} is not indexed.");
- if (stored.SettingsVersion != local.SettingsVersion) return null;
+ if (await positions.ToLocalAsync(payload.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } stored) return null;
// Settled as it is: an archive that is offline or read-only does not keep the link from being agreed.
- var mapped = payload with { SettingsJson = await positions.ToLocalAsync(payload.SettingsJson, cancellationToken).ConfigureAwait(false) };
- if (mapped.SameAs(SyncPayload.Of(stored))) return stored;
+ if ((payload with { SettingsJson = stored.SettingsJson }).SameAs(SyncPayload.Of(stored.Local))) return stored.Local;
var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed.");
@@ -716,11 +733,10 @@ public sealed class SyncService
var folder = paths.Sequence(local.Folder);
await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false))
{
- var row = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false)
- ?? throw new InvalidDataException($"Sequence {local.Id} is not indexed.");
- if (row.SettingsVersion != local.SettingsVersion) return null;
// The frames cannot change under the lease: the positions are moved to them as they are now.
- payload = payload with { SettingsJson = await positions.ToLocalAsync(payload.SettingsJson, cancellationToken).ConfigureAwait(false) };
+ if (await positions.ToLocalAsync(payload.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } current) return null;
+ var row = current.Local;
+ payload = payload with { SettingsJson = current.SettingsJson };
var metadata = !payload.SameMetadata(SyncPayload.Of(row));
var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson);
if (!metadata && !settings) return row;
diff --git a/src/Titano.Server/Api/SequenceEndpoints.cs b/src/Titano.Server/Api/SequenceEndpoints.cs
index f7a375ba..6d6983c3 100644
--- a/src/Titano.Server/Api/SequenceEndpoints.cs
+++ b/src/Titano.Server/Api/SequenceEndpoints.cs
@@ -27,6 +27,9 @@ public static class SequenceEndpoints
private const int MaxNotes = 2000;
private const int MaxTags = 50;
+ /// Listings of a sequence's frames read without a lock before one is read under its row lock ().
+ private const int SnapshotAttempts = 3;
+
public static void Map(RouteGroupBuilder api)
{
// Uploaders list sequences to pick an upload target.
@@ -220,14 +223,16 @@ public static class SequenceEndpoints
///
/// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5), with the proxies the server asks
- /// for (); ETag over the version counters and the number of proxies asked for.
+ /// for (); ETag over the version counters and the number of proxies asked for. The frames,
+ /// the frames version and the settings whose positions name those frames are of one frames version ().
///
private static async Task ManifestAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive)
{
var ct = context.RequestAborted;
- if (await db.Sequences.GetAsync(id, ct) is not { } sequence) return ApiErrors.NotFound(context, "sequence");
var spec = ServerProxies.Spec(archive.Current);
- var frames = await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, ct);
+ if (await SnapshotAsync(db, id, (on, s, c) => ServerProxies.FramesAsync(on, archive.Current, s, spec, c), ct) is not { } snapshot)
+ return ApiErrors.NotFound(context, "sequence");
+ var (sequence, frames) = snapshot;
var etag = string.Create(CultureInfo.InvariantCulture,
$"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}-{frames.Count(f => f.WantsProxy)}\"");
if (context.Request.Headers.IfNoneMatch == etag) return Results.StatusCode(StatusCodes.Status304NotModified);
@@ -238,6 +243,30 @@ public static class SequenceEndpoints
[.. renders.Select(Dto.Render)], spec), ProtocolJson.Default.SequenceManifest);
}
+ ///
+ /// The sequence and its frames as of one frames version: a client pairs the settings' frame positions with these
+ /// frames. reads them in pages, and a commit of earlier photos renumbers them and moves
+ /// the settings' positions at once, so a listing read across one would mix two orders. It is read again while the
+ /// frames version moved during it, and then inside a transaction holding the sequence's row, which every commit of
+ /// frames locks first (on SQLite, the write lock: the listing runs on that transaction's own connection). Null when
+ /// the sequence is gone.
+ ///
+ internal static async Task<(Sequence Sequence, List Frames)?> SnapshotAsync(TitanoDatabase db, Guid id,
+ Func>> listFrames, CancellationToken cancellationToken)
+ {
+ for (var attempt = 0; attempt < SnapshotAttempts; attempt++)
+ {
+ if (await db.Sequences.GetAsync(id, cancellationToken) is not { } sequence) return null;
+ var frames = await listFrames(db, sequence, cancellationToken);
+ if (await db.Sequences.GetAsync(id, cancellationToken) is not { } after) return null;
+ if (after.FramesVersion == sequence.FramesVersion) return (sequence, frames);
+ }
+ // Commits keep landing (an upload of earlier photos): the frames are listed while none can.
+ await using var tx = await db.BeginAsync(cancellationToken);
+ if (await tx.Sequences.GetForUpdateAsync(id, cancellationToken) is not { } locked) return null;
+ return (locked, await listFrames(tx, locked, cancellationToken));
+ }
+
private static async Task FramesAsync(HttpContext context, Guid id, string? cursor, int? limit, TitanoDatabase db)
{
var ct = context.RequestAborted;
diff --git a/src/Titano.Server/Storage/ServerProxies.cs b/src/Titano.Server/Storage/ServerProxies.cs
index 34e73cc3..308d340b 100644
--- a/src/Titano.Server/Storage/ServerProxies.cs
+++ b/src/Titano.Server/Storage/ServerProxies.cs
@@ -38,7 +38,7 @@ public static class ServerProxies
/// built them all or one was sent for the frame; it is wanted for a frame the server cannot decode that has none
/// (and is not missing), unless is null.
///
- public static async Task> FramesAsync(TitanoDatabase db, Archive archive, Sequence sequence, ProxySpec? spec,
+ public static async Task> FramesAsync(TitanoRepositories db, Archive archive, Sequence sequence, ProxySpec? spec,
CancellationToken cancellationToken)
{
var proxies = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
diff --git a/tests/Titano.Data.Tests/Sync/SyncFixture.cs b/tests/Titano.Data.Tests/Sync/SyncFixture.cs
index 0e10ea5a..bafc9206 100644
--- a/tests/Titano.Data.Tests/Sync/SyncFixture.cs
+++ b/tests/Titano.Data.Tests/Sync/SyncFixture.cs
@@ -74,6 +74,18 @@ internal sealed class SyncFixture : IAsyncDisposable
return sequence;
}
+ /// Files one more original into a local sequence through the archive ingest (an earlier photo lands before the others).
+ public async Task AddLocalAsync(Guid sequenceId, byte[] original)
+ {
+ var staging = Path.Combine(Archive.Paths.StagingDirectory, "test");
+ Directory.CreateDirectory(staging);
+ var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
+ await File.WriteAllBytesAsync(path, original, Ct);
+ var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(original));
+ await Archive.Service.Ingest.CommitAsync(new IngestRequest(Archive.Archive.Id, sequenceId, null, null, path, $"IMG_{sha[..8]}.JPG", original.Length, sha,
+ null, null), Ct);
+ }
+
/// An edit on this PC, as the editor makes it (project file and index, settings version + 1).
public async Task EditLocalSettingsAsync(Guid id, string settingsJson)
{
diff --git a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs
index 672194b3..d57dea1f 100644
--- a/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs
+++ b/tests/Titano.Data.Tests/Sync/SyncRulesTests.cs
@@ -79,6 +79,11 @@ public sealed class SyncRulesTests
Assert.Equal((0, 3), SyncRules.CompareOriginals(Array.Empty(), ["", "", ""]));
// Hashed on both sides, the hash decides even when size and time agree.
Assert.Equal((1, 1), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("cc", 100, 0)]));
+ // A photo the adopted folder holds twice ('IMG_0001.JPG' and 'IMG_0001 (1).JPG') is here in this PC's one copy,
+ // as the download finds it: nothing is left to fetch.
+ Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("", 100, 0), Photo("", 100, 0)]));
+ Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo(null, 100, 0)], [Photo("", 100, 0), Photo("", 100, 0)]));
+ Assert.Equal((0, 1), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("", 100, 0), Photo("", 100, 0), Photo("", 100, 10)]));
}
[Test]
diff --git a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs
index b976b40d..9d090650 100644
--- a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs
+++ b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs
@@ -144,6 +144,53 @@ public sealed class SyncServiceTests(TestContext context)
Assert.Equal((SyncStatus.ServerAhead, 1), ((await f.EntryAsync("Harbour")).Status, (await f.EntryAsync("Harbour")).Link!.MissingLocal));
}
+ [Test]
+ public async Task SettingsSentWhileAnImportHereMovesTheirPositionsStayOnTheirPhotos()
+ {
+ await using var f = await SyncFixture.CreateAsync(context);
+ var photos = f.NewPhotos(4);
+ var local = await f.LocalAsync("Harbour", photos[1..]);
+ var remote = f.Remote.Add("Harbour", photos);
+ await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
+ await f.RefreshAsync();
+ await f.EditLocalSettingsAsync(local.Id, new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson());
+
+ // The earliest photo is imported here after the comparison read this copy, before its settings are translated.
+ var racing = new RacingRemote(f.Remote, () => f.AddLocalAsync(local.Id, photos[0]).GetAwaiter().GetResult(), beforeManifest: true);
+ await f.Sync.RefreshAsync(racing, new SyncPolicy(), f.Ct);
+
+ var mine = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!);
+ Assert.SequenceEqual([2], mine.ExcludedFrames);
+ Assert.Equal(3, mine.HeroFrame);
+ var sent = ProjectSettings.FromJson(remote.SettingsJson!);
+ Assert.SequenceEqual([Sha(photos[2])], sent.ExcludedFrames.Select(i => remote.Frames[i].Sha256));
+ Assert.Equal(Sha(photos[3]), remote.Frames[sent.HeroFrame!.Value].Sha256);
+ Assert.Null(sent.PendingFrames);
+ }
+
+ [Test]
+ public async Task SettingsComparedWhileAnImportHereMovesTheirPositionsAreAgreed()
+ {
+ await using var f = await SyncFixture.CreateAsync(context);
+ var photos = f.NewPhotos(4);
+ var local = await f.LocalAsync("Harbour", photos[1..]);
+ var remote = f.Remote.Add("Harbour", photos);
+ await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
+ await f.RefreshAsync();
+ // Both sides exclude the same photo, each by its own frames.
+ await f.EditLocalSettingsAsync(local.Id, new ProjectSettings { ExcludedFrames = [1] }.ToJson());
+ f.Remote.EditSettings(remote.Id, new ProjectSettings { ExcludedFrames = [2] }.ToJson());
+
+ var racing = new RacingRemote(f.Remote, () => f.AddLocalAsync(local.Id, photos[0]).GetAwaiter().GetResult(), beforeManifest: true);
+ await f.Sync.RefreshAsync(racing, new SyncPolicy(), f.Ct);
+
+ var entry = await f.EntryAsync("Harbour");
+ Assert.Null(entry.Link!.ConflictRemote);
+ Assert.Equal(entry.Local!.SettingsVersion, entry.Link.BaseLocalSettings);
+ Assert.Equal(remote.SettingsVersion, entry.Link.BaseRemoteSettings);
+ Assert.SequenceEqual([2], ProjectSettings.FromJson(entry.Local.SettingsJson!).ExcludedFrames);
+ }
+
[Test]
public async Task TheSameSettingsFromTheServerAreAgreedWhileTheArchiveIsOffline()
{
diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
index 921834ba..61984077 100644
--- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
+++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
@@ -366,6 +366,9 @@ public sealed class SyncTransferTests(TestContext context)
Assert.Null(taken.Hero);
Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes);
Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them");
+ // Each waits with its capture time too, whichever side hashed what: the same project reads the same either way.
+ var waiting = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).PendingFrames!;
+ Assert.True(waiting.All(p => p.CapturedAt is not null), "a photo waits with its capture time");
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
diff --git a/tests/Titano.Server.Tests/Api/SequenceTests.cs b/tests/Titano.Server.Tests/Api/SequenceTests.cs
index 3d32d116..31226433 100644
--- a/tests/Titano.Server.Tests/Api/SequenceTests.cs
+++ b/tests/Titano.Server.Tests/Api/SequenceTests.cs
@@ -1,7 +1,9 @@
using System.Net;
using System.Net.Http.Headers;
using System.Net.Http.Json;
+using Titano.Data;
using Titano.Protocol;
+using Titano.Server.Api;
using Titano.Testing;
namespace Titano.Server.Tests.Api;
@@ -114,6 +116,59 @@ public sealed class SequenceTests(TestContext context)
Assert.Count(3, (await changed.ReadAsync(ProtocolJson.Default.SequenceManifest)).Frames);
}
+ [Test]
+ public async Task TheManifestListsTheFramesOfOneFramesVersion()
+ {
+ await using var server = await TestServer.StartAsync(context);
+ await FramesOfOneVersionAsync(server);
+ }
+
+ /// The manifest's frames are read again while frames are committed, then under the row lock (also run on PostgreSQL).
+ internal static async Task FramesOfOneVersionAsync(TestServer server)
+ {
+ var (editor, _, _) = await server.UserAsync("alberto", "editor");
+ var first = await Uploads.UploadAsync(editor, Uploads.Photo(2000, 51), "a.CR3", sessionKey: "k");
+ var db = server.Database;
+ var id = first.SequenceId;
+ var start = (await db.Sequences.GetAsync(id))!.FramesVersion;
+
+ // Frames are committed while the first listing is read: it is read again.
+ var listings = new List();
+ var snapshot = await SequenceEndpoints.SnapshotAsync(db, id, async (_, sequence, ct) =>
+ {
+ listings.Add(sequence.FramesVersion);
+ if (listings.Count == 1) await db.Sequences.RefreshStatsAsync(id, ct);
+ return [];
+ }, CancellationToken.None);
+ Assert.SequenceEqual([start, start + 1], listings);
+ Assert.Equal(start + 1, Assert.NotNull(snapshot).Sequence.FramesVersion);
+
+ // Commits keep landing: the last listing is read while none can.
+ Task? waiting = null;
+ listings.Clear();
+ var locked = await SequenceEndpoints.SnapshotAsync(db, id, async (on, sequence, ct) =>
+ {
+ listings.Add(sequence.FramesVersion);
+ if (listings.Count <= 3)
+ {
+ await db.Sequences.RefreshStatsAsync(id, ct);
+ return [];
+ }
+ // Listed on the transaction that holds the row (on SQLite, the only connection that may while it writes).
+ Assert.True(on is TitanoTransaction, "the frames are listed inside the locking transaction");
+ Assert.Count(1, await on.Frames.ListBySequenceAsync(id, cancellationToken: ct));
+ waiting = db.Sequences.RefreshStatsAsync(id, ct);
+ await Task.WhenAny(waiting, Task.Delay(300, ct));
+ Assert.False(waiting.IsCompleted, "a commit of frames waits until the frames are listed");
+ return [];
+ }, CancellationToken.None);
+ await Assert.NotNull(waiting);
+ Assert.SequenceEqual([start + 1, start + 2, start + 3, start + 4], listings);
+ Assert.Equal(start + 4, Assert.NotNull(locked).Sequence.FramesVersion);
+ Assert.Equal(start + 5, (await db.Sequences.GetAsync(id))!.FramesVersion);
+ Assert.Null(await SequenceEndpoints.SnapshotAsync(db, Guid.NewGuid(), (_, _, _) => Task.FromResult(new List()), CancellationToken.None));
+ }
+
[Test]
public async Task ListingFiltersAndPagesWithACursor()
{
diff --git a/tests/Titano.Server.Tests/Postgres/PostgresServerTests.cs b/tests/Titano.Server.Tests/Postgres/PostgresServerTests.cs
index 511be137..53b9f73e 100644
--- a/tests/Titano.Server.Tests/Postgres/PostgresServerTests.cs
+++ b/tests/Titano.Server.Tests/Postgres/PostgresServerTests.cs
@@ -54,6 +54,19 @@ public sealed class PostgresServerTests(TestContext context)
Assert.Equal("ok", (await health.ReadAsync(ProtocolJson.Default.HealthStatus)).Checks["database"]);
}
+ [Test]
+ public async Task TheManifestListsTheFramesOfOneFramesVersionOnPostgres()
+ {
+ await using var schema = await Schema.CreateAsync(context);
+ await using var server = await TestServer.StartAsync(context, o =>
+ {
+ o.Database = DatabaseKind.Postgres;
+ o.DbConnectionString = schema.ConnectionString;
+ });
+ Assert.Equal(Dialect.Postgres, server.Database.Dialect);
+ await Api.SequenceTests.FramesOfOneVersionAsync(server);
+ }
+
/// A throw-away schema on the test server, selected through Search Path.
private sealed class Schema : IAsyncDisposable
{