fix(sync): translate positions only against the frames they name
Positions crossing copies could still land on other photos in three windows: - A push translated a row snapshot's settings against local frames listed later, after a manifest round trip: an import of earlier photos meanwhile moved the row's positions, and the server stored shifted exclusions, hero and keyframes. SyncPositions now reads this copy's row with its frames as one state (frames listed between two reads of the row at the same frames version) and translates that row's settings, at the settings version being pushed. The conflict comparison and ApplyLocallyAsync compare against that same row. When the copy was edited since, or its frames keep changing, nothing is sent or applied and the next comparison translates again (Keep mine reports a changed conflict; PushAll sends an edit saved meanwhile). - The manifest read the sequence row, then paged its frames: an upload committed in between paired settings of one frames version with frames of the next. The server now lists them again while the frames version moved during the listing, then inside a transaction holding the row (on SQLite, on that transaction's connection, so no second pooled connection waits behind the write lock). The client also refuses to translate against a manifest whose frames are not each listed once at 0..n-1. - Server capture times were fetched only when something was unhashed, so the same parked photo was written with or without capturedAt. They are now fetched whenever the two frame lists differ; an equal project then compares equal, and an entry can be placed on a photo that arrives unhashed. CompareOriginals counted unhashed server frames one-to-one against this PC's photos, so two copies of one photo in a folder the server adopted left Download on offer forever while the download fetched nothing. A photo on either side is now there when any photo of the other side matches it, as downloads and uploads find them. Paged frame listings keep each frame once (an insertion between pages lists one again). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
12 files changed
+322
-95
No files matched your search
@@ -1,3 +1,4 @@
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Protocol;
|
||||
using SequenceFrames = Titano.Data.Processing.SequenceFrames;
|
||||
@@ -11,47 +12,113 @@ namespace Titano.Data.Sync;
|
||||
/// copy's frames before they are compared or applied, and this copy's to the server's frames as
|
||||
/// <paramref name="manifest"/> lists them before they are sent. A position on a photo the receiving copy lacks waits
|
||||
/// there as a pending frame and takes its place when the photo arrives (<see cref="ProjectSettings.TranslateFramePositions"/>).
|
||||
/// Positions are only ever translated against the frames they name: this copy's row is read with its frames as one
|
||||
/// state (frames inserted here move the row's positions in the same step), and a manifest whose frames mix two orders
|
||||
/// is not translated at all.
|
||||
/// </summary>
|
||||
internal sealed class SyncPositions(TitanoDatabase db, ISyncRemote remote, Guid localId, SequenceManifest manifest)
|
||||
{
|
||||
private FramePhotos? _server;
|
||||
/// <summary>Times this copy's frames are listed again when an insertion lands while they are read.</summary>
|
||||
private const int Attempts = 3;
|
||||
|
||||
/// <summary>The manifest's frames by hash, and with their capture times once fetched.</summary>
|
||||
private FramePhotos? _hashed, _timed;
|
||||
|
||||
/// <summary>The server's frames version the translation to the server's frames is made against.</summary>
|
||||
public int ServerFrames => manifest.FramesVersion;
|
||||
|
||||
/// <summary>Settings of the server sequence with their positions on this copy's frames, as they are now.</summary>
|
||||
public async Task<string?> ToLocalAsync(string? settingsJson, CancellationToken cancellationToken)
|
||||
/// <summary>
|
||||
/// The server's settings with their positions on this copy's frames as they are now, with this copy's row at those
|
||||
/// frames; null when that row is no longer at <paramref name="settingsVersion"/> (an edit was saved here since), or
|
||||
/// either copy's frames cannot be read as one state now (the next comparison translates again).
|
||||
/// </summary>
|
||||
public async Task<TranslatedSettings?> ToLocalAsync(string? settingsJson, int settingsVersion, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!ProjectSettings.NamesFrames(settingsJson)) return settingsJson;
|
||||
var local = await LocalAsync(cancellationToken).ConfigureAwait(false);
|
||||
var server = await ServerAsync(local, cancellationToken).ConfigureAwait(false);
|
||||
return ProjectSettings.TranslateFramePositions(settingsJson!, server, local) ?? settingsJson;
|
||||
if (!ProjectSettings.NamesFrames(settingsJson))
|
||||
return await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is { } row && row.SettingsVersion == settingsVersion
|
||||
? new TranslatedSettings(row, settingsJson)
|
||||
: null;
|
||||
if (await LocalAsync(cancellationToken).ConfigureAwait(false) is not { } local || local.Row.SettingsVersion != settingsVersion) return null;
|
||||
if (await ServerAsync(local.Frames, cancellationToken).ConfigureAwait(false) is not { } server) return null;
|
||||
return new TranslatedSettings(local.Row, ProjectSettings.TranslateFramePositions(settingsJson!, server, local.Frames) ?? settingsJson);
|
||||
}
|
||||
|
||||
/// <summary>Settings of this copy with their positions on the server's frames.</summary>
|
||||
public async Task<string?> ToServerAsync(string? settingsJson, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!ProjectSettings.NamesFrames(settingsJson)) return settingsJson;
|
||||
var local = await LocalAsync(cancellationToken).ConfigureAwait(false);
|
||||
var server = await ServerAsync(local, cancellationToken).ConfigureAwait(false);
|
||||
return ProjectSettings.TranslateFramePositions(settingsJson!, local, server) ?? settingsJson;
|
||||
}
|
||||
|
||||
/// <summary>This copy's frames, read each time (an import may insert some between a comparison and a write).</summary>
|
||||
private async Task<FramePhotos> LocalAsync(CancellationToken cancellationToken) =>
|
||||
new((await SequenceFrames.ListAsync(db, localId, cancellationToken).ConfigureAwait(false))
|
||||
.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
|
||||
|
||||
/// <summary>
|
||||
/// The server's frames from the manifest; their capture times are fetched only when a frame of either side is not
|
||||
/// hashed, since such a frame is matched by size and capture time.
|
||||
/// This copy's settings at <paramref name="settingsVersion"/> with their positions on the server's frames, with the
|
||||
/// row they were taken from: frames inserted here since the copy was read moved its positions without a new settings
|
||||
/// version, so the row is read again with its frames. Null when the row is at another settings version (an edit was
|
||||
/// saved since), or either copy's frames cannot be read as one state now.
|
||||
/// </summary>
|
||||
private async Task<FramePhotos> ServerAsync(FramePhotos local, CancellationToken cancellationToken)
|
||||
public async Task<TranslatedSettings?> ToServerAsync(int settingsVersion, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_server is not null) return _server;
|
||||
IReadOnlyDictionary<Guid, DateTime?> captured = new Dictionary<Guid, DateTime?>();
|
||||
if (local.HasUnhashed || manifest.Frames.Any(f => f.Sha256.Length == 0))
|
||||
captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
|
||||
return _server = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, captured.GetValueOrDefault(f.Id)))));
|
||||
if (await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is not { } row || row.SettingsVersion != settingsVersion) return null;
|
||||
// Positions cannot appear in a project that names no frame: an insertion only moves those it has.
|
||||
if (!ProjectSettings.NamesFrames(row.SettingsJson)) return new TranslatedSettings(row, row.SettingsJson);
|
||||
if (await LocalAsync(cancellationToken).ConfigureAwait(false) is not { } local || local.Row.SettingsVersion != settingsVersion) return null;
|
||||
if (await ServerAsync(local.Frames, cancellationToken).ConfigureAwait(false) is not { } server) return null;
|
||||
var settings = local.Row.SettingsJson;
|
||||
return new TranslatedSettings(local.Row, settings is null ? null : ProjectSettings.TranslateFramePositions(settings, local.Frames, server) ?? settings);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// This copy's row and frames as one state: the frames are listed between two reads of the row that find the same
|
||||
/// frames version (an insertion renumbers the frames and moves the row's positions in one transaction, and bumps the
|
||||
/// version). Null when the sequence is gone or an insertion lands during every attempt.
|
||||
/// </summary>
|
||||
private async Task<LocalState?> LocalAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
for (var attempt = 0; attempt < Attempts; attempt++)
|
||||
{
|
||||
if (await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is not { } before) return null;
|
||||
var frames = await SequenceFrames.ListAsync(db, localId, cancellationToken).ConfigureAwait(false);
|
||||
if (await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is not { } after) return null;
|
||||
if (after.FramesVersion == before.FramesVersion)
|
||||
return new LocalState(after, new FramePhotos(frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The server's frames from the manifest; null when they do not read as one frame list (an index given twice, or
|
||||
/// not the indices 0…n-1: pages read across an insertion). Their capture times are fetched unless both copies hold
|
||||
/// the same hashed photos at the same places: a frame is matched by size and capture time when either side has not
|
||||
/// hashed it, and a position parked on a server photo always names its capture time, so the same project reads the
|
||||
/// same whichever side hashed what.
|
||||
/// </summary>
|
||||
private async Task<FramePhotos?> ServerAsync(FramePhotos local, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_hashed is null)
|
||||
{
|
||||
if (!OneList(manifest.Frames)) return null;
|
||||
_hashed = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, null))));
|
||||
}
|
||||
if (_hashed.SameAs(local)) return _hashed;
|
||||
if (_timed is null)
|
||||
{
|
||||
var captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
|
||||
_timed = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, captured.GetValueOrDefault(f.Id)))));
|
||||
}
|
||||
return _timed;
|
||||
}
|
||||
|
||||
/// <summary>Each frame once, at the indices 0…n-1 a sequence's frames always take.</summary>
|
||||
private static bool OneList(IReadOnlyList<ManifestFrame> frames)
|
||||
{
|
||||
var seen = new bool[frames.Count];
|
||||
var ids = new HashSet<Guid>();
|
||||
foreach (var frame in frames)
|
||||
{
|
||||
if ((uint)frame.Index >= (uint)seen.Length || seen[frame.Index] || !ids.Add(frame.Id)) return false;
|
||||
seen[frame.Index] = true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private sealed record LocalState(Sequence Row, FramePhotos Frames);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Settings translated by <see cref="SyncPositions"/>, with this copy's row at the frames their positions were
|
||||
/// translated to or from (its own settings may have been moved since it was first read).
|
||||
/// </summary>
|
||||
internal sealed record TranslatedSettings(Sequence Local, string? SettingsJson);
|
||||
@@ -20,7 +20,7 @@ public interface ISyncRemote : IDisposable
|
||||
|
||||
Task<SequenceManifest> GetManifestAsync(Guid id, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>Every frame row of a sequence, in index order.</summary>
|
||||
/// <summary>Every frame row of a sequence, each once, in index order.</summary>
|
||||
Task<IReadOnlyList<FrameDto>> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>The first frame row of a sequence (its thumbnail stands for a sequence without a hero frame); null when it has none yet.</summary>
|
||||
@@ -101,11 +101,13 @@ public sealed class TitanoSyncRemote(TitanoClient client, bool ownsClient) : ISy
|
||||
public async Task<IReadOnlyList<FrameDto>> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
var all = new List<FrameDto>();
|
||||
// Pages follow the frame index: frames inserted while they are read list a frame again, which is kept once.
|
||||
var seen = new HashSet<Guid>();
|
||||
string? cursor = null;
|
||||
do
|
||||
{
|
||||
var page = await client.ListFramesAsync(sequenceId, cursor, FramePage, cancellationToken).ConfigureAwait(false);
|
||||
all.AddRange(page.Items);
|
||||
all.AddRange(page.Items.Where(f => seen.Add(f.Id)));
|
||||
cursor = page.NextCursor;
|
||||
}
|
||||
while (cursor is not null);
|
||||
|
||||
@@ -132,8 +132,10 @@ public static class SyncRules
|
||||
/// <summary>
|
||||
/// Originals missing on each side by content (BRIEF §14.5: originals never conflict): the same SHA-256 where both
|
||||
/// sides hashed them, else the same size and capture time (<see cref="FramePhoto.Matches"/>; a folder adopted on
|
||||
/// either side and not verified yet). Local frames not hashed yet that nothing matched are assumed to match server
|
||||
/// frames that nothing matched.
|
||||
/// either side and not verified yet). A photo is there when any photo of the other side matches it, as downloads and
|
||||
/// uploads find them (<see cref="FramePhotos.IndexOf"/>): two copies of one photo in a folder the server adopted are
|
||||
/// both held by the one original this PC keeps. Local frames not hashed yet that nothing matched are assumed to match
|
||||
/// server frames that nothing matched.
|
||||
/// </summary>
|
||||
public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection<FramePhoto> local, IReadOnlyCollection<FramePhoto> remote)
|
||||
{
|
||||
@@ -144,39 +146,15 @@ public static class SyncRules
|
||||
// What no hash matched: each content once, each frame not hashed on its own.
|
||||
var localLeft = Unmatched(local, remoteHashes);
|
||||
var remoteLeft = Unmatched(remote, localHashes);
|
||||
var byTime = new Dictionary<(long Size, long Time), List<int>>();
|
||||
for (var i = 0; i < remoteLeft.Count; i++)
|
||||
var localPhotos = new FramePhotos(localLeft.Select((p, i) => (i, p)));
|
||||
var remotePhotos = new FramePhotos(remoteLeft.Select((p, i) => (i, p)));
|
||||
var remoteUnmatched = remoteLeft.Count(p => localPhotos.IndexOf(p) is null);
|
||||
int localHashedLeft = 0, localUnhashedLeft = 0;
|
||||
foreach (var photo in localLeft.Where(p => remotePhotos.IndexOf(p) is null))
|
||||
{
|
||||
if (remoteLeft[i].CapturedAt is not { } time) continue;
|
||||
var key = (remoteLeft[i].Size, time.Ticks / TimeSpan.TicksPerMillisecond);
|
||||
if (!byTime.TryGetValue(key, out var list)) byTime[key] = list = [];
|
||||
list.Add(i);
|
||||
if (photo.IsHashed) localHashedLeft++;
|
||||
else localUnhashedLeft++;
|
||||
}
|
||||
var paired = new bool[remoteLeft.Count];
|
||||
int localHashedLeft = 0, localUnhashedLeft = 0, remoteMatched = 0;
|
||||
foreach (var photo in localLeft)
|
||||
{
|
||||
var match = -1;
|
||||
if (photo.CapturedAt is { } time && byTime.TryGetValue((photo.Size, time.Ticks / TimeSpan.TicksPerMillisecond), out var candidates))
|
||||
{
|
||||
var at = candidates.FindIndex(i => !paired[i] && remoteLeft[i].Matches(photo));
|
||||
if (at >= 0) match = candidates[at];
|
||||
}
|
||||
if (match >= 0)
|
||||
{
|
||||
paired[match] = true;
|
||||
remoteMatched++;
|
||||
}
|
||||
else if (photo.IsHashed)
|
||||
{
|
||||
localHashedLeft++;
|
||||
}
|
||||
else
|
||||
{
|
||||
localUnhashedLeft++;
|
||||
}
|
||||
}
|
||||
var remoteUnmatched = remoteLeft.Count - remoteMatched;
|
||||
return (localHashedLeft + Math.Max(0, localUnhashedLeft - remoteUnmatched), Math.Max(0, remoteUnmatched - localUnhashedLeft));
|
||||
|
||||
static List<FramePhoto> Unmatched(IEnumerable<FramePhoto> side, HashSet<string> other) =>
|
||||
|
||||
@@ -197,9 +197,16 @@ public sealed class SyncService
|
||||
try
|
||||
{
|
||||
var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false);
|
||||
var request = await RequestAsync(remote, local, manifest, seenRemote, cancellationToken).ConfigureAwait(false);
|
||||
var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false);
|
||||
link = Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion);
|
||||
// Edited here since it was shown, or frames that kept changing while they were read: compared again.
|
||||
if (await RequestAsync(remote, local, manifest, seenRemote, cancellationToken).ConfigureAwait(false) is not { } request)
|
||||
{
|
||||
changed = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false);
|
||||
link = Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion);
|
||||
}
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict)
|
||||
{
|
||||
@@ -349,7 +356,13 @@ public sealed class SyncService
|
||||
var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var request = await RequestAsync(remote, local, manifest, manifest.SettingsVersion, cancellationToken).ConfigureAwait(false);
|
||||
if (await RequestAsync(remote, local, manifest, manifest.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } request)
|
||||
{
|
||||
// An edit saved here meanwhile goes instead; frames that keep changing here leave it to the next comparison.
|
||||
if (attempt >= 3 || await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) is not { } fresh) return;
|
||||
local = fresh;
|
||||
continue;
|
||||
}
|
||||
var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false);
|
||||
await _links.UpsertAsync(Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
@@ -569,9 +582,13 @@ public sealed class SyncService
|
||||
manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
|
||||
// The listing and the manifest disagree: the server changed during the comparison; settle it next time.
|
||||
if (manifest.SettingsVersion != dto.SettingsVersion) return (link, local);
|
||||
// Compared (and shown) with its positions on this copy's frames, against this copy as it is at those frames: the
|
||||
// same photos are the same settings. Edited here meanwhile, or frames that kept changing: compared next time.
|
||||
if (await Positions(remote, local, manifest).ToLocalAsync(manifest.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false)
|
||||
is not { } translated) return (link, local);
|
||||
local = translated.Local;
|
||||
var mine = SyncPayload.Of(local);
|
||||
// Compared (and shown) with its positions on this copy's frames: the same photos are the same settings.
|
||||
var theirs = SyncPayload.Of(dto, await Positions(remote, local, manifest).ToLocalAsync(manifest.SettingsJson, cancellationToken).ConfigureAwait(false));
|
||||
var theirs = SyncPayload.Of(dto, translated.SettingsJson);
|
||||
if (mine.SameAs(theirs)) return (Agreed(link, local.SettingsVersion, manifest.SettingsVersion), local);
|
||||
// Last writer wins only when one side changed; both changed differently is the user's choice (BRIEF §14.5).
|
||||
return (link with
|
||||
@@ -602,14 +619,14 @@ public sealed class SyncService
|
||||
|
||||
/// <summary>
|
||||
/// Sends this PC's payload unless the server moved past <paramref name="expected"/>, which shows up as a conflict next,
|
||||
/// or its frames changed since <paramref name="manifest"/> (the positions are translated again next time).
|
||||
/// or either copy's frames changed since they were read (the positions are translated again next time).
|
||||
/// </summary>
|
||||
private async Task<SyncLink> PushAsync(ISyncRemote remote, SyncLink link, Sequence local, SequenceManifest manifest, int? expected,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var request = await RequestAsync(remote, local, manifest, expected, cancellationToken).ConfigureAwait(false);
|
||||
if (await RequestAsync(remote, local, manifest, expected, cancellationToken).ConfigureAwait(false) is not { } request) return link;
|
||||
var updated = await remote.UpdateSequenceAsync(link.RemoteId, request, cancellationToken).ConfigureAwait(false);
|
||||
return Agreed(Observe(link, link.Server, updated), local.SettingsVersion, updated.SettingsVersion);
|
||||
}
|
||||
@@ -620,16 +637,18 @@ public sealed class SyncService
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The PATCH that makes the server's copy equal to <paramref name="local"/>, its frame positions moved to the server's
|
||||
/// frames as <paramref name="manifest"/> lists them; the server refuses it when its frames changed since.
|
||||
/// The PATCH that makes the server's copy equal to this copy at <paramref name="local"/>'s settings version (read again
|
||||
/// with its frames, which may have moved its positions since), its frame positions moved to the server's frames as
|
||||
/// <paramref name="manifest"/> lists them; the server refuses it when its frames changed since. Null when this copy
|
||||
/// was edited since <paramref name="local"/> was read, or either copy's frames cannot be read as one state now.
|
||||
/// </summary>
|
||||
private async Task<UpdateSequenceRequest> RequestAsync(ISyncRemote remote, Sequence local, SequenceManifest manifest, int? expected,
|
||||
private async Task<UpdateSequenceRequest?> RequestAsync(ISyncRemote remote, Sequence local, SequenceManifest manifest, int? expected,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var positions = Positions(remote, local, manifest);
|
||||
var settings = await positions.ToServerAsync(local.SettingsJson, cancellationToken).ConfigureAwait(false);
|
||||
var frames = ProjectSettings.NamesFrames(local.SettingsJson) ? positions.ServerFrames : (int?)null;
|
||||
return (SyncPayload.Of(local) with { SettingsJson = settings }).ToRequest(expected, frames);
|
||||
if (await positions.ToServerAsync(local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } translated) return null;
|
||||
var frames = ProjectSettings.NamesFrames(translated.Local.SettingsJson) ? positions.ServerFrames : (int?)null;
|
||||
return (SyncPayload.Of(translated.Local) with { SettingsJson = translated.SettingsJson }).ToRequest(expected, frames);
|
||||
}
|
||||
|
||||
private SyncPositions Positions(ISyncRemote remote, Sequence local, SequenceManifest manifest) => new(_db, remote, local.Id, manifest);
|
||||
@@ -695,19 +714,17 @@ public sealed class SyncService
|
||||
|
||||
/// <summary>
|
||||
/// Writes the server's payload into the local index, project and sidecar in one step (settings version + 1) under the
|
||||
/// sequence's lease, which the editor's saves take too, and only while the local copy is still at the settings version
|
||||
/// <paramref name="local"/> was read at. The settings' frame positions are moved to this copy's frames as they are
|
||||
/// under the lease (<paramref name="positions"/>). Null when an edit was saved on this PC since: nothing is written
|
||||
/// then. A payload the local copy already holds writes nothing, so it needs neither a writable archive nor the lease.
|
||||
/// sequence's lease, which the editor's saves and the archive's frame commits take too, and only while the local copy
|
||||
/// is still at the settings version <paramref name="local"/> was read at. The settings' frame positions are moved to
|
||||
/// this copy's frames as they are under the lease (<paramref name="positions"/>). Null when an edit was saved on this
|
||||
/// PC since, or the frames of either copy could not be read as one state: nothing is written then. A payload the local
|
||||
/// copy already holds writes nothing, so it needs neither a writable archive nor the lease.
|
||||
/// </summary>
|
||||
private async Task<Sequence?> ApplyLocallyAsync(Sequence local, SyncPayload payload, SyncPositions positions, CancellationToken cancellationToken)
|
||||
{
|
||||
var stored = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidDataException($"Sequence {local.Id} is not indexed.");
|
||||
if (stored.SettingsVersion != local.SettingsVersion) return null;
|
||||
if (await positions.ToLocalAsync(payload.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } stored) return null;
|
||||
// Settled as it is: an archive that is offline or read-only does not keep the link from being agreed.
|
||||
var mapped = payload with { SettingsJson = await positions.ToLocalAsync(payload.SettingsJson, cancellationToken).ConfigureAwait(false) };
|
||||
if (mapped.SameAs(SyncPayload.Of(stored))) return stored;
|
||||
if ((payload with { SettingsJson = stored.SettingsJson }).SameAs(SyncPayload.Of(stored.Local))) return stored.Local;
|
||||
|
||||
var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed.");
|
||||
@@ -716,11 +733,10 @@ public sealed class SyncService
|
||||
var folder = paths.Sequence(local.Folder);
|
||||
await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var row = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidDataException($"Sequence {local.Id} is not indexed.");
|
||||
if (row.SettingsVersion != local.SettingsVersion) return null;
|
||||
// The frames cannot change under the lease: the positions are moved to them as they are now.
|
||||
payload = payload with { SettingsJson = await positions.ToLocalAsync(payload.SettingsJson, cancellationToken).ConfigureAwait(false) };
|
||||
if (await positions.ToLocalAsync(payload.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } current) return null;
|
||||
var row = current.Local;
|
||||
payload = payload with { SettingsJson = current.SettingsJson };
|
||||
var metadata = !payload.SameMetadata(SyncPayload.Of(row));
|
||||
var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson);
|
||||
if (!metadata && !settings) return row;
|
||||
|
||||
@@ -27,6 +27,9 @@ public static class SequenceEndpoints
|
||||
private const int MaxNotes = 2000;
|
||||
private const int MaxTags = 50;
|
||||
|
||||
/// <summary>Listings of a sequence's frames read without a lock before one is read under its row lock (<see cref="SnapshotAsync"/>).</summary>
|
||||
private const int SnapshotAttempts = 3;
|
||||
|
||||
public static void Map(RouteGroupBuilder api)
|
||||
{
|
||||
// Uploaders list sequences to pick an upload target.
|
||||
@@ -220,14 +223,16 @@ public static class SequenceEndpoints
|
||||
|
||||
/// <summary>
|
||||
/// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5), with the proxies the server asks
|
||||
/// for (<see cref="ServerProxies"/>); ETag over the version counters and the number of proxies asked for.
|
||||
/// for (<see cref="ServerProxies"/>); ETag over the version counters and the number of proxies asked for. The frames,
|
||||
/// the frames version and the settings whose positions name those frames are of one frames version (<see cref="SnapshotAsync"/>).
|
||||
/// </summary>
|
||||
private static async Task<IResult> ManifestAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive)
|
||||
{
|
||||
var ct = context.RequestAborted;
|
||||
if (await db.Sequences.GetAsync(id, ct) is not { } sequence) return ApiErrors.NotFound(context, "sequence");
|
||||
var spec = ServerProxies.Spec(archive.Current);
|
||||
var frames = await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, ct);
|
||||
if (await SnapshotAsync(db, id, (on, s, c) => ServerProxies.FramesAsync(on, archive.Current, s, spec, c), ct) is not { } snapshot)
|
||||
return ApiErrors.NotFound(context, "sequence");
|
||||
var (sequence, frames) = snapshot;
|
||||
var etag = string.Create(CultureInfo.InvariantCulture,
|
||||
$"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}-{frames.Count(f => f.WantsProxy)}\"");
|
||||
if (context.Request.Headers.IfNoneMatch == etag) return Results.StatusCode(StatusCodes.Status304NotModified);
|
||||
@@ -238,6 +243,30 @@ public static class SequenceEndpoints
|
||||
[.. renders.Select(Dto.Render)], spec), ProtocolJson.Default.SequenceManifest);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The sequence and its frames as of one frames version: a client pairs the settings' frame positions with these
|
||||
/// frames. <paramref name="listFrames"/> reads them in pages, and a commit of earlier photos renumbers them and moves
|
||||
/// the settings' positions at once, so a listing read across one would mix two orders. It is read again while the
|
||||
/// frames version moved during it, and then inside a transaction holding the sequence's row, which every commit of
|
||||
/// frames locks first (on SQLite, the write lock: the listing runs on that transaction's own connection). Null when
|
||||
/// the sequence is gone.
|
||||
/// </summary>
|
||||
internal static async Task<(Sequence Sequence, List<ManifestFrame> Frames)?> SnapshotAsync(TitanoDatabase db, Guid id,
|
||||
Func<TitanoRepositories, Sequence, CancellationToken, Task<List<ManifestFrame>>> listFrames, CancellationToken cancellationToken)
|
||||
{
|
||||
for (var attempt = 0; attempt < SnapshotAttempts; attempt++)
|
||||
{
|
||||
if (await db.Sequences.GetAsync(id, cancellationToken) is not { } sequence) return null;
|
||||
var frames = await listFrames(db, sequence, cancellationToken);
|
||||
if (await db.Sequences.GetAsync(id, cancellationToken) is not { } after) return null;
|
||||
if (after.FramesVersion == sequence.FramesVersion) return (sequence, frames);
|
||||
}
|
||||
// Commits keep landing (an upload of earlier photos): the frames are listed while none can.
|
||||
await using var tx = await db.BeginAsync(cancellationToken);
|
||||
if (await tx.Sequences.GetForUpdateAsync(id, cancellationToken) is not { } locked) return null;
|
||||
return (locked, await listFrames(tx, locked, cancellationToken));
|
||||
}
|
||||
|
||||
private static async Task<IResult> FramesAsync(HttpContext context, Guid id, string? cursor, int? limit, TitanoDatabase db)
|
||||
{
|
||||
var ct = context.RequestAborted;
|
||||
|
||||
@@ -38,7 +38,7 @@ public static class ServerProxies
|
||||
/// built them all or one was sent for the frame; it is wanted for a frame the server cannot decode that has none
|
||||
/// (and is not missing), unless <paramref name="spec"/> is null.
|
||||
/// </summary>
|
||||
public static async Task<List<ManifestFrame>> FramesAsync(TitanoDatabase db, Archive archive, Sequence sequence, ProxySpec? spec,
|
||||
public static async Task<List<ManifestFrame>> FramesAsync(TitanoRepositories db, Archive archive, Sequence sequence, ProxySpec? spec,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var proxies = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
|
||||
|
||||
@@ -74,6 +74,18 @@ internal sealed class SyncFixture : IAsyncDisposable
|
||||
return sequence;
|
||||
}
|
||||
|
||||
/// <summary>Files one more original into a local sequence through the archive ingest (an earlier photo lands before the others).</summary>
|
||||
public async Task AddLocalAsync(Guid sequenceId, byte[] original)
|
||||
{
|
||||
var staging = Path.Combine(Archive.Paths.StagingDirectory, "test");
|
||||
Directory.CreateDirectory(staging);
|
||||
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
|
||||
await File.WriteAllBytesAsync(path, original, Ct);
|
||||
var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(original));
|
||||
await Archive.Service.Ingest.CommitAsync(new IngestRequest(Archive.Archive.Id, sequenceId, null, null, path, $"IMG_{sha[..8]}.JPG", original.Length, sha,
|
||||
null, null), Ct);
|
||||
}
|
||||
|
||||
/// <summary>An edit on this PC, as the editor makes it (project file and index, settings version + 1).</summary>
|
||||
public async Task<Sequence> EditLocalSettingsAsync(Guid id, string settingsJson)
|
||||
{
|
||||
|
||||
@@ -79,6 +79,11 @@ public sealed class SyncRulesTests
|
||||
Assert.Equal((0, 3), SyncRules.CompareOriginals(Array.Empty<string?>(), ["", "", ""]));
|
||||
// Hashed on both sides, the hash decides even when size and time agree.
|
||||
Assert.Equal((1, 1), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("cc", 100, 0)]));
|
||||
// A photo the adopted folder holds twice ('IMG_0001.JPG' and 'IMG_0001 (1).JPG') is here in this PC's one copy,
|
||||
// as the download finds it: nothing is left to fetch.
|
||||
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("", 100, 0), Photo("", 100, 0)]));
|
||||
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo(null, 100, 0)], [Photo("", 100, 0), Photo("", 100, 0)]));
|
||||
Assert.Equal((0, 1), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("", 100, 0), Photo("", 100, 0), Photo("", 100, 10)]));
|
||||
}
|
||||
|
||||
[Test]
|
||||
|
||||
@@ -144,6 +144,53 @@ public sealed class SyncServiceTests(TestContext context)
|
||||
Assert.Equal((SyncStatus.ServerAhead, 1), ((await f.EntryAsync("Harbour")).Status, (await f.EntryAsync("Harbour")).Link!.MissingLocal));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task SettingsSentWhileAnImportHereMovesTheirPositionsStayOnTheirPhotos()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var photos = f.NewPhotos(4);
|
||||
var local = await f.LocalAsync("Harbour", photos[1..]);
|
||||
var remote = f.Remote.Add("Harbour", photos);
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
await f.RefreshAsync();
|
||||
await f.EditLocalSettingsAsync(local.Id, new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson());
|
||||
|
||||
// The earliest photo is imported here after the comparison read this copy, before its settings are translated.
|
||||
var racing = new RacingRemote(f.Remote, () => f.AddLocalAsync(local.Id, photos[0]).GetAwaiter().GetResult(), beforeManifest: true);
|
||||
await f.Sync.RefreshAsync(racing, new SyncPolicy(), f.Ct);
|
||||
|
||||
var mine = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!);
|
||||
Assert.SequenceEqual([2], mine.ExcludedFrames);
|
||||
Assert.Equal(3, mine.HeroFrame);
|
||||
var sent = ProjectSettings.FromJson(remote.SettingsJson!);
|
||||
Assert.SequenceEqual([Sha(photos[2])], sent.ExcludedFrames.Select(i => remote.Frames[i].Sha256));
|
||||
Assert.Equal(Sha(photos[3]), remote.Frames[sent.HeroFrame!.Value].Sha256);
|
||||
Assert.Null(sent.PendingFrames);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task SettingsComparedWhileAnImportHereMovesTheirPositionsAreAgreed()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var photos = f.NewPhotos(4);
|
||||
var local = await f.LocalAsync("Harbour", photos[1..]);
|
||||
var remote = f.Remote.Add("Harbour", photos);
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
await f.RefreshAsync();
|
||||
// Both sides exclude the same photo, each by its own frames.
|
||||
await f.EditLocalSettingsAsync(local.Id, new ProjectSettings { ExcludedFrames = [1] }.ToJson());
|
||||
f.Remote.EditSettings(remote.Id, new ProjectSettings { ExcludedFrames = [2] }.ToJson());
|
||||
|
||||
var racing = new RacingRemote(f.Remote, () => f.AddLocalAsync(local.Id, photos[0]).GetAwaiter().GetResult(), beforeManifest: true);
|
||||
await f.Sync.RefreshAsync(racing, new SyncPolicy(), f.Ct);
|
||||
|
||||
var entry = await f.EntryAsync("Harbour");
|
||||
Assert.Null(entry.Link!.ConflictRemote);
|
||||
Assert.Equal(entry.Local!.SettingsVersion, entry.Link.BaseLocalSettings);
|
||||
Assert.Equal(remote.SettingsVersion, entry.Link.BaseRemoteSettings);
|
||||
Assert.SequenceEqual([2], ProjectSettings.FromJson(entry.Local.SettingsJson!).ExcludedFrames);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheSameSettingsFromTheServerAreAgreedWhileTheArchiveIsOffline()
|
||||
{
|
||||
|
||||
@@ -366,6 +366,9 @@ public sealed class SyncTransferTests(TestContext context)
|
||||
Assert.Null(taken.Hero);
|
||||
Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes);
|
||||
Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them");
|
||||
// Each waits with its capture time too, whichever side hashed what: the same project reads the same either way.
|
||||
var waiting = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).PendingFrames!;
|
||||
Assert.True(waiting.All(p => p.CapturedAt is not null), "a photo waits with its capture time");
|
||||
var engine = await f.StartEngineAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using Titano.Data;
|
||||
using Titano.Protocol;
|
||||
using Titano.Server.Api;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Server.Tests.Api;
|
||||
@@ -114,6 +116,59 @@ public sealed class SequenceTests(TestContext context)
|
||||
Assert.Count(3, (await changed.ReadAsync(ProtocolJson.Default.SequenceManifest)).Frames);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheManifestListsTheFramesOfOneFramesVersion()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
await FramesOfOneVersionAsync(server);
|
||||
}
|
||||
|
||||
/// <summary>The manifest's frames are read again while frames are committed, then under the row lock (also run on PostgreSQL).</summary>
|
||||
internal static async Task FramesOfOneVersionAsync(TestServer server)
|
||||
{
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var first = await Uploads.UploadAsync(editor, Uploads.Photo(2000, 51), "a.CR3", sessionKey: "k");
|
||||
var db = server.Database;
|
||||
var id = first.SequenceId;
|
||||
var start = (await db.Sequences.GetAsync(id))!.FramesVersion;
|
||||
|
||||
// Frames are committed while the first listing is read: it is read again.
|
||||
var listings = new List<int>();
|
||||
var snapshot = await SequenceEndpoints.SnapshotAsync(db, id, async (_, sequence, ct) =>
|
||||
{
|
||||
listings.Add(sequence.FramesVersion);
|
||||
if (listings.Count == 1) await db.Sequences.RefreshStatsAsync(id, ct);
|
||||
return [];
|
||||
}, CancellationToken.None);
|
||||
Assert.SequenceEqual([start, start + 1], listings);
|
||||
Assert.Equal(start + 1, Assert.NotNull(snapshot).Sequence.FramesVersion);
|
||||
|
||||
// Commits keep landing: the last listing is read while none can.
|
||||
Task? waiting = null;
|
||||
listings.Clear();
|
||||
var locked = await SequenceEndpoints.SnapshotAsync(db, id, async (on, sequence, ct) =>
|
||||
{
|
||||
listings.Add(sequence.FramesVersion);
|
||||
if (listings.Count <= 3)
|
||||
{
|
||||
await db.Sequences.RefreshStatsAsync(id, ct);
|
||||
return [];
|
||||
}
|
||||
// Listed on the transaction that holds the row (on SQLite, the only connection that may while it writes).
|
||||
Assert.True(on is TitanoTransaction, "the frames are listed inside the locking transaction");
|
||||
Assert.Count(1, await on.Frames.ListBySequenceAsync(id, cancellationToken: ct));
|
||||
waiting = db.Sequences.RefreshStatsAsync(id, ct);
|
||||
await Task.WhenAny(waiting, Task.Delay(300, ct));
|
||||
Assert.False(waiting.IsCompleted, "a commit of frames waits until the frames are listed");
|
||||
return [];
|
||||
}, CancellationToken.None);
|
||||
await Assert.NotNull(waiting);
|
||||
Assert.SequenceEqual([start + 1, start + 2, start + 3, start + 4], listings);
|
||||
Assert.Equal(start + 4, Assert.NotNull(locked).Sequence.FramesVersion);
|
||||
Assert.Equal(start + 5, (await db.Sequences.GetAsync(id))!.FramesVersion);
|
||||
Assert.Null(await SequenceEndpoints.SnapshotAsync(db, Guid.NewGuid(), (_, _, _) => Task.FromResult(new List<ManifestFrame>()), CancellationToken.None));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ListingFiltersAndPagesWithACursor()
|
||||
{
|
||||
|
||||
@@ -54,6 +54,19 @@ public sealed class PostgresServerTests(TestContext context)
|
||||
Assert.Equal("ok", (await health.ReadAsync(ProtocolJson.Default.HealthStatus)).Checks["database"]);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheManifestListsTheFramesOfOneFramesVersionOnPostgres()
|
||||
{
|
||||
await using var schema = await Schema.CreateAsync(context);
|
||||
await using var server = await TestServer.StartAsync(context, o =>
|
||||
{
|
||||
o.Database = DatabaseKind.Postgres;
|
||||
o.DbConnectionString = schema.ConnectionString;
|
||||
});
|
||||
Assert.Equal(Dialect.Postgres, server.Database.Dialect);
|
||||
await Api.SequenceTests.FramesOfOneVersionAsync(server);
|
||||
}
|
||||
|
||||
/// <summary>A throw-away schema on the test server, selected through <c>Search Path</c>.</summary>
|
||||
private sealed class Schema : IAsyncDisposable
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user