fix(sync): finish stuck downloads and keep edits saved during a comparison
A download whose .part already held the whole original (the job stopped while hashing it) or more asked the server to resume past the end, got 416 and failed on every retry; such a part is now hashed again, and one the server cannot resume starts over (CD-1). Auto-sync pulled the server's settings with the copy of the sequence read when the comparison began, so an edit saved on this PC meanwhile was overwritten unseen. The payload is now applied in one step under the sequence lease and only while the local settings version is the one read; otherwise nothing is written and the change is settled as a conflict (CD-3). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
4 files changed
+137
-27
No files matched your search
@@ -58,7 +58,7 @@ public sealed class SyncService
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SyncLinkStore _links;
|
||||
private readonly ProjectStore _projects;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
|
||||
/// <summary>Unlinked pairs whose manifests were compared at these frame versions without a match (not fetched again).</summary>
|
||||
@@ -70,7 +70,7 @@ public sealed class SyncService
|
||||
Archives = archives;
|
||||
Options = options;
|
||||
_links = new SyncLinkStore(db);
|
||||
_projects = new ProjectStore(db, new SequenceWriter(options.Archive));
|
||||
_writer = new SequenceWriter(options.Archive);
|
||||
Proxies = new ProxyCache(options.ProxyCacheDirectory);
|
||||
}
|
||||
|
||||
@@ -211,8 +211,17 @@ public sealed class SyncService
|
||||
else
|
||||
{
|
||||
if (resolution == SyncResolution.SaveBoth) saved = await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false);
|
||||
var applied = await ApplyLocallyAsync(local, theirs, cancellationToken).ConfigureAwait(false);
|
||||
link = Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, seenRemote);
|
||||
if (await ApplyLocallyAsync(local, theirs, cancellationToken).ConfigureAwait(false) is { } applied)
|
||||
{
|
||||
link = Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, seenRemote);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Edited on this PC since it was read: the copy of a version the user did not choose to keep goes.
|
||||
changed = true;
|
||||
if (saved is not null) File.Delete(saved);
|
||||
saved = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!changed)
|
||||
@@ -359,7 +368,8 @@ public sealed class SyncService
|
||||
var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false);
|
||||
var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false);
|
||||
var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false);
|
||||
var applied = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false);
|
||||
// Edited on this PC meanwhile: nothing is agreed yet, so the next comparison settles both sides' content.
|
||||
if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false) is not { } applied) return;
|
||||
await _links.UpsertAsync(Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, manifest.SettingsVersion), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
@@ -547,8 +557,13 @@ public sealed class SyncService
|
||||
return (policy.CanEdit ? await PushAsync(remote, link, local, link.BaseRemoteSettings, cancellationToken).ConfigureAwait(false) : link, local);
|
||||
manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
|
||||
if (manifest.SettingsVersion != dto.SettingsVersion) return (link, local);
|
||||
var pulled = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false);
|
||||
return (Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion), pulled);
|
||||
if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false) is { } pulled)
|
||||
return (Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion), pulled);
|
||||
// The local copy was read when the comparison began and an edit was saved on this PC since: both sides changed,
|
||||
// so the server's version must not replace it unseen (BRIEF §14.5).
|
||||
var fresh = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (fresh is null || fresh.SettingsVersion == local.SettingsVersion) return (link, local);
|
||||
return await SettleSettingsAsync(remote, link, dto, fresh, manifest, policy, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Sends this PC's payload unless the server moved past <paramref name="expected"/>, which shows up as a conflict next.</summary>
|
||||
@@ -621,20 +636,64 @@ public sealed class SyncService
|
||||
|
||||
private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified);
|
||||
|
||||
/// <summary>Writes a payload into the local sidecar, project and index (each write bumps the local settings version).</summary>
|
||||
private async Task<Sequence> ApplyLocallyAsync(Sequence local, SyncPayload payload, CancellationToken cancellationToken)
|
||||
/// <summary>
|
||||
/// Writes a payload into the local index, project and sidecar in one step (settings version + 1) under the sequence's
|
||||
/// lease, which the editor's saves take too, and only while the local copy is still at the settings version
|
||||
/// <paramref name="local"/> was read at. Null when an edit was saved on this PC since: nothing is written then.
|
||||
/// </summary>
|
||||
private async Task<Sequence?> ApplyLocallyAsync(Sequence local, SyncPayload payload, CancellationToken cancellationToken)
|
||||
{
|
||||
var current = local;
|
||||
if (!payload.SameMetadata(SyncPayload.Of(local)))
|
||||
current = await Archives.SetSequenceInfoAsync(local.Id, payload.Title, payload.Tags, payload.Rating, payload.Notes, cancellationToken).ConfigureAwait(false);
|
||||
if (!SyncRules.SameSettings(payload.SettingsJson, current.SettingsJson))
|
||||
var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed.");
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(local.Folder);
|
||||
await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var archive = await _db.Archives.GetAsync(current.ArchiveId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidDataException($"Archive {current.ArchiveId} is not indexed.");
|
||||
var project = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson));
|
||||
await _projects.UpdateAsync(new ArchivePaths(archive.RootPath), current, _ => project, cancellationToken).ConfigureAwait(false);
|
||||
var row = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidDataException($"Sequence {local.Id} is not indexed.");
|
||||
if (row.SettingsVersion != local.SettingsVersion) return null;
|
||||
var metadata = !payload.SameMetadata(SyncPayload.Of(row));
|
||||
var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson);
|
||||
if (!metadata && !settings) return row;
|
||||
|
||||
string? project = null;
|
||||
Guid? hero = null;
|
||||
if (settings)
|
||||
{
|
||||
var edited = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson));
|
||||
project = edited.ToJson();
|
||||
hero = await HeroFrame.EditAsync(_db, row.Id, ProjectStore.Read(row, folder).HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
var edit = new SequenceEdit
|
||||
{
|
||||
Title = metadata ? payload.Title?.Trim() ?? string.Empty : null,
|
||||
Tags = metadata ? payload.Tags : null,
|
||||
Rating = metadata ? payload.Rating : null,
|
||||
Notes = metadata ? payload.Notes ?? string.Empty : null,
|
||||
SettingsJson = project,
|
||||
HeroFrameId = hero,
|
||||
};
|
||||
// The index first, at the version read under the lease: a writer that skips the lease still cannot be overwritten.
|
||||
var result = await _db.Sequences.EditAsync(row.Id, edit, row.SettingsVersion, cancellationToken).ConfigureAwait(false);
|
||||
if (result is not { Outcome: SequenceEditOutcome.Updated, Sequence: { } updated }) return null;
|
||||
if (project is not null) AtomicFile.WriteAllText(folder.ProjectPath, project);
|
||||
if (metadata)
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(folder, row.Id);
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
|
||||
{
|
||||
Title = updated.Title,
|
||||
Tags = updated.Tags,
|
||||
Rating = updated.Rating,
|
||||
Notes = updated.Notes,
|
||||
Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
await Archives.Indexer.RecordStampsAsync(archive, folder, row.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
return await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) ?? current;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -397,7 +397,11 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two attempts).</summary>
|
||||
/// <summary>
|
||||
/// A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two
|
||||
/// attempts). A part as long as the file (a job stopped while hashing it) is only hashed again; a part the server can
|
||||
/// no longer resume (416) starts over.
|
||||
/// </summary>
|
||||
private static async Task<string> FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging, CancellationToken cancellationToken)
|
||||
{
|
||||
var sha = frame.Sha256.ToLowerInvariant();
|
||||
@@ -407,8 +411,19 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
var part = target + ".part";
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
|
||||
await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false);
|
||||
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
|
||||
await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1)
|
||||
{
|
||||
File.Delete(part);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
|
||||
{
|
||||
File.Move(part, target, overwrite: true);
|
||||
|
||||
@@ -94,6 +94,31 @@ public sealed class SyncServiceTests(TestContext context)
|
||||
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnEditSavedHereDuringTheComparisonIsNotReplacedByTheServers()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var photos = f.NewPhotos(1);
|
||||
var local = await f.LocalAsync("Harbour", photos);
|
||||
var remote = f.Remote.Add("Harbour", photos);
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
await f.RefreshAsync();
|
||||
|
||||
// The server's copy changes; the editor saves this PC's edit after the comparison read the local copy, before it
|
||||
// takes the server's version.
|
||||
f.Remote.EditSettings(remote.Id, SyncFixture.Window33);
|
||||
var racing = new RacingRemote(f.Remote, () => f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41).GetAwaiter().GetResult(),
|
||||
beforeManifest: true);
|
||||
await f.Sync.RefreshAsync(racing, new SyncPolicy(), f.Ct);
|
||||
|
||||
var entry = await f.EntryAsync("Harbour");
|
||||
Assert.Equal(SyncStatus.Conflict, entry.Status);
|
||||
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local!.SettingsJson), "this PC's edit is kept");
|
||||
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(entry.Local.Folder).ProjectPath, f.Ct)));
|
||||
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, entry.Link!.ConflictRemote!.SettingsJson), "the user picks");
|
||||
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, remote.SettingsJson));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task WithoutAutomaticSyncAnEditWaitsForTheUser()
|
||||
{
|
||||
@@ -432,8 +457,11 @@ public sealed class SyncServiceTests(TestContext context)
|
||||
return (local, remote);
|
||||
}
|
||||
|
||||
/// <summary>A remote where someone else edits the settings right after the first read of a sequence, or right before the first edit.</summary>
|
||||
private sealed class RacingRemote(FakeSyncRemote inner, Action race, bool beforeUpdate = false) : ISyncRemote
|
||||
/// <summary>
|
||||
/// A remote where someone edits right after the first read of a sequence, right before the first edit, or right
|
||||
/// before the first manifest is read.
|
||||
/// </summary>
|
||||
private sealed class RacingRemote(FakeSyncRemote inner, Action race, bool beforeUpdate = false, bool beforeManifest = false) : ISyncRemote
|
||||
{
|
||||
private Action? _race = race;
|
||||
|
||||
@@ -442,12 +470,17 @@ public sealed class SyncServiceTests(TestContext context)
|
||||
public async Task<SequenceDto> GetSequenceAsync(Guid id, CancellationToken cancellationToken)
|
||||
{
|
||||
var dto = await inner.GetSequenceAsync(id, cancellationToken);
|
||||
if (!beforeUpdate) Interlocked.Exchange(ref _race, null)?.Invoke();
|
||||
if (!beforeUpdate && !beforeManifest) Interlocked.Exchange(ref _race, null)?.Invoke();
|
||||
return dto;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<SequenceDto>> ListSequencesAsync(CancellationToken cancellationToken) => inner.ListSequencesAsync(cancellationToken);
|
||||
public Task<SequenceManifest> GetManifestAsync(Guid id, CancellationToken cancellationToken) => inner.GetManifestAsync(id, cancellationToken);
|
||||
public Task<SequenceManifest> GetManifestAsync(Guid id, CancellationToken cancellationToken)
|
||||
{
|
||||
if (beforeManifest) Interlocked.Exchange(ref _race, null)?.Invoke();
|
||||
return inner.GetManifestAsync(id, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<FrameDto>> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken) => inner.ListFramesAsync(sequenceId, cancellationToken);
|
||||
public Task<SequenceDto> UpdateSequenceAsync(Guid id, UpdateSequenceRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
@@ -112,16 +112,19 @@ public sealed class SyncServerTests(TestContext context)
|
||||
var (_, _, token) = await server.UserAsync("alberto", "editor");
|
||||
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
|
||||
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
|
||||
var photos = Enumerable.Range(20, 3).Select(SyncClient.Photo).ToList();
|
||||
var photos = Enumerable.Range(20, 4).Select(SyncClient.Photo).ToList();
|
||||
var local = await studio.LocalAsync("Aurora", photos);
|
||||
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
|
||||
var remoteId = (await studio.EntryAsync("Aurora")).Link!.RemoteId;
|
||||
|
||||
// An earlier attempt left half of one original, and a damaged half of another.
|
||||
// An earlier attempt left half of one original, a damaged half of another, a third complete (stopped while it was
|
||||
// being hashed) and a fourth longer than the file: the server cannot resume either of the last two (416).
|
||||
var staging = Path.Combine(laptop.Archive.RootPath, ".titano", "staging", "sync-" + remoteId.ToString("N"));
|
||||
Directory.CreateDirectory(staging);
|
||||
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[0]) + ".jpg.part"), photos[0][..(photos[0].Length / 2)]);
|
||||
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[1]) + ".jpg.part"), new byte[photos[1].Length / 2]);
|
||||
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[2]) + ".jpg.part"), photos[2]);
|
||||
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[3]) + ".jpg.part"), [.. photos[3], .. new byte[64]]);
|
||||
|
||||
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
|
||||
var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Aurora", SyncDownloadMode.Originals, laptop.Archive.Id);
|
||||
|
||||
Reference in new issue
Block a user