Owner review 2, R8, phase 5b (client part), ported onto the final phase 5a.
A Chronos Server with the "projects" capability is now the home of
workspace projects. A server sequence is edited in the workspace like a
local one, not inside the library, and its versions sync with the server
"like git".
- Data: ServerProjectHome is an IProjectHome over ChronosClient.
- These are HomeUnavailableException, so the workspace shows Offline: a
server out of reach, an expired sign-in, a 503 (lease, offline archive,
projects_unreadable), a 409 folder_in_use, and a server without projects.
- 404 sequence/project/version and refused names are ProjectException;
other refusals keep their meaning.
- A deleted project's head is 404 project, so final 5a's HeadAsync NotFound
rule reads it as Gone. The original 5b's IsLiveAsync and
WorkspaceStatus.Gone are dropped. IProjectHome gains rename and delete.
- Data: WorkspaceJobs.CopyFromServer downloads into the workspace, through
the desktop's session, the proxies of one frames version (or the
thumbnails standing in for them), with the aliases of the server's
conversions. It is rebuilt on final 5a's copy handler:
- the copy:frame:fv:folder checkpoint, EnsureSpace (from the server's proxy
sizes), WritingMedia, and the cancelled-copy clean-up;
- the editor's current file is never overwritten;
- each file is hashed as it arrives and checked on disk, four at a time.
It never writes this PC's archive. Originals are still refused until 5c.
- Data: SyncAgent passes on the server's projectChanged events, and after a
lost event stream it reports that changes may have been missed.
- App: SyncHost gives the server home over one session client. The Server
screen offers Open in workspace, disabled with its reason on older
servers, whose sequences keep opening with proxies.
- App: Edit opens or checks out server projects with final 5a's refusals:
unreadable copies, a disabled damaged or newer Main, a refused folder. The
"sequence left the index" refusal now applies only to projects of this
PC's archive.
- App: the picker and the chip follow projectChanged (Server newer, Not on
server). Every state, conflict, delete, remove and failure text uses its
".Server" form when one exists (WorkspaceText.ForHome). That includes 5a's
newer-format conflict, the save conflict, the taken-values toast, Gone and
version-too-large.
- App: a render of a server project first offers to send what is not on the
server, then renders the version the server holds. Offline, Gone, Damaged
and missing media hold the render. Analyses and renders run as server jobs
(ServerJobs, shared with RemoteSequenceSource).
- Pruning keeps final 5a's rule: a version is pruned only after HoldsAsync
reads it back.
- Strings in en and it; ARCHITECTURE, DECISIONS and CHANGELOG.
Tests:
- ServerProjectHome error mapping, using a fake handler.
- Against the in-process server:
- two workspaces on one project: fast-forward, replay and merge, keep mine,
take server, save both, restore;
- the copy job verifying, resuming and following new frames;
- a project deleted on the server (Gone from its head; a push sends
nothing);
- an offline server.
- SyncAgent project events.
- App tests for the host, the project menu and renders of a server home.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
348 lines
18 KiB
C#
348 lines
18 KiB
C#
using Chronos.Core.Model;
|
|
using Chronos.Data.Sync;
|
|
using Chronos.Protocol;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Sync;
|
|
|
|
/// <summary>The background agent: event-driven comparisons, retries with backoff, proxies kept offline.</summary>
|
|
[Category("sync")]
|
|
public sealed class SyncAgentTests(TestContext context)
|
|
{
|
|
/// <summary>How long any wait may take on a loaded runner; every wait ends as soon as what it waits for happened.</summary>
|
|
private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30);
|
|
|
|
/// <summary>Full comparisons only at start, so anything later comes from events or explicit requests.</summary>
|
|
private static readonly SyncAgentOptions EventsOnly = new()
|
|
{
|
|
RefreshInterval = TimeSpan.FromHours(1),
|
|
EventDelay = TimeSpan.FromMilliseconds(20),
|
|
MinBackoff = TimeSpan.FromMilliseconds(20),
|
|
MaxBackoff = TimeSpan.FromMilliseconds(200),
|
|
};
|
|
|
|
[Test]
|
|
public async Task ServerEventsBringChangesWithoutAFullComparison()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
var first = agent.Snapshot!;
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
f.Remote.EditSettings(remote.Id, SyncFixture.Window33);
|
|
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
|
|
}, "the comparison of the server's edit");
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson),
|
|
"the server's edit reached this PC");
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
var created = f.Remote.Add("New on the server", f.NewPhotos(1));
|
|
f.Remote.Publish(new SequenceChangedEvent(created.Id, "created", 1, 1));
|
|
}, "the comparison of a sequence created on the server");
|
|
Assert.Contains(await f.Sync.ListAsync(f.Server, f.Ct), e => e.Title == "New on the server");
|
|
Assert.Same(first, agent.Snapshot);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProjectChangesAreToldAndALostStreamSaysSomeWereMissed()
|
|
{
|
|
// Owner review 2, R8, phase 5b: the server's projectChanged reaches the workspace through the agent.
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
f.Remote.Add("Harbour", f.NewPhotos(1));
|
|
f.Remote.FailingEvents = 1;
|
|
var told = new List<ProjectChangedEvent?>();
|
|
var missed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var changed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
agent.ProjectChanged += e =>
|
|
{
|
|
lock (told) told.Add(e);
|
|
(e is null ? missed : changed).TrySetResult();
|
|
// What a listener throws never ends the event loop.
|
|
throw new InvalidOperationException("a listener that fails");
|
|
};
|
|
agent.Start();
|
|
|
|
// The stream was lost once: the comparison after it says changes may have been missed.
|
|
await SignalAsync(missed.Task, "the missed changes told");
|
|
var change = new ProjectChangedEvent(Guid.CreateVersion7(), Guid.CreateVersion7(), ProjectChanges.Pushed, 3, Guid.CreateVersion7());
|
|
f.Remote.Publish(change);
|
|
await SignalAsync(changed.Task, "the project change told");
|
|
lock (told) Assert.Equal(change, told.Single(e => e is not null));
|
|
Assert.Equal(1, agent.Snapshot!.ServerSequences);
|
|
}
|
|
|
|
[Test]
|
|
public async Task FailedComparisonsAreRetriedWithBackoff()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
f.Remote.Add("Harbour", f.NewPhotos(1));
|
|
f.Remote.FailingListings = 3;
|
|
var changes = 0;
|
|
var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
// The agent stores the snapshot, then clears the last error, then raises Changed: only Changed sees both.
|
|
agent.Changed += () =>
|
|
{
|
|
Interlocked.Increment(ref changes);
|
|
if (agent.Snapshot is not null) compared.TrySetResult();
|
|
};
|
|
agent.Start();
|
|
|
|
await SignalAsync(compared.Task, "a comparison after the failures");
|
|
Assert.Null(agent.LastError);
|
|
Assert.Equal(0, f.Remote.FailingListings);
|
|
Assert.GreaterOrEqual(changes, 4);
|
|
Assert.Equal(1, agent.Snapshot!.ServerSequences);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ATimedOutComparisonIsRecordedAndLiveUpdatesGoOn()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(1);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
f.Remote.TimeoutOnce = remote.Id;
|
|
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
|
|
}, "the comparison that timed out");
|
|
Assert.NotNull((await f.EntryAsync("Harbour")).Link!.Error);
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
f.Remote.EditSettings(remote.Id, SyncFixture.Window33);
|
|
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
|
|
}, "the comparison of a later event");
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson),
|
|
"a later event still reaches this PC");
|
|
Assert.Null((await f.EntryAsync("Harbour")).Link!.Error);
|
|
}
|
|
|
|
[Test]
|
|
public async Task KeepingProxiesOfflineRefreshesTheCachesThisPcOpened()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, f.Remote.Add("Harbour", photos).Id, local.Id, f.Ct);
|
|
var opened = f.Remote.Add("Construction cam", f.NewPhotos(2));
|
|
var never = f.Remote.Add("Never opened", f.NewPhotos(1));
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
opened.AddFrame(f.NewPhotos(1)[0]);
|
|
var keep = false;
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: keep), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
|
|
|
|
keep = true;
|
|
await agent.RefreshAsync(f.Ct);
|
|
// The download writes the cache, then the link, and only then ends: until it ends its row shows a transfer.
|
|
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
|
|
Assert.Equal(3, Assert.NotNull(f.Sync.OpenRemote(f.Server, opened.Id)).CachedCount, "the stale cache brought up to date");
|
|
Assert.Null(f.Sync.OpenRemote(f.Server, never.Id));
|
|
var entries = await f.Sync.ListAsync(f.Server, f.Ct);
|
|
Assert.Equal(SyncStatus.ProxiesOnly, entries.Single(e => e.Title == "Construction cam").Status);
|
|
Assert.Equal(SyncStatus.RemoteOnly, entries.Single(e => e.Title == "Never opened").Status);
|
|
Assert.Equal(SyncStatus.InSync, entries.Single(e => e.Title == "Harbour").Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFailedProxyRefreshWaitsForItsBackoff()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var opened = f.Remote.Add("Construction cam", f.NewPhotos(2));
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null)).Id,
|
|
JobState.Done);
|
|
opened.AddFrame(f.NewPhotos(1)[0]);
|
|
f.Remote.FailAssets = true;
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine,
|
|
EventsOnly with { MinBackoff = TimeSpan.FromHours(1), MaxBackoff = TimeSpan.FromHours(2) });
|
|
agent.Start();
|
|
await WaitAsync(async () => (await DownloadsAsync(f)).Any(j => j.State == JobState.Failed), "the background refresh failing");
|
|
|
|
await agent.RefreshAsync(f.Ct);
|
|
await agent.RefreshAsync(f.Ct);
|
|
|
|
Assert.Equal(2, (await DownloadsAsync(f)).Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThumbnailsStandingInAreReplacedOnceTheServerOffersTheirProxies()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
// The server cannot decode these originals and waits for their proxies; one frame (missing or damaged) never gets
|
|
// one, so the sequence's proxy flag never comes.
|
|
var remote = f.Remote.Add("Fog", f.NewPhotos(3), status: SequenceStatus.ThumbsBuilt);
|
|
remote.Undecodable = true;
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(1, await DownloadsAsync(f));
|
|
|
|
// Another PC sends the proxies of two frames; the server's proxy pass then records the sequence's state.
|
|
foreach (var frame in remote.Frames.Take(2)) f.Remote.SentProxies[frame.Id] = frame.Proxy;
|
|
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
|
await agent.RefreshAsync(f.Ct);
|
|
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
|
|
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Select(file => (int)new FileInfo(file!).Length));
|
|
// The frame still waiting keeps its thumbnail; nothing is queued again until the server's sequence changes.
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(2, await DownloadsAsync(f));
|
|
}
|
|
|
|
[Test]
|
|
public async Task FramesTheServerHadNotDrawnAreFetchedAgainOnceItDid()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Construction cam", f.NewPhotos(2));
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
|
|
// A photo reaches the server, whose proxy flag stays set until its next proxy pass draws the photo: the refresh
|
|
// that follows the upload caches nothing for it.
|
|
var added = remote.AddFrame(f.NewPhotos(1)[0]);
|
|
f.Remote.Undrawn.UnionWith([(added.Id, "proxy"), (added.Id, "thumb")]);
|
|
await agent.RefreshAsync(f.Ct);
|
|
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
|
|
Assert.Null(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files[2]);
|
|
Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
|
|
// Nothing is fetched again while the server's sequence stands still.
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(2, await DownloadsAsync(f));
|
|
|
|
// The pass draws the photo and records the sequence's state.
|
|
f.Remote.Undrawn.Clear();
|
|
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
|
await agent.RefreshAsync(f.Ct);
|
|
var drawn = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != refresh.Id);
|
|
await f.Archive.WaitForAsync(drawn.Id, JobState.Done);
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
|
|
Assert.False((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(3, await DownloadsAsync(f));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AStandInDownloadThatFailedIsQueuedAgainAfterItsBackoff()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Fog", f.NewPhotos(3), status: SequenceStatus.ThumbsBuilt);
|
|
remote.Undecodable = true;
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
|
|
// Two frames' proxies reach the server, but their download fails (the server restarts meanwhile).
|
|
foreach (var frame in remote.Frames.Take(2)) f.Remote.SentProxies[frame.Id] = frame.Proxy;
|
|
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
|
f.Remote.FailAssets = true;
|
|
await agent.RefreshAsync(f.Ct);
|
|
var failed = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(failed.Id, JobState.Failed);
|
|
|
|
// The server's sequence does not change again: once the backoff has passed, the download is queued anyway.
|
|
f.Remote.FailAssets = false;
|
|
await WaitAsync(async () =>
|
|
{
|
|
await agent.RefreshAsync(f.Ct);
|
|
return (await DownloadsAsync(f)).Count == 3;
|
|
}, "the download queued again");
|
|
var retry = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != failed.Id);
|
|
await f.Archive.WaitForAsync(retry.Id, JobState.Done);
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Select(file => (int)new FileInfo(file!).Length));
|
|
}
|
|
|
|
private static async Task<IReadOnlyList<Job>> DownloadsAsync(SyncFixture f) =>
|
|
(await f.Db.Jobs.ListAsync(null, JobKind.Download, 100, null, f.Ct)).Items;
|
|
|
|
/// <summary>
|
|
/// Runs <paramref name="cause"/> and returns once the comparison it starts has finished. The engine raises
|
|
/// <see cref="SyncService.Changed"/> after a comparison stored every link and state, the moment the Server screen
|
|
/// reloads; a poll could instead see the local copy updated before the link that goes with it is stored. No other
|
|
/// comparison may be running.
|
|
/// </summary>
|
|
private async Task ComparedAfterAsync(SyncFixture f, Action cause, string what)
|
|
{
|
|
// Continuations run on the pool, not inside the agent loop that raises Changed and that the test later stops.
|
|
var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
Action<string> onChanged = _ => compared.TrySetResult();
|
|
f.Sync.Changed += onChanged;
|
|
try
|
|
{
|
|
cause();
|
|
await SignalAsync(compared.Task, what);
|
|
}
|
|
finally
|
|
{
|
|
f.Sync.Changed -= onChanged;
|
|
}
|
|
}
|
|
|
|
private async Task SignalAsync(Task signal, string what)
|
|
{
|
|
try
|
|
{
|
|
await signal.WaitAsync(Deadline, context.Cancellation);
|
|
}
|
|
catch (TimeoutException)
|
|
{
|
|
Assert.Fail("Timed out waiting for " + what);
|
|
}
|
|
}
|
|
|
|
private async Task WaitAsync(Func<bool> condition, string what) => await WaitAsync(() => Task.FromResult(condition()), what);
|
|
|
|
private async Task WaitAsync(Func<Task<bool>> condition, string what)
|
|
{
|
|
var until = DateTime.UtcNow + Deadline;
|
|
while (!await condition())
|
|
{
|
|
if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what);
|
|
await Task.Delay(20, context.Cancellation);
|
|
}
|
|
}
|
|
}
|