fix(sync): tell missed project changes only once the event stream is open again

The agent told "changes may have been missed" as soon as the stream dropped,
so a workspace re-read before the server's events could reach it again and
missed changes made in between. The signal now waits for the reconnected
stream and the refresh that follows it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alby96andClaude Opus 5.5 committed 2026-11-04 12:31:51 +01:00
1 parent 353024fc71
commit ed7452e739
4 files changed
+102 -9

No files matched your search

+26 -5
View File
@@ -60,8 +60,16 @@ public sealed class SyncAgent : IAsyncDisposable
private readonly SemaphoreSlim _refreshing = new(1, 1);
private Task _running = Task.CompletedTask;
/// <summary>1 while project changes may have been missed (the event stream was lost), until a comparison reached the server.</summary>
/// <summary>
/// Whether project changes may have been missed: <see cref="Missed"/> from the moment the event stream was lost,
/// <see cref="Reopened"/> once it is open again (no change is missed from then on), back to <see cref="InStep"/> when a
/// comparison that started after that reached the server and <see cref="ProjectChanged"/> said so.
/// </summary>
private int _projectsMissed;
private const int InStep = 0;
private const int Missed = 1;
private const int Reopened = 2;
private SyncSnapshot? _snapshot;
private Exception? _lastError;
@@ -114,13 +122,16 @@ public sealed class SyncAgent : IAsyncDisposable
await _refreshing.WaitAsync(linked.Token).ConfigureAwait(false);
try
{
// Only a comparison that started once the stream was open again covers every change missed meanwhile: one that
// started before may end while the stream is still down, and a change said then would be missed for good.
var reopened = Volatile.Read(ref _projectsMissed) == Reopened;
var policy = _policy();
var snapshot = await _sync.RefreshAsync(_remote, policy, linked.Token).ConfigureAwait(false);
Volatile.Write(ref _snapshot, snapshot);
Volatile.Write(ref _lastError, null);
if (policy.KeepProxiesOffline && _engine is not null) await QueueProxiesAsync(snapshot, linked.Token).ConfigureAwait(false);
Changed?.Invoke();
if (Interlocked.Exchange(ref _projectsMissed, 0) == 1) TellProjects(null);
if (reopened && Interlocked.CompareExchange(ref _projectsMissed, InStep, Reopened) == Reopened) TellProjects(null);
return snapshot;
}
finally
@@ -180,7 +191,7 @@ public sealed class SyncAgent : IAsyncDisposable
{
try
{
await foreach (var e in _remote.EventsAsync(cancellationToken).ConfigureAwait(false))
await foreach (var e in _remote.EventsAsync(Reconnected, cancellationToken).ConfigureAwait(false))
{
failures = 0;
if (e is ProjectChangedEvent project)
@@ -201,8 +212,9 @@ public sealed class SyncAgent : IAsyncDisposable
{
Log.Info(LogCategory, $"Event stream of {Server} lost: {ex.Message}");
}
// Project changes said meanwhile are missed: the next comparison that reaches the server says so.
Volatile.Write(ref _projectsMissed, 1);
// Project changes said meanwhile are missed: the first comparison that reaches the server once the stream is open
// again says so (Reconnected).
Volatile.Write(ref _projectsMissed, Missed);
try
{
await Task.Delay(Backoff(failures++), cancellationToken).ConfigureAwait(false);
@@ -215,6 +227,15 @@ public sealed class SyncAgent : IAsyncDisposable
}
}
/// <summary>
/// The event stream is open (again): after a loss, a comparison is asked for now, and the one that starts from here says
/// that project changes may have been missed (<see cref="ProjectChanged"/> with null).
/// </summary>
private void Reconnected()
{
if (Interlocked.CompareExchange(ref _projectsMissed, Reopened, Missed) == Missed) RequestRefresh();
}
/// <summary>Tells <see cref="ProjectChanged"/>; what a listener throws is logged, never ends the event loop.</summary>
private void TellProjects(ProjectChangedEvent? change)
{
+14
View File
@@ -64,6 +64,18 @@ public interface ISyncRemote : IDisposable
/// <summary>Server events until cancelled or the socket closes.</summary>
IAsyncEnumerable<ServerEvent> EventsAsync(CancellationToken cancellationToken);
/// <summary>
/// <see cref="EventsAsync(CancellationToken)"/>, calling <paramref name="connected"/> once the stream is open, before its
/// first event: from then on no event the server sends is missed. A remote that cannot say calls it when the stream is
/// asked for.
/// </summary>
IAsyncEnumerable<ServerEvent> EventsAsync(Action connected, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(connected);
connected();
return EventsAsync(cancellationToken);
}
/// <summary>
/// Every frame of a sequence as of one frames version, with the photo aliases of its conversions (owner review 2, R8:
/// what a workspace copy of a server project holds; servers with the <c>projects</c> capability). A remote without the
@@ -148,6 +160,8 @@ public sealed class ChronosSyncRemote(ChronosClient client, bool ownsClient) : I
public IAsyncEnumerable<ServerEvent> EventsAsync(CancellationToken cancellationToken) => client.EventsAsync(cancellationToken);
public IAsyncEnumerable<ServerEvent> EventsAsync(Action connected, CancellationToken cancellationToken) => client.EventsAsync(connected, cancellationToken);
public Task<ProjectFramesDto> GetProjectFramesAsync(Guid sequenceId, CancellationToken cancellationToken) => client.GetProjectFramesAsync(sequenceId, cancellationToken);
public Task<string?> GetAnalysisReportAsync(Guid sequenceId, CancellationToken cancellationToken) => client.GetAnalysisReportAsync(sequenceId, cancellationToken);
@@ -75,7 +75,27 @@ internal sealed class FakeSyncRemote : ISyncRemote
}
}
public void Publish(ServerEvent e) => _events.Writer.TryWrite(e);
/// <summary>
/// Sends an event to the stream open now. With <see cref="DropWhileClosed"/>, one sent while no stream is open is lost,
/// as a real server's is.
/// </summary>
public void Publish(ServerEvent e)
{
if (DropWhileClosed && !Subscribed) return;
_events.Writer.TryWrite(e);
}
/// <summary>Events sent while no stream is open are lost (<see cref="Publish"/>).</summary>
public bool DropWhileClosed { get; set; }
/// <summary>How long a stream takes to open after a lost one (a reconnection).</summary>
public TimeSpan ReconnectDelay { get; set; }
/// <summary>Whether an event stream is open now.</summary>
public bool Subscribed => Volatile.Read(ref _subscribed) > 0;
private int _subscribed;
private int _opened;
public Task<IReadOnlyList<SequenceDto>> ListSequencesAsync(CancellationToken cancellationToken)
{
@@ -252,8 +272,11 @@ internal sealed class FakeSyncRemote : ISyncRemote
}
}
public async IAsyncEnumerable<ServerEvent> EventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
public IAsyncEnumerable<ServerEvent> EventsAsync(CancellationToken cancellationToken) => EventsAsync(() => { }, cancellationToken);
public async IAsyncEnumerable<ServerEvent> EventsAsync(Action connected, [EnumeratorCancellation] CancellationToken cancellationToken)
{
if (Interlocked.Increment(ref _opened) > 1 && ReconnectDelay > TimeSpan.Zero) await Task.Delay(ReconnectDelay, cancellationToken);
lock (_gate)
{
if (FailingEvents > 0)
@@ -262,9 +285,18 @@ internal sealed class FakeSyncRemote : ISyncRemote
throw new HttpRequestException("socket closed");
}
}
while (await _events.Reader.WaitToReadAsync(cancellationToken))
Interlocked.Increment(ref _subscribed);
try
{
while (_events.Reader.TryRead(out var e)) yield return e;
connected();
while (await _events.Reader.WaitToReadAsync(cancellationToken))
{
while (_events.Reader.TryRead(out var e)) yield return e;
}
}
finally
{
Interlocked.Decrement(ref _subscribed);
}
}
@@ -80,6 +80,32 @@ public sealed class SyncAgentTests(TestContext context)
Assert.Equal(1, agent.Snapshot!.ServerSequences);
}
[Test]
public async Task MissedChangesAreToldOnlyOnceTheStreamIsOpenAgain()
{
// A change said between "changes may have been missed" and the stream's return would be lost for good: the signal
// comes from a comparison that started once the stream was open again, so every later change reaches the workspace.
await using var f = await SyncFixture.CreateAsync(context);
f.Remote.Add("Harbour", f.NewPhotos(1));
(f.Remote.FailingEvents, f.Remote.DropWhileClosed, f.Remote.ReconnectDelay) = (1, true, TimeSpan.FromMilliseconds(800));
var subscribedWhenMissed = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
var changed = new TaskCompletionSource<ProjectChangedEvent>(TaskCreationOptions.RunContinuationsAsynchronously);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
agent.ProjectChanged += e =>
{
if (e is null) subscribedWhenMissed.TrySetResult(f.Remote.Subscribed);
else changed.TrySetResult(e);
};
agent.Start();
await SignalAsync(subscribedWhenMissed.Task, "the missed changes told");
Assert.True(await subscribedWhenMissed.Task, "the stream was open again when the missed changes were told");
var change = new ProjectChangedEvent(Guid.CreateVersion7(), Guid.CreateVersion7(), ProjectChanges.Pushed, 2, Guid.CreateVersion7());
f.Remote.Publish(change);
await SignalAsync(changed.Task, "the change said after the signal told");
Assert.Equal(change, await changed.Task);
}
[Test]
public async Task FailedComparisonsAreRetriedWithBackoff()
{