fix(events): card reports cannot crowd the event queues or hold an import
The server publishes one account's card progress at most once a second (sooner reports are taken and dropped) and at most 120 card events of an account a minute (429 rate_limited beyond), so an upload-only account can no longer push job and sequence events out of every client's queue. On the client, the end of a card import waits at most two seconds for its reports to reach the server, then cancels whatever is still on its way, so a cancelled import or one that lost its server ends at once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
5 files changed
+146
-12
No files matched your search
@@ -107,15 +107,18 @@ public sealed class ServerImportSink(TitanoClient client, string stagingDirector
|
||||
/// The reports of one card import into the server (BRIEF §14.3 <c>cardEvent</c>, §14.4), sent one at a time and in order:
|
||||
/// progress at most once a second, and a progress report still waiting is replaced by the next report, so a slow or
|
||||
/// unreachable server never piles them up. A report that fails is logged and the import goes on; a server without the
|
||||
/// endpoint (404) gets no more reports.
|
||||
/// endpoint (404) gets no more reports. Once the import is over its end gets two seconds to reach the server, then
|
||||
/// whatever is still on its way is given up.
|
||||
/// </summary>
|
||||
/// <param name="total">Photos of the plan, skipped ones included (like <see cref="ImportProgress.ItemsTotal"/>).</param>
|
||||
public sealed class CardImportReporter(TitanoClient client, Guid importId, string card, int total, TimeProvider? time = null)
|
||||
{
|
||||
private static readonly TimeSpan ProgressInterval = TimeSpan.FromSeconds(1);
|
||||
private static readonly TimeSpan SendTimeout = TimeSpan.FromSeconds(10);
|
||||
private static readonly TimeSpan FinishWait = TimeSpan.FromSeconds(2);
|
||||
|
||||
private readonly TimeProvider _time = time ?? TimeProvider.System;
|
||||
private readonly CancellationTokenSource _stop = new();
|
||||
private readonly Lock _gate = new();
|
||||
private readonly List<CardImportReport> _queue = [];
|
||||
private Task _sending = Task.CompletedTask;
|
||||
@@ -144,18 +147,33 @@ public sealed class CardImportReporter(TitanoClient client, Guid importId, strin
|
||||
|
||||
/// <summary>
|
||||
/// Reports the end: every photo handled when the import gave an <paramref name="outcome"/>, else as far as its progress
|
||||
/// got. Completes when the reports still queued are sent (each gives up after 10 s).
|
||||
/// got. Waits at most two seconds for the reports still queued, then stops sending: an import that was cancelled or
|
||||
/// lost its server ends at once instead of waiting for a server that does not answer.
|
||||
/// </summary>
|
||||
public Task FinishedAsync(ImportOutcome? outcome)
|
||||
public async Task FinishedAsync(ImportOutcome? outcome)
|
||||
{
|
||||
int done, failed;
|
||||
Task sending;
|
||||
lock (_gate)
|
||||
{
|
||||
done = outcome is null ? _latest?.ItemsDone ?? 0 : total;
|
||||
failed = outcome?.Failures.Count ?? _latest?.Failed ?? 0;
|
||||
}
|
||||
Post(CardImportStates.Finished, done, failed);
|
||||
lock (_gate) return _sending;
|
||||
lock (_gate) sending = _sending;
|
||||
try
|
||||
{
|
||||
await sending.WaitAsync(FinishWait, _time).ConfigureAwait(false);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
Log.Info("import", $"The last reports of the import from {card} were given up: the server did not take them in time");
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Nothing is sent once the import is over: its client goes next.
|
||||
await _stop.CancelAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private void Post(string state, int done, int failed)
|
||||
@@ -180,7 +198,7 @@ public sealed class CardImportReporter(TitanoClient client, Guid importId, strin
|
||||
CardImportReport report;
|
||||
lock (_gate)
|
||||
{
|
||||
if (_queue.Count == 0 || _unsupported)
|
||||
if (_queue.Count == 0 || _unsupported || _stop.IsCancellationRequested)
|
||||
{
|
||||
_queue.Clear();
|
||||
_draining = false;
|
||||
@@ -197,9 +215,14 @@ public sealed class CardImportReporter(TitanoClient client, Guid importId, strin
|
||||
{
|
||||
try
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(SendTimeout);
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(_stop.Token);
|
||||
timeout.CancelAfter(SendTimeout);
|
||||
await client.ReportCardAsync(report, timeout.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (_stop.IsCancellationRequested)
|
||||
{
|
||||
// The import is over and did not wait for this one.
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.MethodNotAllowed)
|
||||
{
|
||||
// A server from before card events.
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
using Titano.Core.Model;
|
||||
using Titano.Protocol;
|
||||
using Titano.Server.Events;
|
||||
using Titano.Server.Security;
|
||||
using Titano.Server.Storage;
|
||||
|
||||
@@ -37,8 +36,11 @@ public static class UploadEndpoints
|
||||
statusCode: r.AlreadyExists ? StatusCodes.Status200OK : StatusCodes.Status201Created));
|
||||
}
|
||||
|
||||
/// <summary>A card import's start, progress or end, published as a <see cref="CardEvent"/> naming who imports it; 204.</summary>
|
||||
private static IResult ReportCard(HttpContext context, Caller caller, CardImportReport report, EventHub events)
|
||||
/// <summary>
|
||||
/// A card import's start, progress or end, published as a <see cref="CardEvent"/> naming who imports it; 204, also for
|
||||
/// progress dropped because it came within a second of the last (<see cref="UploadService.PublishCard"/>).
|
||||
/// </summary>
|
||||
private static IResult ReportCard(HttpContext context, Caller caller, CardImportReport report, UploadService service)
|
||||
{
|
||||
if (report.ImportId == Guid.Empty) return ApiErrors.BadRequest(context, "bad_request", "importId");
|
||||
var card = report.Card?.Trim();
|
||||
@@ -46,8 +48,8 @@ public static class UploadEndpoints
|
||||
if (!CardImportStates.IsKnown(report.State)) return ApiErrors.BadRequest(context, "bad_request", "state");
|
||||
if (report.Total < 0 || report.Done < 0 || report.Done > report.Total || report.Failed < 0 || report.Failed > report.Total)
|
||||
return ApiErrors.BadRequest(context, "bad_request", "done");
|
||||
events.Publish(new CardEvent(report.ImportId, card, report.State, report.Done, report.Total, report.Failed, caller.User.Username, caller.ClientName));
|
||||
return Results.NoContent();
|
||||
return service.PublishCard(caller, new CardEvent(report.ImportId, card, report.State, report.Done, report.Total, report.Failed, caller.User.Username,
|
||||
caller.ClientName)).ToResult(context, _ => Results.NoContent());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -36,7 +36,14 @@ public sealed class UploadService(
|
||||
/// <summary>Space kept free on the upload volume beyond the announced size.</summary>
|
||||
private const long DiskReserve = 64L * 1024 * 1024;
|
||||
|
||||
/// <summary>Card events one account may publish per minute: progress once a second leaves room for every start and end.</summary>
|
||||
private const int CardEventsPerMinute = 120;
|
||||
|
||||
private static readonly TimeSpan CardProgressInterval = TimeSpan.FromSeconds(1);
|
||||
private static readonly TimeSpan CardEventWindow = TimeSpan.FromMinutes(1);
|
||||
|
||||
private readonly ConcurrentDictionary<Guid, int> _chunksInFlight = new();
|
||||
private readonly ConcurrentDictionary<Guid, CardEvents> _cardEvents = new();
|
||||
|
||||
public async Task<Outcome<StartUploadResponse>> StartAsync(Caller caller, StartUploadRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -204,6 +211,32 @@ public sealed class UploadService(
|
||||
return new CompleteUploadResponse(result.FrameId, result.SequenceId, result.Index);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tells every connected client about a card import (BRIEF §14.3 <c>cardEvent</c>), so one account cannot crowd their
|
||||
/// event queues: its progress goes out at most once a second (a report sooner is dropped, the next one replaces it as
|
||||
/// on the client), and at most 120 of its events a minute (429 beyond). False when the report was dropped.
|
||||
/// </summary>
|
||||
public Outcome<bool> PublishCard(Caller caller, CardEvent card)
|
||||
{
|
||||
var now = time.GetUtcNow();
|
||||
var sent = _cardEvents.GetOrAdd(caller.Id, _ => new CardEvents());
|
||||
lock (sent.Gate)
|
||||
{
|
||||
var progress = card.State == CardImportStates.Progress;
|
||||
if (progress && now - sent.LastProgress < CardProgressInterval) return false;
|
||||
while (sent.Recent.Count > 0 && now - sent.Recent.Peek() >= CardEventWindow) sent.Recent.Dequeue();
|
||||
if (sent.Recent.Count >= CardEventsPerMinute)
|
||||
{
|
||||
var wait = Math.Max(1, (int)Math.Ceiling((CardEventWindow - (now - sent.Recent.Peek())).TotalSeconds));
|
||||
return Fail<bool>(429, "rate_limited", wait.ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
sent.Recent.Enqueue(now);
|
||||
if (progress) sent.LastProgress = now;
|
||||
}
|
||||
events.Publish(card);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies a fully received upload to the archive staging folder and checks its SHA-256. On a mismatch the upload
|
||||
/// fails, its files are deleted and null is returned; the client starts over.
|
||||
@@ -256,4 +289,12 @@ public sealed class UploadService(
|
||||
}
|
||||
|
||||
private static Outcome<T> Fail<T>(int status, string code, string? details = null) => Outcome<T>.Fail(status, code, details);
|
||||
|
||||
/// <summary>The card events one account published lately.</summary>
|
||||
private sealed class CardEvents
|
||||
{
|
||||
public Lock Gate { get; } = new();
|
||||
public Queue<DateTimeOffset> Recent { get; } = new();
|
||||
public DateTimeOffset LastProgress { get; set; } = DateTimeOffset.MinValue;
|
||||
}
|
||||
}
|
||||
@@ -172,8 +172,30 @@ public sealed class ServerImportSinkTests(TestContext context)
|
||||
|
||||
// The start was on its way; progress 10 waited and was replaced by 30 (20 came within the second), then by the end.
|
||||
Assert.SequenceEqual([(CardImportStates.Started, 0), (CardImportStates.Finished, 30)], server.Cards.Select(r => (r.State, r.Done)));
|
||||
// Nothing follows the end, not even progress a second later.
|
||||
time.Advance(TimeSpan.FromSeconds(1));
|
||||
reporter.Report(Progress(40));
|
||||
Assert.Count(2, server.Cards);
|
||||
await reporter.FinishedAsync(null).WaitAsync(TimeSpan.FromSeconds(10));
|
||||
Assert.Equal(2, server.CardAttempts);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImportOverDoesNotWaitForAServerThatDoesNotAnswer()
|
||||
{
|
||||
var server = new FakeUploadServer(chunkSize: 4096) { CardGate = new TaskCompletionSource().Task };
|
||||
using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
|
||||
var reporter = new CardImportReporter(client, Guid.CreateVersion7(), "EOS R5 · card A", 100);
|
||||
|
||||
reporter.Started();
|
||||
await server.CardArrived.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
||||
var clock = System.Diagnostics.Stopwatch.StartNew();
|
||||
await reporter.FinishedAsync(null).WaitAsync(TimeSpan.FromSeconds(8));
|
||||
|
||||
// The start hangs (each report would wait 10 s): the end gives both up after two seconds.
|
||||
Assert.True(clock.Elapsed < TimeSpan.FromSeconds(6), $"finished after {clock.Elapsed}");
|
||||
Assert.Empty(server.Cards);
|
||||
for (var i = 0; i < 25 && server.CardAttempts < 2; i++) await Task.Delay(20);
|
||||
Assert.Equal(1, server.CardAttempts, "the end was never sent once given up");
|
||||
}
|
||||
|
||||
private static ImportProgress Progress(int done) => new(done, 100, done * 10L, 1000, done, 0, 0, null);
|
||||
|
||||
@@ -142,6 +142,52 @@ public sealed class EventTests(TestContext context)
|
||||
await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OneAccountsCardReportsCannotCrowdTheEventQueues()
|
||||
{
|
||||
var clock = new ManualTimeProvider();
|
||||
await using var server = await TestServer.StartAsync(context, services: s => s.AddSingleton<TimeProvider>(clock));
|
||||
var (uploader, _, _) = await server.UserAsync("studio-pc", "uploader");
|
||||
var (other, _, _) = await server.UserAsync("roof-pc", "uploader");
|
||||
using var subscription = server.Services.GetRequiredService<Titano.Server.Events.EventHub>().Subscribe();
|
||||
var importId = Guid.CreateVersion7();
|
||||
|
||||
// Progress goes out at most once a second; the reports in between are taken and dropped.
|
||||
for (var done = 1; done <= 5; done++) await ReportCardAsync(uploader, new CardImportReport(importId, "card", CardImportStates.Progress, done, 100, 0));
|
||||
clock.Advance(TimeSpan.FromSeconds(1));
|
||||
await ReportCardAsync(uploader, new CardImportReport(importId, "card", CardImportStates.Progress, 6, 100, 0));
|
||||
Assert.SequenceEqual([1, 6], CardEvents(subscription).Select(e => e.Done));
|
||||
|
||||
// Starts and ends always go out, up to 120 events of one account a minute.
|
||||
for (var i = 0; i < 118; i++) await ReportCardAsync(uploader, new CardImportReport(Guid.CreateVersion7(), "card", CardImportStates.Started, 0, 100, 0));
|
||||
using (var refused = await uploader.PostAsJsonAsync("/api/v1/uploads/cards",
|
||||
new CardImportReport(importId, "card", CardImportStates.Finished, 100, 100, 0), ProtocolJson.Default.CardImportReport))
|
||||
Assert.Equal("59", (await refused.ExpectErrorAsync(HttpStatusCode.TooManyRequests, "rate_limited")).Details);
|
||||
await ReportCardAsync(other, new CardImportReport(Guid.CreateVersion7(), "card", CardImportStates.Started, 0, 100, 0));
|
||||
Assert.Count(119, CardEvents(subscription));
|
||||
|
||||
clock.Advance(TimeSpan.FromMinutes(1));
|
||||
await ReportCardAsync(uploader, new CardImportReport(importId, "card", CardImportStates.Finished, 100, 100, 0));
|
||||
Assert.SequenceEqual([CardImportStates.Finished], CardEvents(subscription).Select(e => e.State));
|
||||
}
|
||||
|
||||
private static async Task ReportCardAsync(HttpClient client, CardImportReport report)
|
||||
{
|
||||
using var response = await client.PostAsJsonAsync("/api/v1/uploads/cards", report, ProtocolJson.Default.CardImportReport);
|
||||
await response.ExpectAsync(HttpStatusCode.NoContent);
|
||||
}
|
||||
|
||||
/// <summary>The card events published since the last call.</summary>
|
||||
private static List<CardEvent> CardEvents(Titano.Server.Events.EventHub.Subscription subscription)
|
||||
{
|
||||
var events = new List<CardEvent>();
|
||||
while (subscription.Reader.TryRead(out var e))
|
||||
{
|
||||
if (e is CardEvent card) events.Add(card);
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
private static async Task WaitForSubscriberAsync(TestServer server)
|
||||
{
|
||||
var hub = server.Services.GetRequiredService<Titano.Server.Events.EventHub>();
|
||||
|
||||
Reference in new issue
Block a user