From 1de6717dda6133bdd8a83d456aa3344e4823ccfa Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 10:56:18 +0000 Subject: [PATCH] fix(events): card reports cannot crowd the event queues or hold an import The server publishes one account's card progress at most once a second (sooner reports are taken and dropped) and at most 120 card events of an account a minute (429 rate_limited beyond), so an upload-only account can no longer push job and sequence events out of every client's queue. On the client, the end of a card import waits at most two seconds for its reports to reach the server, then cancels whatever is still on its way, so a cancelled import or one that lost its server ends at once. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.App/Import/ServerImportSink.cs | 35 +++++++++++--- src/Titano.Server/Api/UploadEndpoints.cs | 12 +++-- src/Titano.Server/Storage/UploadService.cs | 41 +++++++++++++++++ .../Import/ServerImportSinkTests.cs | 24 +++++++++- .../Titano.Server.Tests/Events/EventTests.cs | 46 +++++++++++++++++++ 5 files changed, 146 insertions(+), 12 deletions(-) diff --git a/src/Titano.App/Import/ServerImportSink.cs b/src/Titano.App/Import/ServerImportSink.cs index bd9b5952..f05584fa 100644 --- a/src/Titano.App/Import/ServerImportSink.cs +++ b/src/Titano.App/Import/ServerImportSink.cs @@ -107,15 +107,18 @@ public sealed class ServerImportSink(TitanoClient client, string stagingDirector /// The reports of one card import into the server (BRIEF §14.3 cardEvent, §14.4), sent one at a time and in order: /// progress at most once a second, and a progress report still waiting is replaced by the next report, so a slow or /// unreachable server never piles them up. A report that fails is logged and the import goes on; a server without the -/// endpoint (404) gets no more reports. +/// endpoint (404) gets no more reports. Once the import is over its end gets two seconds to reach the server, then +/// whatever is still on its way is given up. /// /// Photos of the plan, skipped ones included (like ). public sealed class CardImportReporter(TitanoClient client, Guid importId, string card, int total, TimeProvider? time = null) { private static readonly TimeSpan ProgressInterval = TimeSpan.FromSeconds(1); private static readonly TimeSpan SendTimeout = TimeSpan.FromSeconds(10); + private static readonly TimeSpan FinishWait = TimeSpan.FromSeconds(2); private readonly TimeProvider _time = time ?? TimeProvider.System; + private readonly CancellationTokenSource _stop = new(); private readonly Lock _gate = new(); private readonly List _queue = []; private Task _sending = Task.CompletedTask; @@ -144,18 +147,33 @@ public sealed class CardImportReporter(TitanoClient client, Guid importId, strin /// /// Reports the end: every photo handled when the import gave an , else as far as its progress - /// got. Completes when the reports still queued are sent (each gives up after 10 s). + /// got. Waits at most two seconds for the reports still queued, then stops sending: an import that was cancelled or + /// lost its server ends at once instead of waiting for a server that does not answer. /// - public Task FinishedAsync(ImportOutcome? outcome) + public async Task FinishedAsync(ImportOutcome? outcome) { int done, failed; + Task sending; lock (_gate) { done = outcome is null ? _latest?.ItemsDone ?? 0 : total; failed = outcome?.Failures.Count ?? _latest?.Failed ?? 0; } Post(CardImportStates.Finished, done, failed); - lock (_gate) return _sending; + lock (_gate) sending = _sending; + try + { + await sending.WaitAsync(FinishWait, _time).ConfigureAwait(false); + } + catch (TimeoutException) + { + Log.Info("import", $"The last reports of the import from {card} were given up: the server did not take them in time"); + } + finally + { + // Nothing is sent once the import is over: its client goes next. + await _stop.CancelAsync().ConfigureAwait(false); + } } private void Post(string state, int done, int failed) @@ -180,7 +198,7 @@ public sealed class CardImportReporter(TitanoClient client, Guid importId, strin CardImportReport report; lock (_gate) { - if (_queue.Count == 0 || _unsupported) + if (_queue.Count == 0 || _unsupported || _stop.IsCancellationRequested) { _queue.Clear(); _draining = false; @@ -197,9 +215,14 @@ public sealed class CardImportReporter(TitanoClient client, Guid importId, strin { try { - using var timeout = new CancellationTokenSource(SendTimeout); + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(_stop.Token); + timeout.CancelAfter(SendTimeout); await client.ReportCardAsync(report, timeout.Token).ConfigureAwait(false); } + catch (OperationCanceledException) when (_stop.IsCancellationRequested) + { + // The import is over and did not wait for this one. + } catch (TitanoApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.MethodNotAllowed) { // A server from before card events. diff --git a/src/Titano.Server/Api/UploadEndpoints.cs b/src/Titano.Server/Api/UploadEndpoints.cs index 5f0c4f45..56755b08 100644 --- a/src/Titano.Server/Api/UploadEndpoints.cs +++ b/src/Titano.Server/Api/UploadEndpoints.cs @@ -1,6 +1,5 @@ using Titano.Core.Model; using Titano.Protocol; -using Titano.Server.Events; using Titano.Server.Security; using Titano.Server.Storage; @@ -37,8 +36,11 @@ public static class UploadEndpoints statusCode: r.AlreadyExists ? StatusCodes.Status200OK : StatusCodes.Status201Created)); } - /// A card import's start, progress or end, published as a naming who imports it; 204. - private static IResult ReportCard(HttpContext context, Caller caller, CardImportReport report, EventHub events) + /// + /// A card import's start, progress or end, published as a naming who imports it; 204, also for + /// progress dropped because it came within a second of the last (). + /// + private static IResult ReportCard(HttpContext context, Caller caller, CardImportReport report, UploadService service) { if (report.ImportId == Guid.Empty) return ApiErrors.BadRequest(context, "bad_request", "importId"); var card = report.Card?.Trim(); @@ -46,8 +48,8 @@ public static class UploadEndpoints if (!CardImportStates.IsKnown(report.State)) return ApiErrors.BadRequest(context, "bad_request", "state"); if (report.Total < 0 || report.Done < 0 || report.Done > report.Total || report.Failed < 0 || report.Failed > report.Total) return ApiErrors.BadRequest(context, "bad_request", "done"); - events.Publish(new CardEvent(report.ImportId, card, report.State, report.Done, report.Total, report.Failed, caller.User.Username, caller.ClientName)); - return Results.NoContent(); + return service.PublishCard(caller, new CardEvent(report.ImportId, card, report.State, report.Done, report.Total, report.Failed, caller.User.Username, + caller.ClientName)).ToResult(context, _ => Results.NoContent()); } /// diff --git a/src/Titano.Server/Storage/UploadService.cs b/src/Titano.Server/Storage/UploadService.cs index dea51b7b..22f799bc 100644 --- a/src/Titano.Server/Storage/UploadService.cs +++ b/src/Titano.Server/Storage/UploadService.cs @@ -36,7 +36,14 @@ public sealed class UploadService( /// Space kept free on the upload volume beyond the announced size. private const long DiskReserve = 64L * 1024 * 1024; + /// Card events one account may publish per minute: progress once a second leaves room for every start and end. + private const int CardEventsPerMinute = 120; + + private static readonly TimeSpan CardProgressInterval = TimeSpan.FromSeconds(1); + private static readonly TimeSpan CardEventWindow = TimeSpan.FromMinutes(1); + private readonly ConcurrentDictionary _chunksInFlight = new(); + private readonly ConcurrentDictionary _cardEvents = new(); public async Task> StartAsync(Caller caller, StartUploadRequest request, CancellationToken cancellationToken) { @@ -204,6 +211,32 @@ public sealed class UploadService( return new CompleteUploadResponse(result.FrameId, result.SequenceId, result.Index); } + /// + /// Tells every connected client about a card import (BRIEF §14.3 cardEvent), so one account cannot crowd their + /// event queues: its progress goes out at most once a second (a report sooner is dropped, the next one replaces it as + /// on the client), and at most 120 of its events a minute (429 beyond). False when the report was dropped. + /// + public Outcome PublishCard(Caller caller, CardEvent card) + { + var now = time.GetUtcNow(); + var sent = _cardEvents.GetOrAdd(caller.Id, _ => new CardEvents()); + lock (sent.Gate) + { + var progress = card.State == CardImportStates.Progress; + if (progress && now - sent.LastProgress < CardProgressInterval) return false; + while (sent.Recent.Count > 0 && now - sent.Recent.Peek() >= CardEventWindow) sent.Recent.Dequeue(); + if (sent.Recent.Count >= CardEventsPerMinute) + { + var wait = Math.Max(1, (int)Math.Ceiling((CardEventWindow - (now - sent.Recent.Peek())).TotalSeconds)); + return Fail(429, "rate_limited", wait.ToString(CultureInfo.InvariantCulture)); + } + sent.Recent.Enqueue(now); + if (progress) sent.LastProgress = now; + } + events.Publish(card); + return true; + } + /// /// Copies a fully received upload to the archive staging folder and checks its SHA-256. On a mismatch the upload /// fails, its files are deleted and null is returned; the client starts over. @@ -256,4 +289,12 @@ public sealed class UploadService( } private static Outcome Fail(int status, string code, string? details = null) => Outcome.Fail(status, code, details); + + /// The card events one account published lately. + private sealed class CardEvents + { + public Lock Gate { get; } = new(); + public Queue Recent { get; } = new(); + public DateTimeOffset LastProgress { get; set; } = DateTimeOffset.MinValue; + } } diff --git a/tests/Titano.App.Tests/Import/ServerImportSinkTests.cs b/tests/Titano.App.Tests/Import/ServerImportSinkTests.cs index bfdb6380..b957ad21 100644 --- a/tests/Titano.App.Tests/Import/ServerImportSinkTests.cs +++ b/tests/Titano.App.Tests/Import/ServerImportSinkTests.cs @@ -172,8 +172,30 @@ public sealed class ServerImportSinkTests(TestContext context) // The start was on its way; progress 10 waited and was replaced by 30 (20 came within the second), then by the end. Assert.SequenceEqual([(CardImportStates.Started, 0), (CardImportStates.Finished, 30)], server.Cards.Select(r => (r.State, r.Done))); + // Nothing follows the end, not even progress a second later. + time.Advance(TimeSpan.FromSeconds(1)); reporter.Report(Progress(40)); - Assert.Count(2, server.Cards); + await reporter.FinishedAsync(null).WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Equal(2, server.CardAttempts); + } + + [Test] + public async Task AnImportOverDoesNotWaitForAServerThatDoesNotAnswer() + { + var server = new FakeUploadServer(chunkSize: 4096) { CardGate = new TaskCompletionSource().Task }; + using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" }; + var reporter = new CardImportReporter(client, Guid.CreateVersion7(), "EOS R5 · card A", 100); + + reporter.Started(); + await server.CardArrived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + var clock = System.Diagnostics.Stopwatch.StartNew(); + await reporter.FinishedAsync(null).WaitAsync(TimeSpan.FromSeconds(8)); + + // The start hangs (each report would wait 10 s): the end gives both up after two seconds. + Assert.True(clock.Elapsed < TimeSpan.FromSeconds(6), $"finished after {clock.Elapsed}"); + Assert.Empty(server.Cards); + for (var i = 0; i < 25 && server.CardAttempts < 2; i++) await Task.Delay(20); + Assert.Equal(1, server.CardAttempts, "the end was never sent once given up"); } private static ImportProgress Progress(int done) => new(done, 100, done * 10L, 1000, done, 0, 0, null); diff --git a/tests/Titano.Server.Tests/Events/EventTests.cs b/tests/Titano.Server.Tests/Events/EventTests.cs index 59201709..bf9a3fe5 100644 --- a/tests/Titano.Server.Tests/Events/EventTests.cs +++ b/tests/Titano.Server.Tests/Events/EventTests.cs @@ -142,6 +142,52 @@ public sealed class EventTests(TestContext context) await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); } + [Test] + public async Task OneAccountsCardReportsCannotCrowdTheEventQueues() + { + var clock = new ManualTimeProvider(); + await using var server = await TestServer.StartAsync(context, services: s => s.AddSingleton(clock)); + var (uploader, _, _) = await server.UserAsync("studio-pc", "uploader"); + var (other, _, _) = await server.UserAsync("roof-pc", "uploader"); + using var subscription = server.Services.GetRequiredService().Subscribe(); + var importId = Guid.CreateVersion7(); + + // Progress goes out at most once a second; the reports in between are taken and dropped. + for (var done = 1; done <= 5; done++) await ReportCardAsync(uploader, new CardImportReport(importId, "card", CardImportStates.Progress, done, 100, 0)); + clock.Advance(TimeSpan.FromSeconds(1)); + await ReportCardAsync(uploader, new CardImportReport(importId, "card", CardImportStates.Progress, 6, 100, 0)); + Assert.SequenceEqual([1, 6], CardEvents(subscription).Select(e => e.Done)); + + // Starts and ends always go out, up to 120 events of one account a minute. + for (var i = 0; i < 118; i++) await ReportCardAsync(uploader, new CardImportReport(Guid.CreateVersion7(), "card", CardImportStates.Started, 0, 100, 0)); + using (var refused = await uploader.PostAsJsonAsync("/api/v1/uploads/cards", + new CardImportReport(importId, "card", CardImportStates.Finished, 100, 100, 0), ProtocolJson.Default.CardImportReport)) + Assert.Equal("59", (await refused.ExpectErrorAsync(HttpStatusCode.TooManyRequests, "rate_limited")).Details); + await ReportCardAsync(other, new CardImportReport(Guid.CreateVersion7(), "card", CardImportStates.Started, 0, 100, 0)); + Assert.Count(119, CardEvents(subscription)); + + clock.Advance(TimeSpan.FromMinutes(1)); + await ReportCardAsync(uploader, new CardImportReport(importId, "card", CardImportStates.Finished, 100, 100, 0)); + Assert.SequenceEqual([CardImportStates.Finished], CardEvents(subscription).Select(e => e.State)); + } + + private static async Task ReportCardAsync(HttpClient client, CardImportReport report) + { + using var response = await client.PostAsJsonAsync("/api/v1/uploads/cards", report, ProtocolJson.Default.CardImportReport); + await response.ExpectAsync(HttpStatusCode.NoContent); + } + + /// The card events published since the last call. + private static List CardEvents(Titano.Server.Events.EventHub.Subscription subscription) + { + var events = new List(); + while (subscription.Reader.TryRead(out var e)) + { + if (e is CardEvent card) events.Add(card); + } + return events; + } + private static async Task WaitForSubscriberAsync(TestServer server) { var hub = server.Services.GetRequiredService();