Files
Febe/src/Febe.Core/Engine/CircuitBreaker.cs
T
Claude 0ceffffe50 Interfaccia: correzioni della revisione (web-1, web-2, web-3, web-4, web-5, web-6, web-7, web-9, web-10, web-11, web-12, web-13, web-14, web-15)
Il browser preso dalle pagine Browser e Collegamento si restituisce uscendo senza «Ho finito» (anche chiudendo o ricaricando la scheda); dopo «Ho finito» la vista remota si chiude e non si riprende il browser. Vista remota: AltGr/Option come testo, pulsante rilasciato anche fuori dalla vista. Mosaico e scheda account senza risposte vecchie né viste sovrascritte; nuovi elementi dopo la libreria vuota; intestazione sempre aggiornata. Indietro del browser con la richiesta delle modifiche non salvate; nulla resta registrato da una pagina lasciata mentre caricava; maniglie della fascia trascinabili. «Da una data» come giorno locale; niente «@null» nel riquadro di stato; orari futuri dei fermi «verso le». Prove nel browser vero (InterfaceTests).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
2026-10-14 01:38:13 +00:00

410 lines
17 KiB
C#

using System.Globalization;
using Febe.Core.Abstractions;
using Febe.Core.Domain;
using Febe.Core.Storage;
namespace Febe.Core.Engine;
/// <summary>
/// The circuit breaker («interruttore», spec §7.6).
/// <list type="bullet">
/// <item>Throttle: closes the session at once; stops 1 hour the first time, 6 hours the second within 24 hours,
/// 24 hours the third; then halved budgets for 48 hours. Reopens by itself.</item>
/// <item>Challenge and disconnection: stop everything and never restart alone; only <see cref="ResumeAsync"/>
/// (Alberto's «Riprendi», after fixing things in the browser) reopens, and the resume starts the gradual start.</item>
/// </list>
/// Every stop is an <c>incident</c> row and a line of the Activity timeline; the state is restored from the database
/// at start-up. <see cref="Report"/> is synchronous (signals arrive from capture threads): the state changes at once
/// and the rows are written in order in the background (<see cref="FlushAsync"/> waits for them).
/// </summary>
public sealed class CircuitBreaker : ICircuitBreaker
{
/// <summary>Fixed stop durations (spec §7.6: not editable in version 1).</summary>
public static readonly IReadOnlyList<TimeSpan> ThrottleStops = [TimeSpan.FromHours(1), TimeSpan.FromHours(6), TimeSpan.FromHours(24)];
public static readonly TimeSpan HalvedPeriod = TimeSpan.FromHours(48);
public static readonly TimeSpan EscalationWindow = TimeSpan.FromHours(24);
public static readonly TimeSpan ProposalWindow = TimeSpan.FromDays(7);
public const int ProposalThreshold = 3;
private readonly IFebeStorage _storage;
private readonly IClock _clock;
private readonly EngineRuntime _runtime;
private readonly EngineEvents? _events;
private readonly Lock _gate = new();
private readonly List<DateTimeOffset> _recentThrottles = [];
private readonly List<long> _openIncidents = [];
private BreakerState _state = BreakerState.Normal;
private Task _tail = Task.CompletedTask;
private CircuitBreaker(IFebeStorage storage, IClock clock, EngineRuntime runtime, EngineEvents? events)
{
_storage = storage;
_clock = clock;
_runtime = runtime;
_events = events;
}
/// <summary>Raised after every change of state (outside the internal lock). The engine closes the session on it.</summary>
public event EventHandler<BreakerState>? StateChanged;
/// <summary>Current state; a throttle stop that has expired turns into Closed here.</summary>
public BreakerState State
{
get
{
BreakerState state;
bool changed;
lock (_gate)
{
changed = ExpireLocked(_clock.Now);
state = _state;
}
if (changed)
{
StateChanged?.Invoke(this, state);
}
return state;
}
}
/// <summary>Restores the state from the incidents table (open stops, recent throttles, halved budgets).</summary>
public static async Task<CircuitBreaker> LoadAsync(
IFebeStorage storage,
IClock clock,
EngineRuntime runtime,
EngineEvents? events = null,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(storage);
ArgumentNullException.ThrowIfNull(clock);
ArgumentNullException.ThrowIfNull(runtime);
var breaker = new CircuitBreaker(storage, clock, runtime, events);
var now = clock.Now;
var recent = await storage.Incidents.ListAsync(now - ProposalWindow - EscalationWindow, now.AddSeconds(1), ct).ConfigureAwait(false);
DateTimeOffset? halvedUntil = null;
foreach (var incident in recent.Where(i => i.Level == SignalLevel.Throttle))
{
breaker._recentThrottles.Add(incident.At);
if (incident.StoppedUntil is { } until && until + HalvedPeriod > now && (halvedUntil is null || until + HalvedPeriod > halvedUntil))
{
halvedUntil = until + HalvedPeriod;
}
}
var state = new BreakerState(BreakerStatus.Closed, null, null, halvedUntil);
foreach (var incident in await storage.Incidents.ListOpenAsync(ct).ConfigureAwait(false))
{
var signal = new Signal(incident.Level, incident.Signal, incident.Url, incident.At);
switch (incident.Level)
{
case SignalLevel.Challenge:
state = state with { Status = BreakerStatus.Challenge, Until = null, LastSignal = signal };
breaker._openIncidents.Add(incident.Id);
break;
case SignalLevel.Disconnect:
state = state with { Status = BreakerStatus.Disconnected, Until = null, LastSignal = signal };
breaker._openIncidents.Add(incident.Id);
break;
case SignalLevel.Throttle when incident.StoppedUntil is { } until && until > now:
if (state.Status == BreakerStatus.Closed)
{
state = state with { Status = BreakerStatus.Throttled, Until = until, LastSignal = signal };
}
breaker._openIncidents.Add(incident.Id);
break;
default:
// An expired throttle left open by a previous process: resolved at its planned end.
await storage.Incidents.ResolveAsync(incident.Id, incident.StoppedUntil ?? now, ct).ConfigureAwait(false);
break;
}
}
breaker._state = state;
return breaker;
}
public void Report(Signal signal) => _ = ReportAsync(signal);
/// <summary>Same as <see cref="Report"/>, returning when the incident and the event are stored.</summary>
public Task ReportAsync(Signal signal, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(signal);
BreakerState state;
Task persisted;
lock (_gate)
{
var now = _clock.Now;
ExpireLocked(now);
var sessionId = _runtime.CurrentSession?.SessionId;
switch (signal.Level)
{
case SignalLevel.Throttle:
if (_state.Status != BreakerStatus.Closed)
{
// Already stopped: a late signal of the same stop does not escalate it.
return _tail;
}
_recentThrottles.RemoveAll(t => t < now - ProposalWindow);
var within24Hours = _recentThrottles.Count(t => t > now - EscalationWindow);
var stop = ThrottleStops[Math.Min(within24Hours, ThrottleStops.Count - 1)];
_recentThrottles.Add(now);
var until = now + stop;
_state = new BreakerState(BreakerStatus.Throttled, until, signal, until + HalvedPeriod);
var withinWeek = _recentThrottles.Count(t => t > now - ProposalWindow);
persisted = EnqueueLocked(c => PersistThrottleAsync(signal, now, until, stop, sessionId, withinWeek, c));
break;
case SignalLevel.Challenge:
case SignalLevel.Disconnect:
var status = signal.Level == SignalLevel.Challenge ? BreakerStatus.Challenge : BreakerStatus.Disconnected;
if (_state.Status == status)
{
return _tail;
}
var wasThrottled = _state.Status == BreakerStatus.Throttled;
_state = new BreakerState(status, null, signal, _state.HalvedBudgetsUntil);
persisted = EnqueueLocked(c => PersistManualStopAsync(signal, now, wasThrottled, sessionId, c));
break;
default:
throw new ArgumentOutOfRangeException(nameof(signal), signal.Level, null);
}
state = _state;
}
StateChanged?.Invoke(this, state);
return persisted;
}
/// <summary>
/// «Riprendi» after a verification or a disconnection, once Alberto fixed things in the browser. Resolves the open
/// incidents and restarts the gradual start. False when the breaker is not in such a stop (a throttle stop has a
/// fixed duration and cannot be shortened).
/// </summary>
public async Task<bool> ResumeAsync(CancellationToken ct = default)
{
BreakerState state;
Task persisted;
lock (_gate)
{
var now = _clock.Now;
if (_state.Status is not (BreakerStatus.Challenge or BreakerStatus.Disconnected))
{
return false;
}
var previous = _state.Status;
_state = _state with { Status = BreakerStatus.Closed, Until = null };
_runtime.RampStart = now;
persisted = EnqueueLocked(c => PersistResumeAsync(previous, now, c));
state = _state;
}
await persisted.WaitAsync(ct).ConfigureAwait(false);
StateChanged?.Invoke(this, state);
return true;
}
/// <summary>Waits until every incident and event reported so far is stored.</summary>
public Task FlushAsync()
{
lock (_gate)
{
return _tail;
}
}
private bool ExpireLocked(DateTimeOffset now)
{
if (_state.Status != BreakerStatus.Throttled || _state.Until is not { } until || now < until)
{
return false;
}
_state = _state with { Status = BreakerStatus.Closed, Until = null };
var halvedUntil = _state.HalvedBudgetsUntil;
EnqueueLocked(ct => PersistThrottleEndAsync(until, halvedUntil, ct));
return true;
}
private Task EnqueueLocked(Func<CancellationToken, Task> work)
{
// Rows are written strictly in order: an incident is opened before it is resolved.
_tail = _tail.ContinueWith(_ => work(CancellationToken.None), CancellationToken.None, TaskContinuationOptions.None, TaskScheduler.Default).Unwrap();
return _tail;
}
private async Task PersistThrottleAsync(Signal signal, DateTimeOffset now, DateTimeOffset until, TimeSpan stop, long? sessionId, int withinWeek, CancellationToken ct)
{
var incident = await _storage.Incidents.OpenAsync(
new Incident
{
At = signal.At,
Level = SignalLevel.Throttle,
Signal = signal.Code,
Url = signal.Url,
StoppedUntil = until,
SessionId = sessionId,
},
ct).ConfigureAwait(false);
lock (_gate)
{
_openIncidents.Add(incident.Id);
}
if (_events is null)
{
return;
}
var zone = _clock.LocalZone;
await _events.WriteAsync(
EventSeverity.Warning,
EngineEventCodes.ThrottleStop,
$"Instagram ha chiesto di rallentare (segnale {signal.Code}). Ho chiuso la sessione e mi fermo per {ItalianText.Duration(stop)}: "
+ $"riparto {ItalianText.Around(until, now, zone)}. Poi i tetti restano dimezzati per 48 ore.",
sessionId: sessionId,
ct: ct).ConfigureAwait(false);
if (withinWeek >= ProposalThreshold)
{
var profile = _runtime.Profile.Kind;
var proposal = profile is CautionProfileKind.Prudent
? "Tre rallentamenti in sette giorni anche con il profilo «Prudente»: conviene ridurre gli account seguiti o abbassare le loro frequenze."
: $"Tre rallentamenti in sette giorni con il profilo «{ItalianText.Profile(profile)}»: conviene passare al profilo «{ItalianText.Profile(MorePrudent(profile))}» o ridurre gli account seguiti.";
await _events.WriteAsync(EventSeverity.Warning, EngineEventCodes.ProposeMorePrudence, proposal, ct: ct).ConfigureAwait(false);
}
}
private async Task PersistManualStopAsync(Signal signal, DateTimeOffset now, bool wasThrottled, long? sessionId, CancellationToken ct)
{
if (wasThrottled)
{
await ResolveOpenAsync(now, ct).ConfigureAwait(false);
}
var incident = await _storage.Incidents.OpenAsync(
new Incident
{
At = signal.At,
Level = signal.Level,
Signal = signal.Code,
Url = signal.Url,
SessionId = sessionId,
},
ct).ConfigureAwait(false);
lock (_gate)
{
_openIncidents.Add(incident.Id);
}
if (_events is null)
{
return;
}
var at = ItalianText.At(now, _clock.LocalZone);
var message = signal.Level == SignalLevel.Challenge
? $"Instagram chiede una verifica (segnale {signal.Code}). Ho fermato tutto {at} e non riparto da solo: apri il browser, completa la verifica a mano, poi premi «Riprendi»."
: $"La sessione di Instagram non c'è più (segnale {signal.Code}). Ho fermato tutto {at} e non provo a rientrare: apri il browser, accedi a mano, poi premi «Riprendi».";
await _events.WriteAsync(
EventSeverity.Error,
signal.Level == SignalLevel.Challenge ? EngineEventCodes.ChallengeStop : EngineEventCodes.DisconnectStop,
message,
sessionId: sessionId,
ct: ct).ConfigureAwait(false);
}
private async Task PersistThrottleEndAsync(DateTimeOffset until, DateTimeOffset? halvedUntil, CancellationToken ct)
{
await ResolveOpenAsync(until, ct).ConfigureAwait(false);
if (_events is null)
{
return;
}
var tail = halvedUntil is { } h
? $" Riprendo con i tetti dimezzati, di nuovo pieni {ItalianText.Around(h, until, _clock.LocalZone)}."
: " Riprendo con i tetti normali.";
await _events.WriteAsync(EventSeverity.Info, EngineEventCodes.ThrottleEnd, "Fine del fermo per rallentamento." + tail, ct: ct).ConfigureAwait(false);
}
private async Task PersistResumeAsync(BreakerStatus previous, DateTimeOffset now, CancellationToken ct)
{
await ResolveOpenAsync(now, ct).ConfigureAwait(false);
await _storage.Settings.SetAsync(
EngineSettingKeys.RampStart,
now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture),
ct).ConfigureAwait(false);
if (_events is null)
{
return;
}
var profile = _runtime.Profile;
var what = previous == BreakerStatus.Challenge ? "Ripresa dopo la verifica." : "Ripresa dopo il nuovo accesso.";
var ramp = profile.GradualStart && profile.RampUpDays > 0
? $" I tetti ripartono dalla metà e tornano pieni in {ItalianText.Count(profile.RampUpDays, "giorno", "giorni")}."
: string.Empty;
await _events.WriteAsync(EventSeverity.Success, EngineEventCodes.Resumed, what + ramp, ct: ct).ConfigureAwait(false);
}
private async Task ResolveOpenAsync(DateTimeOffset at, CancellationToken ct)
{
long[] open;
lock (_gate)
{
open = [.. _openIncidents];
_openIncidents.Clear();
}
foreach (var id in open)
{
await _storage.Incidents.ResolveAsync(id, at, ct).ConfigureAwait(false);
}
}
private static CautionProfileKind MorePrudent(CautionProfileKind kind) => kind switch
{
CautionProfileKind.Fast => CautionProfileKind.Balanced,
_ => CautionProfileKind.Prudent,
};
}
/// <summary>Stable codes of the engine's timeline events.</summary>
public static class EngineEventCodes
{
public const string ThrottleStop = "fermo-rallentamento";
public const string ThrottleEnd = "fine-fermo";
public const string ChallengeStop = "verifica-richiesta";
public const string DisconnectStop = "disconnessione";
public const string Resumed = "ripresa";
public const string ProposeMorePrudence = "proposta-prudenza";
public const string Paused = "pausa";
public const string Unpaused = "fine-pausa";
public const string BudgetReached = "tetto-raggiunto";
public const string SessionEnded = "sessione";
public const string BackfillCompleted = "storico-completato";
public const string BackfillWaiting = "storico-in-attesa";
public const string RecipeSuspended = "ricetta-sospesa";
public const string RecipeResumed = "ricetta-riattivata";
public const string FormatUnrecognized = "formato-non-riconosciuto";
public const string TargetNotFound = "bersaglio-non-trovato";
public const string AccountUnreachable = "account-non-raggiungibile";
public const string AccountPaused = "account-in-pausa";
public const string ManualCheck = "controlla-ora";
public const string RecipeFailed = "ricetta-fallita";
public const string SettingsChanged = "impostazioni-motore";
/// <summary>Prefix of the per-recipe lines («Appena fatto»): "ricetta." + recipe kind.</summary>
public const string RecipePrefix = "ricetta.";
}