Il browser preso dalle pagine Browser e Collegamento si restituisce uscendo senza «Ho finito» (anche chiudendo o ricaricando la scheda); dopo «Ho finito» la vista remota si chiude e non si riprende il browser. Vista remota: AltGr/Option come testo, pulsante rilasciato anche fuori dalla vista. Mosaico e scheda account senza risposte vecchie né viste sovrascritte; nuovi elementi dopo la libreria vuota; intestazione sempre aggiornata. Indietro del browser con la richiesta delle modifiche non salvate; nulla resta registrato da una pagina lasciata mentre caricava; maniglie della fascia trascinabili. «Da una data» come giorno locale; niente «@null» nel riquadro di stato; orari futuri dei fermi «verso le». Prove nel browser vero (InterfaceTests). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
410 lines
17 KiB
C#
410 lines
17 KiB
C#
using System.Globalization;
|
|
using Febe.Core.Abstractions;
|
|
using Febe.Core.Domain;
|
|
using Febe.Core.Storage;
|
|
|
|
namespace Febe.Core.Engine;
|
|
|
|
/// <summary>
|
|
/// The circuit breaker («interruttore», spec §7.6).
|
|
/// <list type="bullet">
|
|
/// <item>Throttle: closes the session at once; stops 1 hour the first time, 6 hours the second within 24 hours,
|
|
/// 24 hours the third; then halved budgets for 48 hours. Reopens by itself.</item>
|
|
/// <item>Challenge and disconnection: stop everything and never restart alone; only <see cref="ResumeAsync"/>
|
|
/// (Alberto's «Riprendi», after fixing things in the browser) reopens, and the resume starts the gradual start.</item>
|
|
/// </list>
|
|
/// Every stop is an <c>incident</c> row and a line of the Activity timeline; the state is restored from the database
|
|
/// at start-up. <see cref="Report"/> is synchronous (signals arrive from capture threads): the state changes at once
|
|
/// and the rows are written in order in the background (<see cref="FlushAsync"/> waits for them).
|
|
/// </summary>
|
|
public sealed class CircuitBreaker : ICircuitBreaker
|
|
{
|
|
/// <summary>Fixed stop durations (spec §7.6: not editable in version 1).</summary>
|
|
public static readonly IReadOnlyList<TimeSpan> ThrottleStops = [TimeSpan.FromHours(1), TimeSpan.FromHours(6), TimeSpan.FromHours(24)];
|
|
|
|
public static readonly TimeSpan HalvedPeriod = TimeSpan.FromHours(48);
|
|
public static readonly TimeSpan EscalationWindow = TimeSpan.FromHours(24);
|
|
public static readonly TimeSpan ProposalWindow = TimeSpan.FromDays(7);
|
|
public const int ProposalThreshold = 3;
|
|
|
|
private readonly IFebeStorage _storage;
|
|
private readonly IClock _clock;
|
|
private readonly EngineRuntime _runtime;
|
|
private readonly EngineEvents? _events;
|
|
private readonly Lock _gate = new();
|
|
private readonly List<DateTimeOffset> _recentThrottles = [];
|
|
private readonly List<long> _openIncidents = [];
|
|
private BreakerState _state = BreakerState.Normal;
|
|
private Task _tail = Task.CompletedTask;
|
|
|
|
private CircuitBreaker(IFebeStorage storage, IClock clock, EngineRuntime runtime, EngineEvents? events)
|
|
{
|
|
_storage = storage;
|
|
_clock = clock;
|
|
_runtime = runtime;
|
|
_events = events;
|
|
}
|
|
|
|
/// <summary>Raised after every change of state (outside the internal lock). The engine closes the session on it.</summary>
|
|
public event EventHandler<BreakerState>? StateChanged;
|
|
|
|
/// <summary>Current state; a throttle stop that has expired turns into Closed here.</summary>
|
|
public BreakerState State
|
|
{
|
|
get
|
|
{
|
|
BreakerState state;
|
|
bool changed;
|
|
lock (_gate)
|
|
{
|
|
changed = ExpireLocked(_clock.Now);
|
|
state = _state;
|
|
}
|
|
|
|
if (changed)
|
|
{
|
|
StateChanged?.Invoke(this, state);
|
|
}
|
|
|
|
return state;
|
|
}
|
|
}
|
|
|
|
/// <summary>Restores the state from the incidents table (open stops, recent throttles, halved budgets).</summary>
|
|
public static async Task<CircuitBreaker> LoadAsync(
|
|
IFebeStorage storage,
|
|
IClock clock,
|
|
EngineRuntime runtime,
|
|
EngineEvents? events = null,
|
|
CancellationToken ct = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(storage);
|
|
ArgumentNullException.ThrowIfNull(clock);
|
|
ArgumentNullException.ThrowIfNull(runtime);
|
|
var breaker = new CircuitBreaker(storage, clock, runtime, events);
|
|
var now = clock.Now;
|
|
|
|
var recent = await storage.Incidents.ListAsync(now - ProposalWindow - EscalationWindow, now.AddSeconds(1), ct).ConfigureAwait(false);
|
|
DateTimeOffset? halvedUntil = null;
|
|
foreach (var incident in recent.Where(i => i.Level == SignalLevel.Throttle))
|
|
{
|
|
breaker._recentThrottles.Add(incident.At);
|
|
if (incident.StoppedUntil is { } until && until + HalvedPeriod > now && (halvedUntil is null || until + HalvedPeriod > halvedUntil))
|
|
{
|
|
halvedUntil = until + HalvedPeriod;
|
|
}
|
|
}
|
|
|
|
var state = new BreakerState(BreakerStatus.Closed, null, null, halvedUntil);
|
|
foreach (var incident in await storage.Incidents.ListOpenAsync(ct).ConfigureAwait(false))
|
|
{
|
|
var signal = new Signal(incident.Level, incident.Signal, incident.Url, incident.At);
|
|
switch (incident.Level)
|
|
{
|
|
case SignalLevel.Challenge:
|
|
state = state with { Status = BreakerStatus.Challenge, Until = null, LastSignal = signal };
|
|
breaker._openIncidents.Add(incident.Id);
|
|
break;
|
|
case SignalLevel.Disconnect:
|
|
state = state with { Status = BreakerStatus.Disconnected, Until = null, LastSignal = signal };
|
|
breaker._openIncidents.Add(incident.Id);
|
|
break;
|
|
case SignalLevel.Throttle when incident.StoppedUntil is { } until && until > now:
|
|
if (state.Status == BreakerStatus.Closed)
|
|
{
|
|
state = state with { Status = BreakerStatus.Throttled, Until = until, LastSignal = signal };
|
|
}
|
|
|
|
breaker._openIncidents.Add(incident.Id);
|
|
break;
|
|
default:
|
|
// An expired throttle left open by a previous process: resolved at its planned end.
|
|
await storage.Incidents.ResolveAsync(incident.Id, incident.StoppedUntil ?? now, ct).ConfigureAwait(false);
|
|
break;
|
|
}
|
|
}
|
|
|
|
breaker._state = state;
|
|
return breaker;
|
|
}
|
|
|
|
public void Report(Signal signal) => _ = ReportAsync(signal);
|
|
|
|
/// <summary>Same as <see cref="Report"/>, returning when the incident and the event are stored.</summary>
|
|
public Task ReportAsync(Signal signal, CancellationToken ct = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(signal);
|
|
BreakerState state;
|
|
Task persisted;
|
|
lock (_gate)
|
|
{
|
|
var now = _clock.Now;
|
|
ExpireLocked(now);
|
|
var sessionId = _runtime.CurrentSession?.SessionId;
|
|
switch (signal.Level)
|
|
{
|
|
case SignalLevel.Throttle:
|
|
if (_state.Status != BreakerStatus.Closed)
|
|
{
|
|
// Already stopped: a late signal of the same stop does not escalate it.
|
|
return _tail;
|
|
}
|
|
|
|
_recentThrottles.RemoveAll(t => t < now - ProposalWindow);
|
|
var within24Hours = _recentThrottles.Count(t => t > now - EscalationWindow);
|
|
var stop = ThrottleStops[Math.Min(within24Hours, ThrottleStops.Count - 1)];
|
|
_recentThrottles.Add(now);
|
|
var until = now + stop;
|
|
_state = new BreakerState(BreakerStatus.Throttled, until, signal, until + HalvedPeriod);
|
|
var withinWeek = _recentThrottles.Count(t => t > now - ProposalWindow);
|
|
persisted = EnqueueLocked(c => PersistThrottleAsync(signal, now, until, stop, sessionId, withinWeek, c));
|
|
break;
|
|
|
|
case SignalLevel.Challenge:
|
|
case SignalLevel.Disconnect:
|
|
var status = signal.Level == SignalLevel.Challenge ? BreakerStatus.Challenge : BreakerStatus.Disconnected;
|
|
if (_state.Status == status)
|
|
{
|
|
return _tail;
|
|
}
|
|
|
|
var wasThrottled = _state.Status == BreakerStatus.Throttled;
|
|
_state = new BreakerState(status, null, signal, _state.HalvedBudgetsUntil);
|
|
persisted = EnqueueLocked(c => PersistManualStopAsync(signal, now, wasThrottled, sessionId, c));
|
|
break;
|
|
|
|
default:
|
|
throw new ArgumentOutOfRangeException(nameof(signal), signal.Level, null);
|
|
}
|
|
|
|
state = _state;
|
|
}
|
|
|
|
StateChanged?.Invoke(this, state);
|
|
return persisted;
|
|
}
|
|
|
|
/// <summary>
|
|
/// «Riprendi» after a verification or a disconnection, once Alberto fixed things in the browser. Resolves the open
|
|
/// incidents and restarts the gradual start. False when the breaker is not in such a stop (a throttle stop has a
|
|
/// fixed duration and cannot be shortened).
|
|
/// </summary>
|
|
public async Task<bool> ResumeAsync(CancellationToken ct = default)
|
|
{
|
|
BreakerState state;
|
|
Task persisted;
|
|
lock (_gate)
|
|
{
|
|
var now = _clock.Now;
|
|
if (_state.Status is not (BreakerStatus.Challenge or BreakerStatus.Disconnected))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var previous = _state.Status;
|
|
_state = _state with { Status = BreakerStatus.Closed, Until = null };
|
|
_runtime.RampStart = now;
|
|
persisted = EnqueueLocked(c => PersistResumeAsync(previous, now, c));
|
|
state = _state;
|
|
}
|
|
|
|
await persisted.WaitAsync(ct).ConfigureAwait(false);
|
|
StateChanged?.Invoke(this, state);
|
|
return true;
|
|
}
|
|
|
|
/// <summary>Waits until every incident and event reported so far is stored.</summary>
|
|
public Task FlushAsync()
|
|
{
|
|
lock (_gate)
|
|
{
|
|
return _tail;
|
|
}
|
|
}
|
|
|
|
private bool ExpireLocked(DateTimeOffset now)
|
|
{
|
|
if (_state.Status != BreakerStatus.Throttled || _state.Until is not { } until || now < until)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
_state = _state with { Status = BreakerStatus.Closed, Until = null };
|
|
var halvedUntil = _state.HalvedBudgetsUntil;
|
|
EnqueueLocked(ct => PersistThrottleEndAsync(until, halvedUntil, ct));
|
|
return true;
|
|
}
|
|
|
|
private Task EnqueueLocked(Func<CancellationToken, Task> work)
|
|
{
|
|
// Rows are written strictly in order: an incident is opened before it is resolved.
|
|
_tail = _tail.ContinueWith(_ => work(CancellationToken.None), CancellationToken.None, TaskContinuationOptions.None, TaskScheduler.Default).Unwrap();
|
|
return _tail;
|
|
}
|
|
|
|
private async Task PersistThrottleAsync(Signal signal, DateTimeOffset now, DateTimeOffset until, TimeSpan stop, long? sessionId, int withinWeek, CancellationToken ct)
|
|
{
|
|
var incident = await _storage.Incidents.OpenAsync(
|
|
new Incident
|
|
{
|
|
At = signal.At,
|
|
Level = SignalLevel.Throttle,
|
|
Signal = signal.Code,
|
|
Url = signal.Url,
|
|
StoppedUntil = until,
|
|
SessionId = sessionId,
|
|
},
|
|
ct).ConfigureAwait(false);
|
|
lock (_gate)
|
|
{
|
|
_openIncidents.Add(incident.Id);
|
|
}
|
|
|
|
if (_events is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var zone = _clock.LocalZone;
|
|
await _events.WriteAsync(
|
|
EventSeverity.Warning,
|
|
EngineEventCodes.ThrottleStop,
|
|
$"Instagram ha chiesto di rallentare (segnale {signal.Code}). Ho chiuso la sessione e mi fermo per {ItalianText.Duration(stop)}: "
|
|
+ $"riparto {ItalianText.Around(until, now, zone)}. Poi i tetti restano dimezzati per 48 ore.",
|
|
sessionId: sessionId,
|
|
ct: ct).ConfigureAwait(false);
|
|
|
|
if (withinWeek >= ProposalThreshold)
|
|
{
|
|
var profile = _runtime.Profile.Kind;
|
|
var proposal = profile is CautionProfileKind.Prudent
|
|
? "Tre rallentamenti in sette giorni anche con il profilo «Prudente»: conviene ridurre gli account seguiti o abbassare le loro frequenze."
|
|
: $"Tre rallentamenti in sette giorni con il profilo «{ItalianText.Profile(profile)}»: conviene passare al profilo «{ItalianText.Profile(MorePrudent(profile))}» o ridurre gli account seguiti.";
|
|
await _events.WriteAsync(EventSeverity.Warning, EngineEventCodes.ProposeMorePrudence, proposal, ct: ct).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
private async Task PersistManualStopAsync(Signal signal, DateTimeOffset now, bool wasThrottled, long? sessionId, CancellationToken ct)
|
|
{
|
|
if (wasThrottled)
|
|
{
|
|
await ResolveOpenAsync(now, ct).ConfigureAwait(false);
|
|
}
|
|
|
|
var incident = await _storage.Incidents.OpenAsync(
|
|
new Incident
|
|
{
|
|
At = signal.At,
|
|
Level = signal.Level,
|
|
Signal = signal.Code,
|
|
Url = signal.Url,
|
|
SessionId = sessionId,
|
|
},
|
|
ct).ConfigureAwait(false);
|
|
lock (_gate)
|
|
{
|
|
_openIncidents.Add(incident.Id);
|
|
}
|
|
|
|
if (_events is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var at = ItalianText.At(now, _clock.LocalZone);
|
|
var message = signal.Level == SignalLevel.Challenge
|
|
? $"Instagram chiede una verifica (segnale {signal.Code}). Ho fermato tutto {at} e non riparto da solo: apri il browser, completa la verifica a mano, poi premi «Riprendi»."
|
|
: $"La sessione di Instagram non c'è più (segnale {signal.Code}). Ho fermato tutto {at} e non provo a rientrare: apri il browser, accedi a mano, poi premi «Riprendi».";
|
|
await _events.WriteAsync(
|
|
EventSeverity.Error,
|
|
signal.Level == SignalLevel.Challenge ? EngineEventCodes.ChallengeStop : EngineEventCodes.DisconnectStop,
|
|
message,
|
|
sessionId: sessionId,
|
|
ct: ct).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task PersistThrottleEndAsync(DateTimeOffset until, DateTimeOffset? halvedUntil, CancellationToken ct)
|
|
{
|
|
await ResolveOpenAsync(until, ct).ConfigureAwait(false);
|
|
if (_events is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var tail = halvedUntil is { } h
|
|
? $" Riprendo con i tetti dimezzati, di nuovo pieni {ItalianText.Around(h, until, _clock.LocalZone)}."
|
|
: " Riprendo con i tetti normali.";
|
|
await _events.WriteAsync(EventSeverity.Info, EngineEventCodes.ThrottleEnd, "Fine del fermo per rallentamento." + tail, ct: ct).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task PersistResumeAsync(BreakerStatus previous, DateTimeOffset now, CancellationToken ct)
|
|
{
|
|
await ResolveOpenAsync(now, ct).ConfigureAwait(false);
|
|
await _storage.Settings.SetAsync(
|
|
EngineSettingKeys.RampStart,
|
|
now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture),
|
|
ct).ConfigureAwait(false);
|
|
if (_events is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var profile = _runtime.Profile;
|
|
var what = previous == BreakerStatus.Challenge ? "Ripresa dopo la verifica." : "Ripresa dopo il nuovo accesso.";
|
|
var ramp = profile.GradualStart && profile.RampUpDays > 0
|
|
? $" I tetti ripartono dalla metà e tornano pieni in {ItalianText.Count(profile.RampUpDays, "giorno", "giorni")}."
|
|
: string.Empty;
|
|
await _events.WriteAsync(EventSeverity.Success, EngineEventCodes.Resumed, what + ramp, ct: ct).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task ResolveOpenAsync(DateTimeOffset at, CancellationToken ct)
|
|
{
|
|
long[] open;
|
|
lock (_gate)
|
|
{
|
|
open = [.. _openIncidents];
|
|
_openIncidents.Clear();
|
|
}
|
|
|
|
foreach (var id in open)
|
|
{
|
|
await _storage.Incidents.ResolveAsync(id, at, ct).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
private static CautionProfileKind MorePrudent(CautionProfileKind kind) => kind switch
|
|
{
|
|
CautionProfileKind.Fast => CautionProfileKind.Balanced,
|
|
_ => CautionProfileKind.Prudent,
|
|
};
|
|
}
|
|
|
|
/// <summary>Stable codes of the engine's timeline events.</summary>
|
|
public static class EngineEventCodes
|
|
{
|
|
public const string ThrottleStop = "fermo-rallentamento";
|
|
public const string ThrottleEnd = "fine-fermo";
|
|
public const string ChallengeStop = "verifica-richiesta";
|
|
public const string DisconnectStop = "disconnessione";
|
|
public const string Resumed = "ripresa";
|
|
public const string ProposeMorePrudence = "proposta-prudenza";
|
|
public const string Paused = "pausa";
|
|
public const string Unpaused = "fine-pausa";
|
|
public const string BudgetReached = "tetto-raggiunto";
|
|
public const string SessionEnded = "sessione";
|
|
public const string BackfillCompleted = "storico-completato";
|
|
public const string BackfillWaiting = "storico-in-attesa";
|
|
public const string RecipeSuspended = "ricetta-sospesa";
|
|
public const string RecipeResumed = "ricetta-riattivata";
|
|
public const string FormatUnrecognized = "formato-non-riconosciuto";
|
|
public const string TargetNotFound = "bersaglio-non-trovato";
|
|
public const string AccountUnreachable = "account-non-raggiungibile";
|
|
public const string AccountPaused = "account-in-pausa";
|
|
public const string ManualCheck = "controlla-ora";
|
|
public const string RecipeFailed = "ricetta-fallita";
|
|
public const string SettingsChanged = "impostazioni-motore";
|
|
|
|
/// <summary>Prefix of the per-recipe lines («Appena fatto»): "ricetta." + recipe kind.</summary>
|
|
public const string RecipePrefix = "ricetta.";
|
|
}
|