using System.Globalization; using Febe.Core.Abstractions; using Febe.Core.Domain; using Febe.Core.Storage; namespace Febe.Core.Engine; /// /// The circuit breaker («interruttore», spec §7.6). /// /// Throttle: closes the session at once; stops 1 hour the first time, 6 hours the second within 24 hours, /// 24 hours the third; then halved budgets for 48 hours. Reopens by itself. /// Challenge and disconnection: stop everything and never restart alone; only /// (Alberto's «Riprendi», after fixing things in the browser) reopens, and the resume starts the gradual start. /// /// Every stop is an incident row and a line of the Activity timeline; the state is restored from the database /// at start-up. is synchronous (signals arrive from capture threads): the state changes at once /// and the rows are written in order in the background ( waits for them). /// public sealed class CircuitBreaker : ICircuitBreaker { /// Fixed stop durations (spec §7.6: not editable in version 1). public static readonly IReadOnlyList ThrottleStops = [TimeSpan.FromHours(1), TimeSpan.FromHours(6), TimeSpan.FromHours(24)]; public static readonly TimeSpan HalvedPeriod = TimeSpan.FromHours(48); public static readonly TimeSpan EscalationWindow = TimeSpan.FromHours(24); public static readonly TimeSpan ProposalWindow = TimeSpan.FromDays(7); public const int ProposalThreshold = 3; private readonly IFebeStorage _storage; private readonly IClock _clock; private readonly EngineRuntime _runtime; private readonly EngineEvents? _events; private readonly Lock _gate = new(); private readonly List _recentThrottles = []; private readonly List _openIncidents = []; private BreakerState _state = BreakerState.Normal; private Task _tail = Task.CompletedTask; private CircuitBreaker(IFebeStorage storage, IClock clock, EngineRuntime runtime, EngineEvents? events) { _storage = storage; _clock = clock; _runtime = runtime; _events = events; } /// Raised after every change of state (outside the internal lock). The engine closes the session on it. public event EventHandler? StateChanged; /// Current state; a throttle stop that has expired turns into Closed here. public BreakerState State { get { BreakerState state; bool changed; lock (_gate) { changed = ExpireLocked(_clock.Now); state = _state; } if (changed) { StateChanged?.Invoke(this, state); } return state; } } /// Restores the state from the incidents table (open stops, recent throttles, halved budgets). public static async Task LoadAsync( IFebeStorage storage, IClock clock, EngineRuntime runtime, EngineEvents? events = null, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(storage); ArgumentNullException.ThrowIfNull(clock); ArgumentNullException.ThrowIfNull(runtime); var breaker = new CircuitBreaker(storage, clock, runtime, events); var now = clock.Now; var recent = await storage.Incidents.ListAsync(now - ProposalWindow - EscalationWindow, now.AddSeconds(1), ct).ConfigureAwait(false); DateTimeOffset? halvedUntil = null; foreach (var incident in recent.Where(i => i.Level == SignalLevel.Throttle)) { breaker._recentThrottles.Add(incident.At); if (incident.StoppedUntil is { } until && until + HalvedPeriod > now && (halvedUntil is null || until + HalvedPeriod > halvedUntil)) { halvedUntil = until + HalvedPeriod; } } var state = new BreakerState(BreakerStatus.Closed, null, null, halvedUntil); foreach (var incident in await storage.Incidents.ListOpenAsync(ct).ConfigureAwait(false)) { var signal = new Signal(incident.Level, incident.Signal, incident.Url, incident.At); switch (incident.Level) { case SignalLevel.Challenge: state = state with { Status = BreakerStatus.Challenge, Until = null, LastSignal = signal }; breaker._openIncidents.Add(incident.Id); break; case SignalLevel.Disconnect: state = state with { Status = BreakerStatus.Disconnected, Until = null, LastSignal = signal }; breaker._openIncidents.Add(incident.Id); break; case SignalLevel.Throttle when incident.StoppedUntil is { } until && until > now: if (state.Status == BreakerStatus.Closed) { state = state with { Status = BreakerStatus.Throttled, Until = until, LastSignal = signal }; } breaker._openIncidents.Add(incident.Id); break; default: // An expired throttle left open by a previous process: resolved at its planned end. await storage.Incidents.ResolveAsync(incident.Id, incident.StoppedUntil ?? now, ct).ConfigureAwait(false); break; } } breaker._state = state; return breaker; } public void Report(Signal signal) => _ = ReportAsync(signal); /// Same as , returning when the incident and the event are stored. public Task ReportAsync(Signal signal, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(signal); BreakerState state; Task persisted; lock (_gate) { var now = _clock.Now; ExpireLocked(now); var sessionId = _runtime.CurrentSession?.SessionId; switch (signal.Level) { case SignalLevel.Throttle: if (_state.Status != BreakerStatus.Closed) { // Already stopped: a late signal of the same stop does not escalate it. return _tail; } _recentThrottles.RemoveAll(t => t < now - ProposalWindow); var within24Hours = _recentThrottles.Count(t => t > now - EscalationWindow); var stop = ThrottleStops[Math.Min(within24Hours, ThrottleStops.Count - 1)]; _recentThrottles.Add(now); var until = now + stop; _state = new BreakerState(BreakerStatus.Throttled, until, signal, until + HalvedPeriod); var withinWeek = _recentThrottles.Count(t => t > now - ProposalWindow); persisted = EnqueueLocked(c => PersistThrottleAsync(signal, now, until, stop, sessionId, withinWeek, c)); break; case SignalLevel.Challenge: case SignalLevel.Disconnect: var status = signal.Level == SignalLevel.Challenge ? BreakerStatus.Challenge : BreakerStatus.Disconnected; if (_state.Status == status) { return _tail; } var wasThrottled = _state.Status == BreakerStatus.Throttled; _state = new BreakerState(status, null, signal, _state.HalvedBudgetsUntil); persisted = EnqueueLocked(c => PersistManualStopAsync(signal, now, wasThrottled, sessionId, c)); break; default: throw new ArgumentOutOfRangeException(nameof(signal), signal.Level, null); } state = _state; } StateChanged?.Invoke(this, state); return persisted; } /// /// «Riprendi» after a verification or a disconnection, once Alberto fixed things in the browser. Resolves the open /// incidents and restarts the gradual start. False when the breaker is not in such a stop (a throttle stop has a /// fixed duration and cannot be shortened). /// public async Task ResumeAsync(CancellationToken ct = default) { BreakerState state; Task persisted; lock (_gate) { var now = _clock.Now; if (_state.Status is not (BreakerStatus.Challenge or BreakerStatus.Disconnected)) { return false; } var previous = _state.Status; _state = _state with { Status = BreakerStatus.Closed, Until = null }; _runtime.RampStart = now; persisted = EnqueueLocked(c => PersistResumeAsync(previous, now, c)); state = _state; } await persisted.WaitAsync(ct).ConfigureAwait(false); StateChanged?.Invoke(this, state); return true; } /// Waits until every incident and event reported so far is stored. public Task FlushAsync() { lock (_gate) { return _tail; } } private bool ExpireLocked(DateTimeOffset now) { if (_state.Status != BreakerStatus.Throttled || _state.Until is not { } until || now < until) { return false; } _state = _state with { Status = BreakerStatus.Closed, Until = null }; var halvedUntil = _state.HalvedBudgetsUntil; EnqueueLocked(ct => PersistThrottleEndAsync(until, halvedUntil, ct)); return true; } private Task EnqueueLocked(Func work) { // Rows are written strictly in order: an incident is opened before it is resolved. _tail = _tail.ContinueWith(_ => work(CancellationToken.None), CancellationToken.None, TaskContinuationOptions.None, TaskScheduler.Default).Unwrap(); return _tail; } private async Task PersistThrottleAsync(Signal signal, DateTimeOffset now, DateTimeOffset until, TimeSpan stop, long? sessionId, int withinWeek, CancellationToken ct) { var incident = await _storage.Incidents.OpenAsync( new Incident { At = signal.At, Level = SignalLevel.Throttle, Signal = signal.Code, Url = signal.Url, StoppedUntil = until, SessionId = sessionId, }, ct).ConfigureAwait(false); lock (_gate) { _openIncidents.Add(incident.Id); } if (_events is null) { return; } var zone = _clock.LocalZone; await _events.WriteAsync( EventSeverity.Warning, EngineEventCodes.ThrottleStop, $"Instagram ha chiesto di rallentare (segnale {signal.Code}). Ho chiuso la sessione e mi fermo per {ItalianText.Duration(stop)}: " + $"riparto {ItalianText.Around(until, now, zone)}. Poi i tetti restano dimezzati per 48 ore.", sessionId: sessionId, ct: ct).ConfigureAwait(false); if (withinWeek >= ProposalThreshold) { var profile = _runtime.Profile.Kind; var proposal = profile is CautionProfileKind.Prudent ? "Tre rallentamenti in sette giorni anche con il profilo «Prudente»: conviene ridurre gli account seguiti o abbassare le loro frequenze." : $"Tre rallentamenti in sette giorni con il profilo «{ItalianText.Profile(profile)}»: conviene passare al profilo «{ItalianText.Profile(MorePrudent(profile))}» o ridurre gli account seguiti."; await _events.WriteAsync(EventSeverity.Warning, EngineEventCodes.ProposeMorePrudence, proposal, ct: ct).ConfigureAwait(false); } } private async Task PersistManualStopAsync(Signal signal, DateTimeOffset now, bool wasThrottled, long? sessionId, CancellationToken ct) { if (wasThrottled) { await ResolveOpenAsync(now, ct).ConfigureAwait(false); } var incident = await _storage.Incidents.OpenAsync( new Incident { At = signal.At, Level = signal.Level, Signal = signal.Code, Url = signal.Url, SessionId = sessionId, }, ct).ConfigureAwait(false); lock (_gate) { _openIncidents.Add(incident.Id); } if (_events is null) { return; } var at = ItalianText.At(now, _clock.LocalZone); var message = signal.Level == SignalLevel.Challenge ? $"Instagram chiede una verifica (segnale {signal.Code}). Ho fermato tutto {at} e non riparto da solo: apri il browser, completa la verifica a mano, poi premi «Riprendi»." : $"La sessione di Instagram non c'è più (segnale {signal.Code}). Ho fermato tutto {at} e non provo a rientrare: apri il browser, accedi a mano, poi premi «Riprendi»."; await _events.WriteAsync( EventSeverity.Error, signal.Level == SignalLevel.Challenge ? EngineEventCodes.ChallengeStop : EngineEventCodes.DisconnectStop, message, sessionId: sessionId, ct: ct).ConfigureAwait(false); } private async Task PersistThrottleEndAsync(DateTimeOffset until, DateTimeOffset? halvedUntil, CancellationToken ct) { await ResolveOpenAsync(until, ct).ConfigureAwait(false); if (_events is null) { return; } var tail = halvedUntil is { } h ? $" Riprendo con i tetti dimezzati, di nuovo pieni {ItalianText.Around(h, until, _clock.LocalZone)}." : " Riprendo con i tetti normali."; await _events.WriteAsync(EventSeverity.Info, EngineEventCodes.ThrottleEnd, "Fine del fermo per rallentamento." + tail, ct: ct).ConfigureAwait(false); } private async Task PersistResumeAsync(BreakerStatus previous, DateTimeOffset now, CancellationToken ct) { await ResolveOpenAsync(now, ct).ConfigureAwait(false); await _storage.Settings.SetAsync( EngineSettingKeys.RampStart, now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture), ct).ConfigureAwait(false); if (_events is null) { return; } var profile = _runtime.Profile; var what = previous == BreakerStatus.Challenge ? "Ripresa dopo la verifica." : "Ripresa dopo il nuovo accesso."; var ramp = profile.GradualStart && profile.RampUpDays > 0 ? $" I tetti ripartono dalla metà e tornano pieni in {ItalianText.Count(profile.RampUpDays, "giorno", "giorni")}." : string.Empty; await _events.WriteAsync(EventSeverity.Success, EngineEventCodes.Resumed, what + ramp, ct: ct).ConfigureAwait(false); } private async Task ResolveOpenAsync(DateTimeOffset at, CancellationToken ct) { long[] open; lock (_gate) { open = [.. _openIncidents]; _openIncidents.Clear(); } foreach (var id in open) { await _storage.Incidents.ResolveAsync(id, at, ct).ConfigureAwait(false); } } private static CautionProfileKind MorePrudent(CautionProfileKind kind) => kind switch { CautionProfileKind.Fast => CautionProfileKind.Balanced, _ => CautionProfileKind.Prudent, }; } /// Stable codes of the engine's timeline events. public static class EngineEventCodes { public const string ThrottleStop = "fermo-rallentamento"; public const string ThrottleEnd = "fine-fermo"; public const string ChallengeStop = "verifica-richiesta"; public const string DisconnectStop = "disconnessione"; public const string Resumed = "ripresa"; public const string ProposeMorePrudence = "proposta-prudenza"; public const string Paused = "pausa"; public const string Unpaused = "fine-pausa"; public const string BudgetReached = "tetto-raggiunto"; public const string SessionEnded = "sessione"; public const string BackfillCompleted = "storico-completato"; public const string BackfillWaiting = "storico-in-attesa"; public const string RecipeSuspended = "ricetta-sospesa"; public const string RecipeResumed = "ricetta-riattivata"; public const string FormatUnrecognized = "formato-non-riconosciuto"; public const string TargetNotFound = "bersaglio-non-trovato"; public const string AccountUnreachable = "account-non-raggiungibile"; public const string AccountPaused = "account-in-pausa"; public const string ManualCheck = "controlla-ora"; public const string RecipeFailed = "ricetta-fallita"; public const string SettingsChanged = "impostazioni-motore"; /// Prefix of the per-recipe lines («Appena fatto»): "ricetta." + recipe kind. public const string RecipePrefix = "ricetta."; }