using Febe.Core.Abstractions; using Febe.Core.Domain; using Febe.Core.Engine; using Febe.Core.Storage; using Febe.Tests.Engine.Support; namespace Febe.Tests.Engine; [TestClass] public sealed class CircuitBreakerTests { private static Signal Throttle(DateTimeOffset at) => new(SignalLevel.Throttle, "http-429", "https://sito.prova/api", at); [TestMethod] public async Task ThrottlesStopOneSixAndTwentyFourHoursThenHalveTheBudgets() { await using var bed = await EngineTestBed.CreateAsync(); var breaker = bed.Breaker; var t0 = bed.Clock.Now; await breaker.ReportAsync(Throttle(t0)); Assert.AreEqual(BreakerStatus.Throttled, breaker.State.Status); Assert.AreEqual(t0 + TimeSpan.FromHours(1), breaker.State.Until); Assert.IsFalse(breaker.State.AllowsSiteActions); bed.Clock.Advance(TimeSpan.FromHours(1)); Assert.AreEqual(BreakerStatus.Closed, breaker.State.Status); Assert.AreEqual(t0 + TimeSpan.FromHours(1) + TimeSpan.FromHours(48), breaker.State.HalvedBudgetsUntil); bed.Clock.Advance(TimeSpan.FromHours(1)); await breaker.ReportAsync(Throttle(bed.Clock.Now)); Assert.AreEqual(bed.Clock.Now + TimeSpan.FromHours(6), breaker.State.Until, "Second within 24 hours: 6 hours."); bed.Clock.Advance(TimeSpan.FromHours(7)); Assert.IsTrue(breaker.State.AllowsSiteActions); await breaker.ReportAsync(Throttle(bed.Clock.Now)); Assert.AreEqual(bed.Clock.Now + TimeSpan.FromHours(24), breaker.State.Until, "Third within 24 hours: 24 hours."); bed.Clock.Advance(TimeSpan.FromHours(24)); Assert.IsTrue(breaker.State.AllowsSiteActions); await breaker.FlushAsync(); var incidents = await bed.Db.Incidents.ListAsync(t0, bed.Clock.Now.AddSeconds(1)); Assert.HasCount(3, incidents); Assert.IsTrue(incidents.All(i => i.Level == SignalLevel.Throttle && i.Signal == "http-429" && i.Url == "https://sito.prova/api")); Assert.AreEqual(t0 + TimeSpan.FromHours(1), incidents[0].StoppedUntil); Assert.AreEqual(incidents[0].StoppedUntil, incidents[0].ResolvedAt); Assert.IsTrue(incidents.All(i => i.ResolvedAt is not null)); Assert.HasCount(0, await bed.Db.Incidents.ListOpenAsync()); var events = await bed.Db.Events.ListAsync(new EventQuery()); Assert.AreEqual(3, events.Count(e => e.Code == EngineEventCodes.ThrottleStop)); Assert.IsTrue(events.Any(e => e.Code == EngineEventCodes.ThrottleEnd)); } [TestMethod] public async Task AThrottleDuringAStopDoesNotEscalateIt() { await using var bed = await EngineTestBed.CreateAsync(); await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now)); var until = bed.Breaker.State.Until; bed.Clock.Advance(TimeSpan.FromMinutes(10)); await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now)); Assert.AreEqual(until, bed.Breaker.State.Until); await bed.Breaker.FlushAsync(); Assert.HasCount(1, await bed.Db.Incidents.ListAsync(bed.Clock.Now.AddDays(-1), bed.Clock.Now.AddDays(1))); } [TestMethod] [DataRow(SignalLevel.Challenge, BreakerStatus.Challenge, EngineEventCodes.ChallengeStop)] [DataRow(SignalLevel.Disconnect, BreakerStatus.Disconnected, EngineEventCodes.DisconnectStop)] public async Task VerificationAndDisconnectionNeverRestartAlone(SignalLevel level, BreakerStatus status, string code) { await using var bed = await EngineTestBed.CreateAsync(gradualStart: true); await bed.Breaker.ReportAsync(new Signal(level, "segnale-prova", "https://sito.prova/x", bed.Clock.Now)); Assert.AreEqual(status, bed.Breaker.State.Status); bed.Clock.Advance(TimeSpan.FromDays(5)); Assert.AreEqual(status, bed.Breaker.State.Status); Assert.IsFalse(bed.Breaker.State.AllowsSiteActions); var open = await bed.Db.Incidents.ListOpenAsync(); Assert.HasCount(1, open); Assert.AreEqual(level, open[0].Level); Assert.IsNull(open[0].StoppedUntil); var resumedAt = bed.Clock.Now; Assert.IsTrue(await bed.Breaker.ResumeAsync()); Assert.AreEqual(BreakerStatus.Closed, bed.Breaker.State.Status); Assert.HasCount(0, await bed.Db.Incidents.ListOpenAsync()); // The resume starts the gradual start again, persisted. Assert.AreEqual(resumedAt, bed.Runtime.RampStart); Assert.AreEqual(resumedAt.ToUnixTimeMilliseconds(), await bed.Db.Settings.GetInt64Async(EngineSettingKeys.RampStart)); Assert.AreEqual(0.5, bed.Governor.GetEffectiveBudgets(bed.Clock.Now).Factor, 1e-9); var events = await bed.Db.Events.ListAsync(new EventQuery()); Assert.IsTrue(events.Any(e => e.Code == code && e.Severity == EventSeverity.Error)); Assert.IsTrue(events.Any(e => e.Code == EngineEventCodes.Resumed)); } [TestMethod] public async Task AThrottleStopCannotBeShortenedByResume() { await using var bed = await EngineTestBed.CreateAsync(); await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now)); Assert.IsFalse(await bed.Breaker.ResumeAsync()); Assert.AreEqual(BreakerStatus.Throttled, bed.Breaker.State.Status); } [TestMethod] public async Task TheStateIsRestoredFromTheDatabase() { await using var bed = await EngineTestBed.CreateAsync(); await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now)); var until = bed.Breaker.State.Until; await bed.Breaker.FlushAsync(); var restarted = await bed.ReopenAsync(seed: 99); Assert.AreEqual(BreakerStatus.Throttled, restarted.Breaker.State.Status); Assert.AreEqual(until, restarted.Breaker.State.Until); bed.Clock.Advance(TimeSpan.FromHours(2)); await restarted.Breaker.ReportAsync(new Signal(SignalLevel.Challenge, "challenge-url", null, bed.Clock.Now)); var again = await bed.ReopenAsync(seed: 100); Assert.AreEqual(BreakerStatus.Challenge, again.Breaker.State.Status); Assert.IsNotNull(again.Breaker.State.HalvedBudgetsUntil, "The halving of the earlier throttle is restored too."); // The expired throttle was resolved; resuming the verification leaves nothing open. bed.Clock.Advance(TimeSpan.FromHours(1)); Assert.IsTrue(await again.Breaker.ResumeAsync()); Assert.HasCount(0, await bed.Db.Incidents.ListOpenAsync()); } [TestMethod] public async Task ThreeThrottlesInSevenDaysProposeAMorePrudentProfile() { await using var bed = await EngineTestBed.CreateAsync(profile: CautionProfileKind.Fast); for (var i = 0; i < 3; i++) { await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now)); bed.Clock.Advance(TimeSpan.FromDays(2)); _ = bed.Breaker.State; } await bed.Breaker.FlushAsync(); var proposals = (await bed.Db.Events.ListAsync(new EventQuery())).Where(e => e.Code == EngineEventCodes.ProposeMorePrudence).ToList(); Assert.HasCount(1, proposals); StringAssert.Contains(proposals[0].Message, "«Equilibrato»"); } [TestMethod] public async Task TwoThrottlesInSevenDaysAreNotYetAProposal() { await using var bed = await EngineTestBed.CreateAsync(); await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now)); bed.Clock.Advance(TimeSpan.FromDays(8)); _ = bed.Breaker.State; await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now)); bed.Clock.Advance(TimeSpan.FromDays(2)); _ = bed.Breaker.State; await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now)); await bed.Breaker.FlushAsync(); Assert.IsFalse((await bed.Db.Events.ListAsync(new EventQuery())).Any(e => e.Code == EngineEventCodes.ProposeMorePrudence)); } }