Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
166 lines
7.7 KiB
C#
166 lines
7.7 KiB
C#
using Febe.Core.Abstractions;
|
|
using Febe.Core.Domain;
|
|
using Febe.Core.Engine;
|
|
using Febe.Core.Storage;
|
|
using Febe.Tests.Engine.Support;
|
|
|
|
namespace Febe.Tests.Engine;
|
|
|
|
[TestClass]
|
|
public sealed class CircuitBreakerTests
|
|
{
|
|
private static Signal Throttle(DateTimeOffset at) => new(SignalLevel.Throttle, "http-429", "https://sito.prova/api", at);
|
|
|
|
[TestMethod]
|
|
public async Task ThrottlesStopOneSixAndTwentyFourHoursThenHalveTheBudgets()
|
|
{
|
|
await using var bed = await EngineTestBed.CreateAsync();
|
|
var breaker = bed.Breaker;
|
|
var t0 = bed.Clock.Now;
|
|
|
|
await breaker.ReportAsync(Throttle(t0));
|
|
Assert.AreEqual(BreakerStatus.Throttled, breaker.State.Status);
|
|
Assert.AreEqual(t0 + TimeSpan.FromHours(1), breaker.State.Until);
|
|
Assert.IsFalse(breaker.State.AllowsSiteActions);
|
|
|
|
bed.Clock.Advance(TimeSpan.FromHours(1));
|
|
Assert.AreEqual(BreakerStatus.Closed, breaker.State.Status);
|
|
Assert.AreEqual(t0 + TimeSpan.FromHours(1) + TimeSpan.FromHours(48), breaker.State.HalvedBudgetsUntil);
|
|
|
|
bed.Clock.Advance(TimeSpan.FromHours(1));
|
|
await breaker.ReportAsync(Throttle(bed.Clock.Now));
|
|
Assert.AreEqual(bed.Clock.Now + TimeSpan.FromHours(6), breaker.State.Until, "Second within 24 hours: 6 hours.");
|
|
|
|
bed.Clock.Advance(TimeSpan.FromHours(7));
|
|
Assert.IsTrue(breaker.State.AllowsSiteActions);
|
|
await breaker.ReportAsync(Throttle(bed.Clock.Now));
|
|
Assert.AreEqual(bed.Clock.Now + TimeSpan.FromHours(24), breaker.State.Until, "Third within 24 hours: 24 hours.");
|
|
|
|
bed.Clock.Advance(TimeSpan.FromHours(24));
|
|
Assert.IsTrue(breaker.State.AllowsSiteActions);
|
|
await breaker.FlushAsync();
|
|
|
|
var incidents = await bed.Db.Incidents.ListAsync(t0, bed.Clock.Now.AddSeconds(1));
|
|
Assert.HasCount(3, incidents);
|
|
Assert.IsTrue(incidents.All(i => i.Level == SignalLevel.Throttle && i.Signal == "http-429" && i.Url == "https://sito.prova/api"));
|
|
Assert.AreEqual(t0 + TimeSpan.FromHours(1), incidents[0].StoppedUntil);
|
|
Assert.AreEqual(incidents[0].StoppedUntil, incidents[0].ResolvedAt);
|
|
Assert.IsTrue(incidents.All(i => i.ResolvedAt is not null));
|
|
Assert.HasCount(0, await bed.Db.Incidents.ListOpenAsync());
|
|
|
|
var events = await bed.Db.Events.ListAsync(new EventQuery());
|
|
Assert.AreEqual(3, events.Count(e => e.Code == EngineEventCodes.ThrottleStop));
|
|
Assert.IsTrue(events.Any(e => e.Code == EngineEventCodes.ThrottleEnd));
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AThrottleDuringAStopDoesNotEscalateIt()
|
|
{
|
|
await using var bed = await EngineTestBed.CreateAsync();
|
|
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
|
|
var until = bed.Breaker.State.Until;
|
|
bed.Clock.Advance(TimeSpan.FromMinutes(10));
|
|
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
|
|
Assert.AreEqual(until, bed.Breaker.State.Until);
|
|
await bed.Breaker.FlushAsync();
|
|
Assert.HasCount(1, await bed.Db.Incidents.ListAsync(bed.Clock.Now.AddDays(-1), bed.Clock.Now.AddDays(1)));
|
|
}
|
|
|
|
[TestMethod]
|
|
[DataRow(SignalLevel.Challenge, BreakerStatus.Challenge, EngineEventCodes.ChallengeStop)]
|
|
[DataRow(SignalLevel.Disconnect, BreakerStatus.Disconnected, EngineEventCodes.DisconnectStop)]
|
|
public async Task VerificationAndDisconnectionNeverRestartAlone(SignalLevel level, BreakerStatus status, string code)
|
|
{
|
|
await using var bed = await EngineTestBed.CreateAsync(gradualStart: true);
|
|
await bed.Breaker.ReportAsync(new Signal(level, "segnale-prova", "https://sito.prova/x", bed.Clock.Now));
|
|
Assert.AreEqual(status, bed.Breaker.State.Status);
|
|
|
|
bed.Clock.Advance(TimeSpan.FromDays(5));
|
|
Assert.AreEqual(status, bed.Breaker.State.Status);
|
|
Assert.IsFalse(bed.Breaker.State.AllowsSiteActions);
|
|
var open = await bed.Db.Incidents.ListOpenAsync();
|
|
Assert.HasCount(1, open);
|
|
Assert.AreEqual(level, open[0].Level);
|
|
Assert.IsNull(open[0].StoppedUntil);
|
|
|
|
var resumedAt = bed.Clock.Now;
|
|
Assert.IsTrue(await bed.Breaker.ResumeAsync());
|
|
Assert.AreEqual(BreakerStatus.Closed, bed.Breaker.State.Status);
|
|
Assert.HasCount(0, await bed.Db.Incidents.ListOpenAsync());
|
|
|
|
// The resume starts the gradual start again, persisted.
|
|
Assert.AreEqual(resumedAt, bed.Runtime.RampStart);
|
|
Assert.AreEqual(resumedAt.ToUnixTimeMilliseconds(), await bed.Db.Settings.GetInt64Async(EngineSettingKeys.RampStart));
|
|
Assert.AreEqual(0.5, bed.Governor.GetEffectiveBudgets(bed.Clock.Now).Factor, 1e-9);
|
|
|
|
var events = await bed.Db.Events.ListAsync(new EventQuery());
|
|
Assert.IsTrue(events.Any(e => e.Code == code && e.Severity == EventSeverity.Error));
|
|
Assert.IsTrue(events.Any(e => e.Code == EngineEventCodes.Resumed));
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AThrottleStopCannotBeShortenedByResume()
|
|
{
|
|
await using var bed = await EngineTestBed.CreateAsync();
|
|
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
|
|
Assert.IsFalse(await bed.Breaker.ResumeAsync());
|
|
Assert.AreEqual(BreakerStatus.Throttled, bed.Breaker.State.Status);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TheStateIsRestoredFromTheDatabase()
|
|
{
|
|
await using var bed = await EngineTestBed.CreateAsync();
|
|
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
|
|
var until = bed.Breaker.State.Until;
|
|
await bed.Breaker.FlushAsync();
|
|
|
|
var restarted = await bed.ReopenAsync(seed: 99);
|
|
Assert.AreEqual(BreakerStatus.Throttled, restarted.Breaker.State.Status);
|
|
Assert.AreEqual(until, restarted.Breaker.State.Until);
|
|
|
|
bed.Clock.Advance(TimeSpan.FromHours(2));
|
|
await restarted.Breaker.ReportAsync(new Signal(SignalLevel.Challenge, "challenge-url", null, bed.Clock.Now));
|
|
var again = await bed.ReopenAsync(seed: 100);
|
|
Assert.AreEqual(BreakerStatus.Challenge, again.Breaker.State.Status);
|
|
Assert.IsNotNull(again.Breaker.State.HalvedBudgetsUntil, "The halving of the earlier throttle is restored too.");
|
|
|
|
// The expired throttle was resolved; resuming the verification leaves nothing open.
|
|
bed.Clock.Advance(TimeSpan.FromHours(1));
|
|
Assert.IsTrue(await again.Breaker.ResumeAsync());
|
|
Assert.HasCount(0, await bed.Db.Incidents.ListOpenAsync());
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task ThreeThrottlesInSevenDaysProposeAMorePrudentProfile()
|
|
{
|
|
await using var bed = await EngineTestBed.CreateAsync(profile: CautionProfileKind.Fast);
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
|
|
bed.Clock.Advance(TimeSpan.FromDays(2));
|
|
_ = bed.Breaker.State;
|
|
}
|
|
|
|
await bed.Breaker.FlushAsync();
|
|
var proposals = (await bed.Db.Events.ListAsync(new EventQuery())).Where(e => e.Code == EngineEventCodes.ProposeMorePrudence).ToList();
|
|
Assert.HasCount(1, proposals);
|
|
StringAssert.Contains(proposals[0].Message, "«Equilibrato»");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TwoThrottlesInSevenDaysAreNotYetAProposal()
|
|
{
|
|
await using var bed = await EngineTestBed.CreateAsync();
|
|
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
|
|
bed.Clock.Advance(TimeSpan.FromDays(8));
|
|
_ = bed.Breaker.State;
|
|
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
|
|
bed.Clock.Advance(TimeSpan.FromDays(2));
|
|
_ = bed.Breaker.State;
|
|
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
|
|
await bed.Breaker.FlushAsync();
|
|
Assert.IsFalse((await bed.Db.Events.ListAsync(new EventQuery())).Any(e => e.Code == EngineEventCodes.ProposeMorePrudence));
|
|
}
|
|
}
|