Files
Febe/tests/Febe.Tests/Engine/CircuitBreakerTests.cs
T

166 lines
7.7 KiB
C#

using Febe.Core.Abstractions;
using Febe.Core.Domain;
using Febe.Core.Engine;
using Febe.Core.Storage;
using Febe.Tests.Engine.Support;
namespace Febe.Tests.Engine;
[TestClass]
public sealed class CircuitBreakerTests
{
private static Signal Throttle(DateTimeOffset at) => new(SignalLevel.Throttle, "http-429", "https://sito.prova/api", at);
[TestMethod]
public async Task ThrottlesStopOneSixAndTwentyFourHoursThenHalveTheBudgets()
{
await using var bed = await EngineTestBed.CreateAsync();
var breaker = bed.Breaker;
var t0 = bed.Clock.Now;
await breaker.ReportAsync(Throttle(t0));
Assert.AreEqual(BreakerStatus.Throttled, breaker.State.Status);
Assert.AreEqual(t0 + TimeSpan.FromHours(1), breaker.State.Until);
Assert.IsFalse(breaker.State.AllowsSiteActions);
bed.Clock.Advance(TimeSpan.FromHours(1));
Assert.AreEqual(BreakerStatus.Closed, breaker.State.Status);
Assert.AreEqual(t0 + TimeSpan.FromHours(1) + TimeSpan.FromHours(48), breaker.State.HalvedBudgetsUntil);
bed.Clock.Advance(TimeSpan.FromHours(1));
await breaker.ReportAsync(Throttle(bed.Clock.Now));
Assert.AreEqual(bed.Clock.Now + TimeSpan.FromHours(6), breaker.State.Until, "Second within 24 hours: 6 hours.");
bed.Clock.Advance(TimeSpan.FromHours(7));
Assert.IsTrue(breaker.State.AllowsSiteActions);
await breaker.ReportAsync(Throttle(bed.Clock.Now));
Assert.AreEqual(bed.Clock.Now + TimeSpan.FromHours(24), breaker.State.Until, "Third within 24 hours: 24 hours.");
bed.Clock.Advance(TimeSpan.FromHours(24));
Assert.IsTrue(breaker.State.AllowsSiteActions);
await breaker.FlushAsync();
var incidents = await bed.Db.Incidents.ListAsync(t0, bed.Clock.Now.AddSeconds(1));
Assert.HasCount(3, incidents);
Assert.IsTrue(incidents.All(i => i.Level == SignalLevel.Throttle && i.Signal == "http-429" && i.Url == "https://sito.prova/api"));
Assert.AreEqual(t0 + TimeSpan.FromHours(1), incidents[0].StoppedUntil);
Assert.AreEqual(incidents[0].StoppedUntil, incidents[0].ResolvedAt);
Assert.IsTrue(incidents.All(i => i.ResolvedAt is not null));
Assert.HasCount(0, await bed.Db.Incidents.ListOpenAsync());
var events = await bed.Db.Events.ListAsync(new EventQuery());
Assert.AreEqual(3, events.Count(e => e.Code == EngineEventCodes.ThrottleStop));
Assert.IsTrue(events.Any(e => e.Code == EngineEventCodes.ThrottleEnd));
}
[TestMethod]
public async Task AThrottleDuringAStopDoesNotEscalateIt()
{
await using var bed = await EngineTestBed.CreateAsync();
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
var until = bed.Breaker.State.Until;
bed.Clock.Advance(TimeSpan.FromMinutes(10));
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
Assert.AreEqual(until, bed.Breaker.State.Until);
await bed.Breaker.FlushAsync();
Assert.HasCount(1, await bed.Db.Incidents.ListAsync(bed.Clock.Now.AddDays(-1), bed.Clock.Now.AddDays(1)));
}
[TestMethod]
[DataRow(SignalLevel.Challenge, BreakerStatus.Challenge, EngineEventCodes.ChallengeStop)]
[DataRow(SignalLevel.Disconnect, BreakerStatus.Disconnected, EngineEventCodes.DisconnectStop)]
public async Task VerificationAndDisconnectionNeverRestartAlone(SignalLevel level, BreakerStatus status, string code)
{
await using var bed = await EngineTestBed.CreateAsync(gradualStart: true);
await bed.Breaker.ReportAsync(new Signal(level, "segnale-prova", "https://sito.prova/x", bed.Clock.Now));
Assert.AreEqual(status, bed.Breaker.State.Status);
bed.Clock.Advance(TimeSpan.FromDays(5));
Assert.AreEqual(status, bed.Breaker.State.Status);
Assert.IsFalse(bed.Breaker.State.AllowsSiteActions);
var open = await bed.Db.Incidents.ListOpenAsync();
Assert.HasCount(1, open);
Assert.AreEqual(level, open[0].Level);
Assert.IsNull(open[0].StoppedUntil);
var resumedAt = bed.Clock.Now;
Assert.IsTrue(await bed.Breaker.ResumeAsync());
Assert.AreEqual(BreakerStatus.Closed, bed.Breaker.State.Status);
Assert.HasCount(0, await bed.Db.Incidents.ListOpenAsync());
// The resume starts the gradual start again, persisted.
Assert.AreEqual(resumedAt, bed.Runtime.RampStart);
Assert.AreEqual(resumedAt.ToUnixTimeMilliseconds(), await bed.Db.Settings.GetInt64Async(EngineSettingKeys.RampStart));
Assert.AreEqual(0.5, bed.Governor.GetEffectiveBudgets(bed.Clock.Now).Factor, 1e-9);
var events = await bed.Db.Events.ListAsync(new EventQuery());
Assert.IsTrue(events.Any(e => e.Code == code && e.Severity == EventSeverity.Error));
Assert.IsTrue(events.Any(e => e.Code == EngineEventCodes.Resumed));
}
[TestMethod]
public async Task AThrottleStopCannotBeShortenedByResume()
{
await using var bed = await EngineTestBed.CreateAsync();
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
Assert.IsFalse(await bed.Breaker.ResumeAsync());
Assert.AreEqual(BreakerStatus.Throttled, bed.Breaker.State.Status);
}
[TestMethod]
public async Task TheStateIsRestoredFromTheDatabase()
{
await using var bed = await EngineTestBed.CreateAsync();
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
var until = bed.Breaker.State.Until;
await bed.Breaker.FlushAsync();
var restarted = await bed.ReopenAsync(seed: 99);
Assert.AreEqual(BreakerStatus.Throttled, restarted.Breaker.State.Status);
Assert.AreEqual(until, restarted.Breaker.State.Until);
bed.Clock.Advance(TimeSpan.FromHours(2));
await restarted.Breaker.ReportAsync(new Signal(SignalLevel.Challenge, "challenge-url", null, bed.Clock.Now));
var again = await bed.ReopenAsync(seed: 100);
Assert.AreEqual(BreakerStatus.Challenge, again.Breaker.State.Status);
Assert.IsNotNull(again.Breaker.State.HalvedBudgetsUntil, "The halving of the earlier throttle is restored too.");
// The expired throttle was resolved; resuming the verification leaves nothing open.
bed.Clock.Advance(TimeSpan.FromHours(1));
Assert.IsTrue(await again.Breaker.ResumeAsync());
Assert.HasCount(0, await bed.Db.Incidents.ListOpenAsync());
}
[TestMethod]
public async Task ThreeThrottlesInSevenDaysProposeAMorePrudentProfile()
{
await using var bed = await EngineTestBed.CreateAsync(profile: CautionProfileKind.Fast);
for (var i = 0; i < 3; i++)
{
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
bed.Clock.Advance(TimeSpan.FromDays(2));
_ = bed.Breaker.State;
}
await bed.Breaker.FlushAsync();
var proposals = (await bed.Db.Events.ListAsync(new EventQuery())).Where(e => e.Code == EngineEventCodes.ProposeMorePrudence).ToList();
Assert.HasCount(1, proposals);
StringAssert.Contains(proposals[0].Message, "«Equilibrato»");
}
[TestMethod]
public async Task TwoThrottlesInSevenDaysAreNotYetAProposal()
{
await using var bed = await EngineTestBed.CreateAsync();
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
bed.Clock.Advance(TimeSpan.FromDays(8));
_ = bed.Breaker.State;
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
bed.Clock.Advance(TimeSpan.FromDays(2));
_ = bed.Breaker.State;
await bed.Breaker.ReportAsync(Throttle(bed.Clock.Now));
await bed.Breaker.FlushAsync();
Assert.IsFalse((await bed.Db.Events.ListAsync(new EventQuery())).Any(e => e.Code == EngineEventCodes.ProposeMorePrudence));
}
}