Files
Anubis/tests/Anubis.Tests/CollectorTests.cs
T
Alby96andClaude Opus 5.5 094a527ba2
ci / test (postgres) (push) Successful in 2m48s
ci / test (sqlite) (push) Successful in 2m10s
ci / test (postgres) (pull_request) Failing after 3m37s
ci / test (sqlite) (pull_request) Successful in 2m20s
ci / image (pull_request) Skipped
ci / image (push) Successful in 21s
Make two tests steady on a busy CI runner (refs #20 #25)
- The skipped-minute test waited for the recovered bar, but the issue is
  closed a moment after the bar is stored; on PostgreSQL in CI (run 181) it
  read the issue in between and saw it still open. It now waits for the
  issue's final state.
- Creating a PostgreSQL test database retries on a connection timeout: CI
  run 179 on main failed because twelve workers created their databases at
  once and one connection missed the application's 5-second timeout. The
  application's timeout is unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 17:23:58 +01:00

310 lines
17 KiB
C#

using System.Threading.Channels;
using Anubis.Core;
using Anubis.Data;
namespace Anubis.Tests;
/// <summary>The collector of issue #20: backfill, checks, second readings, live feed, drops and gap recovery.</summary>
[TestClass]
public sealed class CollectorTests
{
[TestMethod]
[TestCategory(Category.Integration)]
public async Task FeedDetachedForTenMinutesIsRecoveredOnReconnection()
{
var clock = new FastClock(new DateTimeOffset(2026, 11, 10, 10, 0, 30, TimeSpan.Zero));
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var bus = new EventBus();
using var feed = bus.Subscribe<FeedChanged>();
var freshness = new FreshnessMonitor(clock, bus);
var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false);
var source = new FakeMarketSource(clock);
var quiet = new DateTimeOffset(2026, 11, 10, 9, 30, 0, TimeSpan.Zero);
source.NoTrades.Add(quiet);
var collector = new MarketCollector(source, test.Db, clock, bus, freshness);
var bars = new BarStore(test.Db, clock);
var issues = new DataIssueStore(test.Db, clock);
using var stop = new CancellationTokenSource();
var live = collector.RunLiveAsync([btc], stop.Token);
await OkxTests.WaitForAsync(() => source.Connections == 1).ConfigureAwait(false);
// Startup: two days of minutes downloaded; the minute without trades is recorded, not invented.
var start = clock.UtcNow;
Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 9, 59, 0, TimeSpan.Zero), await bars.LastMinuteAsync(btc.Id, CancellationToken.None).ConfigureAwait(false));
var startup = await bars.MinuteAsync(btc.Id, start.AddDays(-3), start, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(2 * 24 * 60 - 1, startup.Count);
var unrecoverable = (await issues.SinceAsync(start.AddDays(-1), CancellationToken.None).ConfigureAwait(false)).Single();
Assert.AreEqual((IssueKinds.Gap, IssueStatus.Unrecoverable, quiet), (unrecoverable.Kind, unrecoverable.Status, unrecoverable.StartsAt));
StringAssert.Contains(unrecoverable.Detail, "senza scambi");
// Live: a closed bar arrives and the instrument is fresh.
clock.Advance(TimeSpan.FromSeconds(31));
await source.Live!.Writer.WriteAsync(new BarEvent("BTC-EUR", FakeMarketSource.MinuteBar(new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero)), true))
.ConfigureAwait(false);
await OkxTests.WaitForAsync(() => freshness.IsFresh(btc.Id)).ConfigureAwait(false);
Assert.IsTrue(collector.Connected);
// The network goes down for ten minutes: the feed drops and nothing answers, not even the REST calls.
var dropAt = clock.UtcNow;
source.DownUntil = dropAt.AddMinutes(10);
source.Live.Writer.Complete();
await OkxTests.WaitForAsync(() => source.Connections == 2).ConfigureAwait(false);
Assert.IsTrue(clock.UtcNow >= dropAt.AddMinutes(10), "reconnected only after the outage");
Assert.IsFalse(freshness.IsFresh(btc.Id), "no live datum since the drop: no new entries");
// Back: every minute missed is downloaded before the feed resumes.
var resumed = new DateTimeOffset(clock.UtcNow.UtcTicks - (clock.UtcNow.UtcTicks % TimeSpan.TicksPerMinute), TimeSpan.Zero);
var recovered = await bars.MinuteAsync(btc.Id, new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero), resumed, CancellationToken.None).ConfigureAwait(false);
Assert.IsTrue(recovered.Count >= 11, $"at least the ten minutes of the outage, got {recovered.Count}");
CollectionAssert.AreEqual(
QualityChecks.EveryMinute(new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero), resumed).ToList(),
recovered.Select(b => b.OpensAt).ToList(), "no minute is missing");
Assert.AreEqual(1, (await issues.SinceAsync(start.AddDays(-1), CancellationToken.None).ConfigureAwait(false)).Count, "no new issue");
await source.Live!.Writer.WriteAsync(new TradeEvent("BTC-EUR", clock.UtcNow, 50_010m, 0.01m)).ConfigureAwait(false);
await OkxTests.WaitForAsync(() => freshness.IsFresh(btc.Id)).ConfigureAwait(false);
await stop.CancelAsync().ConfigureAwait(false);
await live.ConfigureAwait(false);
var changes = new List<bool>();
while (feed.TryRead(out var change))
{
changes.Add(change!.Connected);
}
CollectionAssert.AreEqual(new[] { true, false, true, false }, changes);
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task SuspectBarsAreReadAgainAndNeverFixedInSilence()
{
var day0 = new DateOnly(2026, 6, 1);
var clock = new FastClock(new DateTimeOffset(day0.AddDays(120).ToDateTime(new TimeOnly(6, 0)), TimeSpan.Zero));
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var bus = new EventBus();
var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false);
var walk = MarketDataLogicTests.Walk(120, day0);
var source = new FakeMarketSource(clock);
foreach (var bar in walk)
{
source.Days[bar.Day] = bar;
}
var spike = day0.AddDays(80);
var broken = day0.AddDays(90);
var lost = day0.AddDays(50);
source.FirstReadings[spike] = new Queue<DailyBar>([walk[80] with { High = walk[80].High * 3, Close = walk[80].Close * 3 }]);
source.Days[broken] = walk[90] with { Low = -1 };
source.MissingOnFirstRead.Add(lost);
var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus));
var result = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false);
var bars = new BarStore(test.Db, clock);
var stored = await bars.DailyAsync(btc.Id, day0, day0.AddDays(200), CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(119, stored.Count, "the impossible day is dropped, the lost one recovered");
Assert.AreEqual(walk[80].Close, stored.Single(b => b.Day == spike).Close, "the second reading replaced the spike");
Assert.IsFalse(stored.Any(b => b.Day == broken));
Assert.IsTrue(stored.Any(b => b.Day == lost));
var issues = await new DataIssueStore(test.Db, clock).SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false);
var byDay = issues.ToLookup(i => DateOnly.FromDateTime(i.StartsAt.UtcDateTime));
Assert.AreEqual(IssueStatus.Corrected, byDay[spike].Single(i => i.Kind == IssueKinds.Jump).Status);
StringAssert.Contains(byDay[spike].Single(i => i.Kind == IssueKinds.Jump).Detail, "invece di");
Assert.AreEqual(IssueStatus.Unrecoverable, byDay[broken].Single(i => i.Kind == IssueKinds.NonPositive).Status);
Assert.AreEqual(IssueStatus.Recovered, byDay[lost].Single(i => i.Kind == IssueKinds.Gap).Status);
Assert.IsTrue(issues.All(i => i.Attempts == 1), "each problem was tried once more");
Assert.AreEqual(0, (await new DataIssueStore(test.Db, clock).BlockedInstrumentsAsync(CancellationToken.None).ConfigureAwait(false)).Count,
"once read again, nothing blocks new entries");
Assert.IsTrue(result.Suspect >= 2 && result.Corrected >= 1 && result.Unrecoverable == 1 && result.Recovered == 1, result.ToString());
// A suspect bar the second reading confirms stays, marked as confirmed.
source.Days[day0.AddDays(120)] = walk[^1] with { Day = day0.AddDays(120), Open = walk[^1].Close, Low = walk[^1].Close * 0.5, Close = walk[^1].Close * 0.5 };
clock.Advance(TimeSpan.FromDays(1));
var next = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(1, next.Confirmed);
Assert.AreEqual(BarQuality.Confirmed, (await bars.DailyAsync(btc.Id, day0.AddDays(120), day0.AddDays(120), CancellationToken.None).ConfigureAwait(false))[0].Quality);
Assert.AreEqual(CollectionResult.None, await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false), "nothing new");
// The first day after the stored history is missing on the first read: the hole between what is held and
// what arrived is a gap like any other, read again and recovered.
var basis = source.Days[day0.AddDays(120)];
foreach (var day in new[] { day0.AddDays(121), day0.AddDays(122) })
{
source.Days[day] = basis with { Day = day, Open = basis.Close, High = basis.Close * 1.002, Low = basis.Close * 0.998, Quality = BarQuality.Ok };
}
source.MissingOnFirstRead.Add(day0.AddDays(121));
clock.Advance(TimeSpan.FromDays(2));
var after = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual((1, 1), (after.Gaps, after.Recovered), after.ToString());
Assert.AreEqual(3, (await bars.DailyAsync(btc.Id, day0.AddDays(120), day0.AddDays(122), CancellationToken.None).ConfigureAwait(false)).Count);
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task LiveMinuteSkippedByTheFeedIsDownloadedAgain()
{
var clock = new FastClock(new DateTimeOffset(2026, 11, 10, 10, 0, 30, TimeSpan.Zero));
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var bus = new EventBus();
var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false);
var source = new FakeMarketSource(clock);
var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus));
var bars = new BarStore(test.Db, clock);
var issues = new DataIssueStore(test.Db, clock);
using var stop = new CancellationTokenSource();
var live = collector.RunLiveAsync([btc], stop.Token);
await OkxTests.WaitForAsync(() => source.Connections == 1).ConfigureAwait(false);
Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 9, 59, 0, TimeSpan.Zero), await bars.LastMinuteAsync(btc.Id, CancellationToken.None).ConfigureAwait(false));
// The stream was subscribed too late for the 10:00 candle and sends 10:01 first.
var skipped = new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero);
clock.Advance(TimeSpan.FromMinutes(2));
await source.Live!.Writer.WriteAsync(new BarEvent("BTC-EUR", FakeMarketSource.MinuteBar(skipped.AddMinutes(1)), true)).ConfigureAwait(false);
// The minute is stored first and the issue closed right after: wait for the issue's final state.
await OkxTests.WaitForAsync(() => issues.SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).GetAwaiter().GetResult()
.Any(i => i.Status != IssueStatus.Open)).ConfigureAwait(false);
var gap = (await issues.SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false)).Single();
Assert.AreEqual((IssueKinds.Gap, IssueStatus.Recovered, skipped), (gap.Kind, gap.Status, gap.StartsAt));
Assert.AreEqual(2, (await bars.MinuteAsync(btc.Id, skipped, skipped.AddMinutes(2), CancellationToken.None).ConfigureAwait(false)).Count);
await stop.CancelAsync().ConfigureAwait(false);
await live.ConfigureAwait(false);
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task ASplitAfterTheHistoryRebasesTheStoredAdjustedPrices()
{
var day0 = new DateOnly(2026, 6, 1);
var clock = new FastClock(new DateTimeOffset(day0.AddDays(100).ToDateTime(new TimeOnly(6, 0)), TimeSpan.Zero));
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var bus = new EventBus();
var stock = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("ACME-EUR"), CancellationToken.None).ConfigureAwait(false);
static DailyBar Adjusted(DailyBar bar, double factor) =>
bar with { Adjusted = new Ohlc(bar.Open * factor, bar.High * factor, bar.Low * factor, bar.Close * factor) };
var walk = MarketDataLogicTests.Walk(100, day0);
var source = new FakeMarketSource(clock);
foreach (var bar in walk)
{
source.Days[bar.Day] = Adjusted(bar, 1);
}
var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus));
Assert.AreEqual(100, (await collector.BackfillDailyAsync(stock, day0, CancellationToken.None).ConfigureAwait(false)).Stored);
// A 2-for-1 split on day 100: the source now gives every older adjusted price halved, and the real price halves.
foreach (var bar in walk)
{
source.Days[bar.Day] = Adjusted(bar, 0.5);
}
var close = walk[^1].Close / 2;
source.Days[day0.AddDays(100)] = Adjusted(new DailyBar(day0.AddDays(100), close, close * 1.003, close * 0.997, close * 1.001, 2000), 1);
clock.Advance(TimeSpan.FromDays(1));
var result = await collector.BackfillDailyAsync(stock, day0, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual((1, 0), (result.Stored, result.Suspect), "the split is not a jump");
var stored = await new BarStore(test.Db, clock).DailyAsync(stock.Id, day0, day0.AddDays(100), CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(walk[50].Close, stored[50].Close, 1e-9, "real prices stay as they were");
Assert.AreEqual(walk[50].Close / 2, stored[50].Adjusted!.Value.Close, 1e-9, "older adjusted prices are on the new basis");
Assert.IsEmpty(await new DataIssueStore(test.Db, clock).SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false));
}
}
/// <summary>
/// A venue for the collector tests: minute bars exist for every past minute except <see cref="NoTrades"/>, daily
/// bars come from <see cref="Days"/>, and the whole venue can be down until <see cref="DownUntil"/>.
/// </summary>
internal sealed class FakeMarketSource(IClock clock) : IMarketDataSource
{
public string Venue => Venues.Okx;
public HashSet<DateTimeOffset> NoTrades { get; } = [];
public Dictionary<DateOnly, DailyBar> Days { get; } = [];
/// <summary>What the first reading of a day returns instead of <see cref="Days"/>.</summary>
public Dictionary<DateOnly, Queue<DailyBar>> FirstReadings { get; } = [];
public HashSet<DateOnly> MissingOnFirstRead { get; } = [];
public DateTimeOffset? DownUntil { get; set; }
public Channel<MarketEvent>? Live { get; private set; }
public int Connections;
public static Bar MinuteBar(DateTimeOffset t)
{
var price = 50_000 + (20 * Math.Sin(t.ToUnixTimeSeconds() / 60.0 / 15));
return new Bar(t, price, price + 5, price - 5, price + 1, 0.5);
}
public Task<IReadOnlyList<InstrumentSpec>> ListInstrumentsAsync(CancellationToken cancellationToken) =>
Task.FromResult<IReadOnlyList<InstrumentSpec>>([MarketDataStoreTests.Okx("BTC-EUR")]);
public Task<IReadOnlyList<DailyBar>> FetchDailyBarsAsync(string symbol, DateOnly first, DateOnly last, CancellationToken cancellationToken)
{
ThrowIfDown();
var list = new List<DailyBar>();
for (var day = first; day <= last; day = day.AddDays(1))
{
if (FirstReadings.TryGetValue(day, out var queue) && queue.Count > 0)
{
list.Add(queue.Dequeue());
}
else if (!MissingOnFirstRead.Remove(day) && Days.TryGetValue(day, out var bar))
{
list.Add(bar);
}
}
return Task.FromResult<IReadOnlyList<DailyBar>>(list);
}
public Task<IReadOnlyList<Bar>> FetchMinuteBarsAsync(string symbol, DateTimeOffset start, DateTimeOffset until, CancellationToken cancellationToken)
{
ThrowIfDown();
var now = clock.UtcNow;
var list = new List<Bar>();
for (var t = start; t < until && t.AddMinutes(1) <= now; t = t.AddMinutes(1))
{
if (!NoTrades.Contains(t))
{
list.Add(MinuteBar(t));
}
}
return Task.FromResult<IReadOnlyList<Bar>>(list);
}
public Task<Quote> FetchQuoteAsync(string symbol, CancellationToken cancellationToken) =>
Task.FromResult(new Quote(symbol, 49_999m, 50_001m, clock.UtcNow));
public async IAsyncEnumerable<MarketEvent> StreamAsync(IReadOnlyCollection<string> symbols,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
ThrowIfDown();
Live = Channel.CreateUnbounded<MarketEvent>();
Interlocked.Increment(ref Connections);
await foreach (var item in Live.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return item;
}
}
private void ThrowIfDown()
{
if (DownUntil is { } until && clock.UtcNow < until)
{
throw new MarketDataException("rete assente", retryable: true);
}
}
}