using System.Threading.Channels; using Anubis.Core; using Anubis.Data; namespace Anubis.Tests; /// The collector of issue #20: backfill, checks, second readings, live feed, drops and gap recovery. [TestClass] public sealed class CollectorTests { [TestMethod] [TestCategory(Category.Integration)] public async Task FeedDetachedForTenMinutesIsRecoveredOnReconnection() { var clock = new FastClock(new DateTimeOffset(2026, 11, 10, 10, 0, 30, TimeSpan.Zero)); await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); var bus = new EventBus(); using var feed = bus.Subscribe(); var freshness = new FreshnessMonitor(clock, bus); var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false); var source = new FakeMarketSource(clock); var quiet = new DateTimeOffset(2026, 11, 10, 9, 30, 0, TimeSpan.Zero); source.NoTrades.Add(quiet); var collector = new MarketCollector(source, test.Db, clock, bus, freshness); var bars = new BarStore(test.Db, clock); var issues = new DataIssueStore(test.Db, clock); using var stop = new CancellationTokenSource(); var live = collector.RunLiveAsync([btc], stop.Token); await OkxTests.WaitForAsync(() => source.Connections == 1).ConfigureAwait(false); // Startup: two days of minutes downloaded; the minute without trades is recorded, not invented. var start = clock.UtcNow; Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 9, 59, 0, TimeSpan.Zero), await bars.LastMinuteAsync(btc.Id, CancellationToken.None).ConfigureAwait(false)); var startup = await bars.MinuteAsync(btc.Id, start.AddDays(-3), start, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(2 * 24 * 60 - 1, startup.Count); var unrecoverable = (await issues.SinceAsync(start.AddDays(-1), CancellationToken.None).ConfigureAwait(false)).Single(); Assert.AreEqual((IssueKinds.Gap, IssueStatus.Unrecoverable, quiet), (unrecoverable.Kind, unrecoverable.Status, unrecoverable.StartsAt)); StringAssert.Contains(unrecoverable.Detail, "senza scambi"); // Live: a closed bar arrives and the instrument is fresh. clock.Advance(TimeSpan.FromSeconds(31)); await source.Live!.Writer.WriteAsync(new BarEvent("BTC-EUR", FakeMarketSource.MinuteBar(new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero)), true)) .ConfigureAwait(false); await OkxTests.WaitForAsync(() => freshness.IsFresh(btc.Id)).ConfigureAwait(false); Assert.IsTrue(collector.Connected); // The network goes down for ten minutes: the feed drops and nothing answers, not even the REST calls. var dropAt = clock.UtcNow; source.DownUntil = dropAt.AddMinutes(10); source.Live.Writer.Complete(); await OkxTests.WaitForAsync(() => source.Connections == 2).ConfigureAwait(false); Assert.IsTrue(clock.UtcNow >= dropAt.AddMinutes(10), "reconnected only after the outage"); Assert.IsFalse(freshness.IsFresh(btc.Id), "no live datum since the drop: no new entries"); // Back: every minute missed is downloaded before the feed resumes. var resumed = new DateTimeOffset(clock.UtcNow.UtcTicks - (clock.UtcNow.UtcTicks % TimeSpan.TicksPerMinute), TimeSpan.Zero); var recovered = await bars.MinuteAsync(btc.Id, new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero), resumed, CancellationToken.None).ConfigureAwait(false); Assert.IsTrue(recovered.Count >= 11, $"at least the ten minutes of the outage, got {recovered.Count}"); CollectionAssert.AreEqual( QualityChecks.EveryMinute(new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero), resumed).ToList(), recovered.Select(b => b.OpensAt).ToList(), "no minute is missing"); Assert.AreEqual(1, (await issues.SinceAsync(start.AddDays(-1), CancellationToken.None).ConfigureAwait(false)).Count, "no new issue"); await source.Live!.Writer.WriteAsync(new TradeEvent("BTC-EUR", clock.UtcNow, 50_010m, 0.01m)).ConfigureAwait(false); await OkxTests.WaitForAsync(() => freshness.IsFresh(btc.Id)).ConfigureAwait(false); await stop.CancelAsync().ConfigureAwait(false); await live.ConfigureAwait(false); var changes = new List(); while (feed.TryRead(out var change)) { changes.Add(change!.Connected); } CollectionAssert.AreEqual(new[] { true, false, true, false }, changes); } [TestMethod] [TestCategory(Category.Integration)] public async Task SuspectBarsAreReadAgainAndNeverFixedInSilence() { var day0 = new DateOnly(2026, 6, 1); var clock = new FastClock(new DateTimeOffset(day0.AddDays(120).ToDateTime(new TimeOnly(6, 0)), TimeSpan.Zero)); await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); var bus = new EventBus(); var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false); var walk = MarketDataLogicTests.Walk(120, day0); var source = new FakeMarketSource(clock); foreach (var bar in walk) { source.Days[bar.Day] = bar; } var spike = day0.AddDays(80); var broken = day0.AddDays(90); var lost = day0.AddDays(50); source.FirstReadings[spike] = new Queue([walk[80] with { High = walk[80].High * 3, Close = walk[80].Close * 3 }]); source.Days[broken] = walk[90] with { Low = -1 }; source.MissingOnFirstRead.Add(lost); var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus)); var result = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false); var bars = new BarStore(test.Db, clock); var stored = await bars.DailyAsync(btc.Id, day0, day0.AddDays(200), CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(119, stored.Count, "the impossible day is dropped, the lost one recovered"); Assert.AreEqual(walk[80].Close, stored.Single(b => b.Day == spike).Close, "the second reading replaced the spike"); Assert.IsFalse(stored.Any(b => b.Day == broken)); Assert.IsTrue(stored.Any(b => b.Day == lost)); var issues = await new DataIssueStore(test.Db, clock).SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false); var byDay = issues.ToLookup(i => DateOnly.FromDateTime(i.StartsAt.UtcDateTime)); Assert.AreEqual(IssueStatus.Corrected, byDay[spike].Single(i => i.Kind == IssueKinds.Jump).Status); StringAssert.Contains(byDay[spike].Single(i => i.Kind == IssueKinds.Jump).Detail, "invece di"); Assert.AreEqual(IssueStatus.Unrecoverable, byDay[broken].Single(i => i.Kind == IssueKinds.NonPositive).Status); Assert.AreEqual(IssueStatus.Recovered, byDay[lost].Single(i => i.Kind == IssueKinds.Gap).Status); Assert.IsTrue(issues.All(i => i.Attempts == 1), "each problem was tried once more"); Assert.AreEqual(0, (await new DataIssueStore(test.Db, clock).BlockedInstrumentsAsync(CancellationToken.None).ConfigureAwait(false)).Count, "once read again, nothing blocks new entries"); Assert.IsTrue(result.Suspect >= 2 && result.Corrected >= 1 && result.Unrecoverable == 1 && result.Recovered == 1, result.ToString()); // A suspect bar the second reading confirms stays, marked as confirmed. source.Days[day0.AddDays(120)] = walk[^1] with { Day = day0.AddDays(120), Open = walk[^1].Close, Low = walk[^1].Close * 0.5, Close = walk[^1].Close * 0.5 }; clock.Advance(TimeSpan.FromDays(1)); var next = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(1, next.Confirmed); Assert.AreEqual(BarQuality.Confirmed, (await bars.DailyAsync(btc.Id, day0.AddDays(120), day0.AddDays(120), CancellationToken.None).ConfigureAwait(false))[0].Quality); Assert.AreEqual(CollectionResult.None, await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false), "nothing new"); // The first day after the stored history is missing on the first read: the hole between what is held and // what arrived is a gap like any other, read again and recovered. var basis = source.Days[day0.AddDays(120)]; foreach (var day in new[] { day0.AddDays(121), day0.AddDays(122) }) { source.Days[day] = basis with { Day = day, Open = basis.Close, High = basis.Close * 1.002, Low = basis.Close * 0.998, Quality = BarQuality.Ok }; } source.MissingOnFirstRead.Add(day0.AddDays(121)); clock.Advance(TimeSpan.FromDays(2)); var after = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual((1, 1), (after.Gaps, after.Recovered), after.ToString()); Assert.AreEqual(3, (await bars.DailyAsync(btc.Id, day0.AddDays(120), day0.AddDays(122), CancellationToken.None).ConfigureAwait(false)).Count); } [TestMethod] [TestCategory(Category.Integration)] public async Task LiveMinuteSkippedByTheFeedIsDownloadedAgain() { var clock = new FastClock(new DateTimeOffset(2026, 11, 10, 10, 0, 30, TimeSpan.Zero)); await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); var bus = new EventBus(); var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false); var source = new FakeMarketSource(clock); var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus)); var bars = new BarStore(test.Db, clock); var issues = new DataIssueStore(test.Db, clock); using var stop = new CancellationTokenSource(); var live = collector.RunLiveAsync([btc], stop.Token); await OkxTests.WaitForAsync(() => source.Connections == 1).ConfigureAwait(false); Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 9, 59, 0, TimeSpan.Zero), await bars.LastMinuteAsync(btc.Id, CancellationToken.None).ConfigureAwait(false)); // The stream was subscribed too late for the 10:00 candle and sends 10:01 first. var skipped = new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero); clock.Advance(TimeSpan.FromMinutes(2)); await source.Live!.Writer.WriteAsync(new BarEvent("BTC-EUR", FakeMarketSource.MinuteBar(skipped.AddMinutes(1)), true)).ConfigureAwait(false); // The minute is stored first and the issue closed right after: wait for the issue's final state. await OkxTests.WaitForAsync(() => issues.SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).GetAwaiter().GetResult() .Any(i => i.Status != IssueStatus.Open)).ConfigureAwait(false); var gap = (await issues.SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false)).Single(); Assert.AreEqual((IssueKinds.Gap, IssueStatus.Recovered, skipped), (gap.Kind, gap.Status, gap.StartsAt)); Assert.AreEqual(2, (await bars.MinuteAsync(btc.Id, skipped, skipped.AddMinutes(2), CancellationToken.None).ConfigureAwait(false)).Count); await stop.CancelAsync().ConfigureAwait(false); await live.ConfigureAwait(false); } [TestMethod] [TestCategory(Category.Integration)] public async Task ASplitAfterTheHistoryRebasesTheStoredAdjustedPrices() { var day0 = new DateOnly(2026, 6, 1); var clock = new FastClock(new DateTimeOffset(day0.AddDays(100).ToDateTime(new TimeOnly(6, 0)), TimeSpan.Zero)); await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); var bus = new EventBus(); var stock = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("ACME-EUR"), CancellationToken.None).ConfigureAwait(false); static DailyBar Adjusted(DailyBar bar, double factor) => bar with { Adjusted = new Ohlc(bar.Open * factor, bar.High * factor, bar.Low * factor, bar.Close * factor) }; var walk = MarketDataLogicTests.Walk(100, day0); var source = new FakeMarketSource(clock); foreach (var bar in walk) { source.Days[bar.Day] = Adjusted(bar, 1); } var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus)); Assert.AreEqual(100, (await collector.BackfillDailyAsync(stock, day0, CancellationToken.None).ConfigureAwait(false)).Stored); // A 2-for-1 split on day 100: the source now gives every older adjusted price halved, and the real price halves. foreach (var bar in walk) { source.Days[bar.Day] = Adjusted(bar, 0.5); } var close = walk[^1].Close / 2; source.Days[day0.AddDays(100)] = Adjusted(new DailyBar(day0.AddDays(100), close, close * 1.003, close * 0.997, close * 1.001, 2000), 1); clock.Advance(TimeSpan.FromDays(1)); var result = await collector.BackfillDailyAsync(stock, day0, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual((1, 0), (result.Stored, result.Suspect), "the split is not a jump"); var stored = await new BarStore(test.Db, clock).DailyAsync(stock.Id, day0, day0.AddDays(100), CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(walk[50].Close, stored[50].Close, 1e-9, "real prices stay as they were"); Assert.AreEqual(walk[50].Close / 2, stored[50].Adjusted!.Value.Close, 1e-9, "older adjusted prices are on the new basis"); Assert.IsEmpty(await new DataIssueStore(test.Db, clock).SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false)); } } /// /// A venue for the collector tests: minute bars exist for every past minute except , daily /// bars come from , and the whole venue can be down until . /// internal sealed class FakeMarketSource(IClock clock) : IMarketDataSource { public string Venue => Venues.Okx; public HashSet NoTrades { get; } = []; public Dictionary Days { get; } = []; /// What the first reading of a day returns instead of . public Dictionary> FirstReadings { get; } = []; public HashSet MissingOnFirstRead { get; } = []; public DateTimeOffset? DownUntil { get; set; } public Channel? Live { get; private set; } public int Connections; public static Bar MinuteBar(DateTimeOffset t) { var price = 50_000 + (20 * Math.Sin(t.ToUnixTimeSeconds() / 60.0 / 15)); return new Bar(t, price, price + 5, price - 5, price + 1, 0.5); } public Task> ListInstrumentsAsync(CancellationToken cancellationToken) => Task.FromResult>([MarketDataStoreTests.Okx("BTC-EUR")]); public Task> FetchDailyBarsAsync(string symbol, DateOnly first, DateOnly last, CancellationToken cancellationToken) { ThrowIfDown(); var list = new List(); for (var day = first; day <= last; day = day.AddDays(1)) { if (FirstReadings.TryGetValue(day, out var queue) && queue.Count > 0) { list.Add(queue.Dequeue()); } else if (!MissingOnFirstRead.Remove(day) && Days.TryGetValue(day, out var bar)) { list.Add(bar); } } return Task.FromResult>(list); } public Task> FetchMinuteBarsAsync(string symbol, DateTimeOffset start, DateTimeOffset until, CancellationToken cancellationToken) { ThrowIfDown(); var now = clock.UtcNow; var list = new List(); for (var t = start; t < until && t.AddMinutes(1) <= now; t = t.AddMinutes(1)) { if (!NoTrades.Contains(t)) { list.Add(MinuteBar(t)); } } return Task.FromResult>(list); } public Task FetchQuoteAsync(string symbol, CancellationToken cancellationToken) => Task.FromResult(new Quote(symbol, 49_999m, 50_001m, clock.UtcNow)); public async IAsyncEnumerable StreamAsync(IReadOnlyCollection symbols, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { ThrowIfDown(); Live = Channel.CreateUnbounded(); Interlocked.Increment(ref Connections); await foreach (var item in Live.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { yield return item; } } private void ThrowIfDown() { if (DownUntil is { } until && clock.UtcNow < until) { throw new MarketDataException("rete assente", retryable: true); } } }