ci / test (postgres) (push) Successful in 2m48s
ci / test (sqlite) (push) Successful in 2m10s
ci / test (postgres) (pull_request) Failing after 3m37s
ci / test (sqlite) (pull_request) Successful in 2m20s
ci / image (pull_request) Skipped
ci / image (push) Successful in 21s
- The skipped-minute test waited for the recovered bar, but the issue is closed a moment after the bar is stored; on PostgreSQL in CI (run 181) it read the issue in between and saw it still open. It now waits for the issue's final state. - Creating a PostgreSQL test database retries on a connection timeout: CI run 179 on main failed because twelve workers created their databases at once and one connection missed the application's 5-second timeout. The application's timeout is unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
310 lines
17 KiB
C#
310 lines
17 KiB
C#
using System.Threading.Channels;
|
|
using Anubis.Core;
|
|
using Anubis.Data;
|
|
|
|
namespace Anubis.Tests;
|
|
|
|
/// <summary>The collector of issue #20: backfill, checks, second readings, live feed, drops and gap recovery.</summary>
|
|
[TestClass]
|
|
public sealed class CollectorTests
|
|
{
|
|
[TestMethod]
|
|
[TestCategory(Category.Integration)]
|
|
public async Task FeedDetachedForTenMinutesIsRecoveredOnReconnection()
|
|
{
|
|
var clock = new FastClock(new DateTimeOffset(2026, 11, 10, 10, 0, 30, TimeSpan.Zero));
|
|
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
|
var bus = new EventBus();
|
|
using var feed = bus.Subscribe<FeedChanged>();
|
|
var freshness = new FreshnessMonitor(clock, bus);
|
|
var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false);
|
|
var source = new FakeMarketSource(clock);
|
|
var quiet = new DateTimeOffset(2026, 11, 10, 9, 30, 0, TimeSpan.Zero);
|
|
source.NoTrades.Add(quiet);
|
|
var collector = new MarketCollector(source, test.Db, clock, bus, freshness);
|
|
var bars = new BarStore(test.Db, clock);
|
|
var issues = new DataIssueStore(test.Db, clock);
|
|
|
|
using var stop = new CancellationTokenSource();
|
|
var live = collector.RunLiveAsync([btc], stop.Token);
|
|
await OkxTests.WaitForAsync(() => source.Connections == 1).ConfigureAwait(false);
|
|
|
|
// Startup: two days of minutes downloaded; the minute without trades is recorded, not invented.
|
|
var start = clock.UtcNow;
|
|
Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 9, 59, 0, TimeSpan.Zero), await bars.LastMinuteAsync(btc.Id, CancellationToken.None).ConfigureAwait(false));
|
|
var startup = await bars.MinuteAsync(btc.Id, start.AddDays(-3), start, CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual(2 * 24 * 60 - 1, startup.Count);
|
|
var unrecoverable = (await issues.SinceAsync(start.AddDays(-1), CancellationToken.None).ConfigureAwait(false)).Single();
|
|
Assert.AreEqual((IssueKinds.Gap, IssueStatus.Unrecoverable, quiet), (unrecoverable.Kind, unrecoverable.Status, unrecoverable.StartsAt));
|
|
StringAssert.Contains(unrecoverable.Detail, "senza scambi");
|
|
|
|
// Live: a closed bar arrives and the instrument is fresh.
|
|
clock.Advance(TimeSpan.FromSeconds(31));
|
|
await source.Live!.Writer.WriteAsync(new BarEvent("BTC-EUR", FakeMarketSource.MinuteBar(new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero)), true))
|
|
.ConfigureAwait(false);
|
|
await OkxTests.WaitForAsync(() => freshness.IsFresh(btc.Id)).ConfigureAwait(false);
|
|
Assert.IsTrue(collector.Connected);
|
|
|
|
// The network goes down for ten minutes: the feed drops and nothing answers, not even the REST calls.
|
|
var dropAt = clock.UtcNow;
|
|
source.DownUntil = dropAt.AddMinutes(10);
|
|
source.Live.Writer.Complete();
|
|
await OkxTests.WaitForAsync(() => source.Connections == 2).ConfigureAwait(false);
|
|
Assert.IsTrue(clock.UtcNow >= dropAt.AddMinutes(10), "reconnected only after the outage");
|
|
Assert.IsFalse(freshness.IsFresh(btc.Id), "no live datum since the drop: no new entries");
|
|
|
|
// Back: every minute missed is downloaded before the feed resumes.
|
|
var resumed = new DateTimeOffset(clock.UtcNow.UtcTicks - (clock.UtcNow.UtcTicks % TimeSpan.TicksPerMinute), TimeSpan.Zero);
|
|
var recovered = await bars.MinuteAsync(btc.Id, new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero), resumed, CancellationToken.None).ConfigureAwait(false);
|
|
Assert.IsTrue(recovered.Count >= 11, $"at least the ten minutes of the outage, got {recovered.Count}");
|
|
CollectionAssert.AreEqual(
|
|
QualityChecks.EveryMinute(new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero), resumed).ToList(),
|
|
recovered.Select(b => b.OpensAt).ToList(), "no minute is missing");
|
|
Assert.AreEqual(1, (await issues.SinceAsync(start.AddDays(-1), CancellationToken.None).ConfigureAwait(false)).Count, "no new issue");
|
|
|
|
await source.Live!.Writer.WriteAsync(new TradeEvent("BTC-EUR", clock.UtcNow, 50_010m, 0.01m)).ConfigureAwait(false);
|
|
await OkxTests.WaitForAsync(() => freshness.IsFresh(btc.Id)).ConfigureAwait(false);
|
|
|
|
await stop.CancelAsync().ConfigureAwait(false);
|
|
await live.ConfigureAwait(false);
|
|
var changes = new List<bool>();
|
|
while (feed.TryRead(out var change))
|
|
{
|
|
changes.Add(change!.Connected);
|
|
}
|
|
|
|
CollectionAssert.AreEqual(new[] { true, false, true, false }, changes);
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Integration)]
|
|
public async Task SuspectBarsAreReadAgainAndNeverFixedInSilence()
|
|
{
|
|
var day0 = new DateOnly(2026, 6, 1);
|
|
var clock = new FastClock(new DateTimeOffset(day0.AddDays(120).ToDateTime(new TimeOnly(6, 0)), TimeSpan.Zero));
|
|
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
|
var bus = new EventBus();
|
|
var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false);
|
|
var walk = MarketDataLogicTests.Walk(120, day0);
|
|
var source = new FakeMarketSource(clock);
|
|
foreach (var bar in walk)
|
|
{
|
|
source.Days[bar.Day] = bar;
|
|
}
|
|
|
|
var spike = day0.AddDays(80);
|
|
var broken = day0.AddDays(90);
|
|
var lost = day0.AddDays(50);
|
|
source.FirstReadings[spike] = new Queue<DailyBar>([walk[80] with { High = walk[80].High * 3, Close = walk[80].Close * 3 }]);
|
|
source.Days[broken] = walk[90] with { Low = -1 };
|
|
source.MissingOnFirstRead.Add(lost);
|
|
|
|
var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus));
|
|
var result = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false);
|
|
|
|
var bars = new BarStore(test.Db, clock);
|
|
var stored = await bars.DailyAsync(btc.Id, day0, day0.AddDays(200), CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual(119, stored.Count, "the impossible day is dropped, the lost one recovered");
|
|
Assert.AreEqual(walk[80].Close, stored.Single(b => b.Day == spike).Close, "the second reading replaced the spike");
|
|
Assert.IsFalse(stored.Any(b => b.Day == broken));
|
|
Assert.IsTrue(stored.Any(b => b.Day == lost));
|
|
|
|
var issues = await new DataIssueStore(test.Db, clock).SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false);
|
|
var byDay = issues.ToLookup(i => DateOnly.FromDateTime(i.StartsAt.UtcDateTime));
|
|
Assert.AreEqual(IssueStatus.Corrected, byDay[spike].Single(i => i.Kind == IssueKinds.Jump).Status);
|
|
StringAssert.Contains(byDay[spike].Single(i => i.Kind == IssueKinds.Jump).Detail, "invece di");
|
|
Assert.AreEqual(IssueStatus.Unrecoverable, byDay[broken].Single(i => i.Kind == IssueKinds.NonPositive).Status);
|
|
Assert.AreEqual(IssueStatus.Recovered, byDay[lost].Single(i => i.Kind == IssueKinds.Gap).Status);
|
|
Assert.IsTrue(issues.All(i => i.Attempts == 1), "each problem was tried once more");
|
|
Assert.AreEqual(0, (await new DataIssueStore(test.Db, clock).BlockedInstrumentsAsync(CancellationToken.None).ConfigureAwait(false)).Count,
|
|
"once read again, nothing blocks new entries");
|
|
Assert.IsTrue(result.Suspect >= 2 && result.Corrected >= 1 && result.Unrecoverable == 1 && result.Recovered == 1, result.ToString());
|
|
|
|
// A suspect bar the second reading confirms stays, marked as confirmed.
|
|
source.Days[day0.AddDays(120)] = walk[^1] with { Day = day0.AddDays(120), Open = walk[^1].Close, Low = walk[^1].Close * 0.5, Close = walk[^1].Close * 0.5 };
|
|
clock.Advance(TimeSpan.FromDays(1));
|
|
var next = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual(1, next.Confirmed);
|
|
Assert.AreEqual(BarQuality.Confirmed, (await bars.DailyAsync(btc.Id, day0.AddDays(120), day0.AddDays(120), CancellationToken.None).ConfigureAwait(false))[0].Quality);
|
|
Assert.AreEqual(CollectionResult.None, await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false), "nothing new");
|
|
|
|
// The first day after the stored history is missing on the first read: the hole between what is held and
|
|
// what arrived is a gap like any other, read again and recovered.
|
|
var basis = source.Days[day0.AddDays(120)];
|
|
foreach (var day in new[] { day0.AddDays(121), day0.AddDays(122) })
|
|
{
|
|
source.Days[day] = basis with { Day = day, Open = basis.Close, High = basis.Close * 1.002, Low = basis.Close * 0.998, Quality = BarQuality.Ok };
|
|
}
|
|
|
|
source.MissingOnFirstRead.Add(day0.AddDays(121));
|
|
clock.Advance(TimeSpan.FromDays(2));
|
|
var after = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual((1, 1), (after.Gaps, after.Recovered), after.ToString());
|
|
Assert.AreEqual(3, (await bars.DailyAsync(btc.Id, day0.AddDays(120), day0.AddDays(122), CancellationToken.None).ConfigureAwait(false)).Count);
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Integration)]
|
|
public async Task LiveMinuteSkippedByTheFeedIsDownloadedAgain()
|
|
{
|
|
var clock = new FastClock(new DateTimeOffset(2026, 11, 10, 10, 0, 30, TimeSpan.Zero));
|
|
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
|
var bus = new EventBus();
|
|
var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false);
|
|
var source = new FakeMarketSource(clock);
|
|
var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus));
|
|
var bars = new BarStore(test.Db, clock);
|
|
var issues = new DataIssueStore(test.Db, clock);
|
|
|
|
using var stop = new CancellationTokenSource();
|
|
var live = collector.RunLiveAsync([btc], stop.Token);
|
|
await OkxTests.WaitForAsync(() => source.Connections == 1).ConfigureAwait(false);
|
|
Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 9, 59, 0, TimeSpan.Zero), await bars.LastMinuteAsync(btc.Id, CancellationToken.None).ConfigureAwait(false));
|
|
|
|
// The stream was subscribed too late for the 10:00 candle and sends 10:01 first.
|
|
var skipped = new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero);
|
|
clock.Advance(TimeSpan.FromMinutes(2));
|
|
await source.Live!.Writer.WriteAsync(new BarEvent("BTC-EUR", FakeMarketSource.MinuteBar(skipped.AddMinutes(1)), true)).ConfigureAwait(false);
|
|
// The minute is stored first and the issue closed right after: wait for the issue's final state.
|
|
await OkxTests.WaitForAsync(() => issues.SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).GetAwaiter().GetResult()
|
|
.Any(i => i.Status != IssueStatus.Open)).ConfigureAwait(false);
|
|
|
|
var gap = (await issues.SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false)).Single();
|
|
Assert.AreEqual((IssueKinds.Gap, IssueStatus.Recovered, skipped), (gap.Kind, gap.Status, gap.StartsAt));
|
|
Assert.AreEqual(2, (await bars.MinuteAsync(btc.Id, skipped, skipped.AddMinutes(2), CancellationToken.None).ConfigureAwait(false)).Count);
|
|
await stop.CancelAsync().ConfigureAwait(false);
|
|
await live.ConfigureAwait(false);
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Integration)]
|
|
public async Task ASplitAfterTheHistoryRebasesTheStoredAdjustedPrices()
|
|
{
|
|
var day0 = new DateOnly(2026, 6, 1);
|
|
var clock = new FastClock(new DateTimeOffset(day0.AddDays(100).ToDateTime(new TimeOnly(6, 0)), TimeSpan.Zero));
|
|
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
|
var bus = new EventBus();
|
|
var stock = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("ACME-EUR"), CancellationToken.None).ConfigureAwait(false);
|
|
static DailyBar Adjusted(DailyBar bar, double factor) =>
|
|
bar with { Adjusted = new Ohlc(bar.Open * factor, bar.High * factor, bar.Low * factor, bar.Close * factor) };
|
|
|
|
var walk = MarketDataLogicTests.Walk(100, day0);
|
|
var source = new FakeMarketSource(clock);
|
|
foreach (var bar in walk)
|
|
{
|
|
source.Days[bar.Day] = Adjusted(bar, 1);
|
|
}
|
|
|
|
var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus));
|
|
Assert.AreEqual(100, (await collector.BackfillDailyAsync(stock, day0, CancellationToken.None).ConfigureAwait(false)).Stored);
|
|
|
|
// A 2-for-1 split on day 100: the source now gives every older adjusted price halved, and the real price halves.
|
|
foreach (var bar in walk)
|
|
{
|
|
source.Days[bar.Day] = Adjusted(bar, 0.5);
|
|
}
|
|
|
|
var close = walk[^1].Close / 2;
|
|
source.Days[day0.AddDays(100)] = Adjusted(new DailyBar(day0.AddDays(100), close, close * 1.003, close * 0.997, close * 1.001, 2000), 1);
|
|
clock.Advance(TimeSpan.FromDays(1));
|
|
var result = await collector.BackfillDailyAsync(stock, day0, CancellationToken.None).ConfigureAwait(false);
|
|
|
|
Assert.AreEqual((1, 0), (result.Stored, result.Suspect), "the split is not a jump");
|
|
var stored = await new BarStore(test.Db, clock).DailyAsync(stock.Id, day0, day0.AddDays(100), CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual(walk[50].Close, stored[50].Close, 1e-9, "real prices stay as they were");
|
|
Assert.AreEqual(walk[50].Close / 2, stored[50].Adjusted!.Value.Close, 1e-9, "older adjusted prices are on the new basis");
|
|
Assert.IsEmpty(await new DataIssueStore(test.Db, clock).SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// A venue for the collector tests: minute bars exist for every past minute except <see cref="NoTrades"/>, daily
|
|
/// bars come from <see cref="Days"/>, and the whole venue can be down until <see cref="DownUntil"/>.
|
|
/// </summary>
|
|
internal sealed class FakeMarketSource(IClock clock) : IMarketDataSource
|
|
{
|
|
public string Venue => Venues.Okx;
|
|
|
|
public HashSet<DateTimeOffset> NoTrades { get; } = [];
|
|
|
|
public Dictionary<DateOnly, DailyBar> Days { get; } = [];
|
|
|
|
/// <summary>What the first reading of a day returns instead of <see cref="Days"/>.</summary>
|
|
public Dictionary<DateOnly, Queue<DailyBar>> FirstReadings { get; } = [];
|
|
|
|
public HashSet<DateOnly> MissingOnFirstRead { get; } = [];
|
|
|
|
public DateTimeOffset? DownUntil { get; set; }
|
|
|
|
public Channel<MarketEvent>? Live { get; private set; }
|
|
|
|
public int Connections;
|
|
|
|
public static Bar MinuteBar(DateTimeOffset t)
|
|
{
|
|
var price = 50_000 + (20 * Math.Sin(t.ToUnixTimeSeconds() / 60.0 / 15));
|
|
return new Bar(t, price, price + 5, price - 5, price + 1, 0.5);
|
|
}
|
|
|
|
public Task<IReadOnlyList<InstrumentSpec>> ListInstrumentsAsync(CancellationToken cancellationToken) =>
|
|
Task.FromResult<IReadOnlyList<InstrumentSpec>>([MarketDataStoreTests.Okx("BTC-EUR")]);
|
|
|
|
public Task<IReadOnlyList<DailyBar>> FetchDailyBarsAsync(string symbol, DateOnly first, DateOnly last, CancellationToken cancellationToken)
|
|
{
|
|
ThrowIfDown();
|
|
var list = new List<DailyBar>();
|
|
for (var day = first; day <= last; day = day.AddDays(1))
|
|
{
|
|
if (FirstReadings.TryGetValue(day, out var queue) && queue.Count > 0)
|
|
{
|
|
list.Add(queue.Dequeue());
|
|
}
|
|
else if (!MissingOnFirstRead.Remove(day) && Days.TryGetValue(day, out var bar))
|
|
{
|
|
list.Add(bar);
|
|
}
|
|
}
|
|
|
|
return Task.FromResult<IReadOnlyList<DailyBar>>(list);
|
|
}
|
|
|
|
public Task<IReadOnlyList<Bar>> FetchMinuteBarsAsync(string symbol, DateTimeOffset start, DateTimeOffset until, CancellationToken cancellationToken)
|
|
{
|
|
ThrowIfDown();
|
|
var now = clock.UtcNow;
|
|
var list = new List<Bar>();
|
|
for (var t = start; t < until && t.AddMinutes(1) <= now; t = t.AddMinutes(1))
|
|
{
|
|
if (!NoTrades.Contains(t))
|
|
{
|
|
list.Add(MinuteBar(t));
|
|
}
|
|
}
|
|
|
|
return Task.FromResult<IReadOnlyList<Bar>>(list);
|
|
}
|
|
|
|
public Task<Quote> FetchQuoteAsync(string symbol, CancellationToken cancellationToken) =>
|
|
Task.FromResult(new Quote(symbol, 49_999m, 50_001m, clock.UtcNow));
|
|
|
|
public async IAsyncEnumerable<MarketEvent> StreamAsync(IReadOnlyCollection<string> symbols,
|
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
|
{
|
|
ThrowIfDown();
|
|
Live = Channel.CreateUnbounded<MarketEvent>();
|
|
Interlocked.Increment(ref Connections);
|
|
await foreach (var item in Live.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
yield return item;
|
|
}
|
|
}
|
|
|
|
private void ThrowIfDown()
|
|
{
|
|
if (DownUntil is { } until && clock.UtcNow < until)
|
|
{
|
|
throw new MarketDataException("rete assente", retryable: true);
|
|
}
|
|
}
|
|
}
|