diff --git a/docs/adr/0008-scelte-fase-2.md b/docs/adr/0008-scelte-fase-2.md index c82e09f..09b1752 100644 --- a/docs/adr/0008-scelte-fase-2.md +++ b/docs/adr/0008-scelte-fase-2.md @@ -24,7 +24,8 @@ La Fase 2 porta in Anubis i dati di mercato: catalogo degli strumenti, storico e - valori diversi: i nuovi lo sostituiscono (**corretto**); - nessun dato o ancora impossibile: il dato si scarta (**non recuperabile**). - Un buco si riscarica una volta. Ciò che manca ancora resta segnato come non recuperabile ("forse minuti senza scambi") e non blocca nulla. -- I buchi al minuto si cercano solo per la crypto, che scambia sempre. Per le azioni il controllo dei minuti arriva con il flusso dal vivo (Fase 3). +- Un buco tra l'ultimo dato salvato e il primo dato nuovo conta come gli altri: anche un minuto saltato dal flusso dal vivo si riscarica. +- I buchi al minuto si cercano solo per la crypto, che scambia sempre. Per le azioni il controllo dei minuti arriva con il flusso dal vivo (Fase 3); i giorni mancanti si cercano sulle sedute del calendario di borsa. **4. Solo le coppie in EUR di OKX Europe.** Il catalogo tiene le coppie spot quotate in EUR. Le coppie in USDC (più profonde per le monete minori, API-NOTES O18) restano fuori finché la Fase 3 non verifica con le chiavi demo quali si possono davvero negoziare. @@ -35,6 +36,11 @@ La Fase 2 porta in Anubis i dati di mercato: catalogo degli strumenti, storico e **6. Azioni societarie gestite.** Frazionamenti (diretti e inversi), dividendi in contanti, dividendi in azioni e cambi di simbolo. Gli altri tipi di Alpaca (unità, scorpori, fusioni) non si scaricano: un titolo che li subisce compare come salto nelle serie e il controllo dei salti lo segnala. +**6b. Prezzi rettificati sempre sulla stessa base.** +- Alpaca dà i prezzi rettificati sulla base del giorno in cui si scaricano. Ogni scaricamento riparte quindi dall'ultimo giorno già salvato, che fa da testimone. +- Se il rapporto tra prezzo rettificato e prezzo reale di quel giorno è cambiato, è arrivato un frazionamento o un dividendo: tutti i prezzi rettificati salvati si moltiplicano per lo stesso fattore prima dei controlli. +- Così un frazionamento non sembra un crollo, e la serie rettificata nel database resta continua. I prezzi reali non si toccano mai. + **7. Fondi riconosciuti anche dal nome.** La colonna ETF di `nasdaqtraded.txt` non segna molti ETN e nessun fondo chiuso. Un titolo è escluso dall'universo azionario se il file lo segna come ETF, oppure se il nome contiene parole come "ETF", "ETN", "Fund", "Closed-End", "Beneficial Interest", "Index", "Leveraged", "Preferred", "Warrant", "Right", "Unit", "Notes" (l'elenco completo è in `NasdaqTraded`). Meglio escludere un'azione vera che comprare un fondo vietato (SPEC 4.6). **8. Universi a una data passata.** diff --git a/src/Anubis/Brokers/Okx.cs b/src/Anubis/Brokers/Okx.cs index 4102005..df27db2 100644 --- a/src/Anubis/Brokers/Okx.cs +++ b/src/Anubis/Brokers/Okx.cs @@ -293,7 +293,11 @@ public sealed class OkxClient : IMarketDataSource foreach (var item in ParseStreamMessage(message)) { - await writer.WriteAsync(item, cancellationToken).ConfigureAwait(false); + // The channel is unbounded, so a write fails only once the other connection has ended the feed. + if (!writer.TryWrite(item)) + { + return; + } } } } @@ -425,7 +429,8 @@ public sealed class OkxClient : IMarketDataSource { await task.ConfigureAwait(false); } - catch (Exception ex) when (ex is OperationCanceledException or MarketDataException or System.Net.WebSockets.WebSocketException or JsonException) + catch (Exception ex) when (ex is OperationCanceledException or MarketDataException or System.Net.WebSockets.WebSocketException or JsonException + or ChannelClosedException or IOException or HttpRequestException or TimeoutException) { // Already reported through the channel, or the feed was stopped on purpose. } diff --git a/src/Anubis/Data/BarStore.cs b/src/Anubis/Data/BarStore.cs index dcc1a24..bd8e666 100644 --- a/src/Anubis/Data/BarStore.cs +++ b/src/Anubis/Data/BarStore.cs @@ -74,6 +74,16 @@ public sealed class BarStore(Database db, IClock clock) public Task CountDailyAsync(Guid instrumentId, CancellationToken cancellationToken) => db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM bars_1d WHERE instrument_id = @i", ("i", instrumentId)), cancellationToken); + /// + /// Multiplies the adjusted prices of the days up to by : a new + /// split or dividend changes the basis of every older adjusted price, and the history must stay on one basis. + /// + public Task RescaleAdjustedAsync(Guid instrumentId, DateOnly through, double factor, CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync( + "UPDATE bars_1d SET adj_open = adj_open * @f, adj_high = adj_high * @f, adj_low = adj_low * @f, adj_close = adj_close * @f, updated_at = @now " + + "WHERE instrument_id = @i AND day <= @d AND adj_close IS NOT NULL", cancellationToken, + ("f", factor), ("now", clock.UtcNow), ("i", instrumentId), ("d", DayText(through))), cancellationToken); + public Task SetDailyQualityAsync(Guid instrumentId, DateOnly day, string quality, CancellationToken cancellationToken) => db.WriteAsync(s => s.ExecuteAsync("UPDATE bars_1d SET quality = @q, updated_at = @now WHERE instrument_id = @i AND day = @d", cancellationToken, ("q", quality), ("now", clock.UtcNow), ("i", instrumentId), ("d", DayText(day))), cancellationToken); diff --git a/src/Anubis/Data/Collector.cs b/src/Anubis/Data/Collector.cs index 3830b73..5380e05 100644 --- a/src/Anubis/Data/Collector.cs +++ b/src/Anubis/Data/Collector.cs @@ -78,13 +78,14 @@ public sealed partial class MarketCollector /// /// Downloads the daily bars after the last one stored (from the first time) up to yesterday. /// , when given, are bars already downloaded for many symbols at once: only the second - /// readings and the gaps call the source again. + /// readings and the gaps call the source again. is the last day to collect: yesterday + /// (UTC) unless the caller knows a later closed session, as for stocks after the New York close. /// public async Task BackfillDailyAsync(Instrument instrument, DateOnly from, CancellationToken cancellationToken, - IReadOnlyList? prefetched = null) + IReadOnlyList? prefetched = null, DateOnly? lastDay = null) { ArgumentNullException.ThrowIfNull(instrument); - var yesterday = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime).AddDays(-1); + var yesterday = lastDay ?? DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime).AddDays(-1); var last = await _bars.LastDayAsync(instrument.Id, cancellationToken).ConfigureAwait(false); var first = last?.AddDays(1) ?? from; if (first > yesterday) @@ -92,19 +93,31 @@ public sealed partial class MarketCollector return CollectionResult.None; } - var fetched = prefetched is null - ? await _source.FetchDailyBarsAsync(instrument.Symbol, first, yesterday, cancellationToken).ConfigureAwait(false) - : prefetched.Where(b => b.Day >= first && b.Day <= yesterday).OrderBy(b => b.Day).ToList(); + // The download starts from the last day held, which works as a witness: if its adjusted prices changed, a split + // or dividend arrived meanwhile and the whole stored history is brought onto the new basis first. + var downloaded = prefetched is null + ? await _source.FetchDailyBarsAsync(instrument.Symbol, last ?? first, yesterday, cancellationToken).ConfigureAwait(false) + : prefetched; + if (last is { } witnessDay && downloaded.FirstOrDefault(b => b.Day == witnessDay) is { Adjusted: { } fresh } witness) + { + await RebaseAdjustedAsync(instrument, witnessDay, witness.Close, fresh.Close, cancellationToken).ConfigureAwait(false); + } + + var fetched = downloaded.Where(b => b.Day >= first && b.Day <= yesterday).OrderBy(b => b.Day).ToList(); if (fetched.Count == 0) { return CollectionResult.None; } var context = await _bars.DailyAsync(instrument.Id, first.AddDays(-Context * 2), first.AddDays(-1), cancellationToken).ConfigureAwait(false); - var all = context.Where(b => b.Day < fetched[0].Day).TakeLast(Context).Concat(fetched).ToList(); + var held = context.Where(b => b.Day < fetched[0].Day).TakeLast(Context).ToList(); + var all = held.Concat(fetched).ToList(); var newFrom = QualityChecks.Midnight(fetched[0].Day); + // A gap is new when it ends after the last bar already held: that includes the hole between the stored history + // and the first bar of this download. Gaps among the held bars were handled when those bars arrived. + var lastHeld = held.Count > 0 ? QualityChecks.Midnight(held[^1].Day) : DateTimeOffset.MinValue; var findings = QualityChecks.CheckDaily(all, _expectedDays(instrument, all[0].Day, all[^1].Day)) - .Where(f => f.StartsAt >= newFrom || (f.Kind == IssueKinds.Gap && f.EndsAt >= newFrom)).ToList(); + .Where(f => f.Kind == IssueKinds.Gap ? f.EndsAt > lastHeld : f.StartsAt >= newFrom).ToList(); var result = CollectionResult.None; var suspectDays = findings.Where(f => IssueKinds.MakesSuspect(f.Kind)).Select(f => DateOnly.FromDateTime(f.StartsAt.UtcDateTime)).ToHashSet(); @@ -133,6 +146,22 @@ public sealed partial class MarketCollector return result; } + private async Task RebaseAdjustedAsync(Instrument instrument, DateOnly day, double close, double adjustedClose, CancellationToken cancellationToken) + { + var held = await _bars.DailyAsync(instrument.Id, day, day, cancellationToken).ConfigureAwait(false); + if (held.Count == 0 || held[0].Adjusted is not { } old || old.Close <= 0 || held[0].Close <= 0 || close <= 0) + { + return; + } + + // Ratios of adjusted to real close, before and now; the real close itself does not change. + var factor = adjustedClose / close / (old.Close / held[0].Close); + if (double.IsFinite(factor) && factor > 0 && Math.Abs(factor - 1) > 1e-6) + { + await _bars.RescaleAdjustedAsync(instrument.Id, day, factor, cancellationToken).ConfigureAwait(false); + } + } + /// Downloads the minute bars after the last one stored, up to the last closed minute. public async Task BackfillMinutesAsync(Instrument instrument, CancellationToken cancellationToken) { @@ -234,9 +263,10 @@ public sealed partial class MarketCollector { break; } - catch (Exception ex) when (ex is MarketDataException or HttpRequestException or System.Net.WebSockets.WebSocketException or IOException - or System.Text.Json.JsonException or TimeoutException or TaskCanceledException) + catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested) { + // Any failure, the network's or the database's (a PostgreSQL restart, a busy SQLite file): the feed is + // marked down, then it waits and starts again. The loop never ends on its own. reason = Redactor.Redact(ex.Message); LogFeedError(_logger, Venue, reason); } @@ -275,11 +305,14 @@ public sealed partial class MarketCollector } var times = fresh.Select(b => b.OpensAt).ToHashSet(); - var all = context.Where(b => !times.Contains(b.OpensAt)).TakeLast(Context).Concat(fresh).ToList(); + var held = context.Where(b => !times.Contains(b.OpensAt)).OrderBy(b => b.OpensAt).TakeLast(Context).ToList(); + var all = held.Concat(fresh).ToList(); var newFrom = fresh.Min(b => b.OpensAt); var expected = _expectedMinutes(instrument, all.Min(b => b.OpensAt), all.Max(b => b.OpensAt).AddMinutes(1)); + // As for days: a gap ending after the last minute held is new, including a minute the live feed skipped. + var lastHeld = held.Count > 0 ? held[^1].OpensAt : DateTimeOffset.MinValue; var findings = QualityChecks.CheckMinutes(all, expected) - .Where(f => f.StartsAt >= newFrom || (f.Kind == IssueKinds.Gap && f.EndsAt >= newFrom)).ToList(); + .Where(f => f.Kind == IssueKinds.Gap ? f.EndsAt > lastHeld : f.StartsAt >= newFrom).ToList(); var suspect = findings.Where(f => IssueKinds.MakesSuspect(f.Kind)).Select(f => f.StartsAt).ToHashSet(); var impossible = findings.Where(f => f.Kind is IssueKinds.NonPositive or IssueKinds.HighBelowLow).Select(f => f.StartsAt).ToHashSet(); diff --git a/src/Anubis/Data/Freshness.cs b/src/Anubis/Data/Freshness.cs index 4789590..8b122e7 100644 --- a/src/Anubis/Data/Freshness.cs +++ b/src/Anubis/Data/Freshness.cs @@ -18,6 +18,9 @@ public sealed class FreshnessMonitor(IClock clock, EventBus bus, TimeSpan? maxAg private readonly ConcurrentDictionary _entries = new(); + // Touch runs on the feed, Check on a timer: updates go one at a time so neither overwrites a newer datum. + private readonly Lock _gate = new(); + public TimeSpan MaxAge { get; } = maxAge ?? DefaultMaxAge; public sealed record Status(Guid InstrumentId, string Symbol, DateTimeOffset? LastDataAt, bool Fresh, TimeSpan? Age); @@ -28,7 +31,13 @@ public sealed class FreshnessMonitor(IClock clock, EventBus bus, TimeSpan? maxAg public void Follow(Guid instrumentId, string symbol) => _entries.TryAdd(instrumentId, new Entry(symbol, clock.UtcNow, null, false)); - public void Unfollow(Guid instrumentId) => _entries.TryRemove(instrumentId, out _); + public void Unfollow(Guid instrumentId) + { + lock (_gate) + { + _entries.TryRemove(instrumentId, out _); + } + } public IReadOnlyCollection Followed => _entries.Keys.ToList(); @@ -37,15 +46,20 @@ public sealed class FreshnessMonitor(IClock clock, EventBus bus, TimeSpan? maxAg { var now = clock.UtcNow; var seen = dataAt > now ? now : dataAt; - if (!_entries.TryGetValue(instrumentId, out var entry)) + Entry entry, updated; + lock (_gate) { - return; + if (!_entries.TryGetValue(instrumentId, out entry!)) + { + return; + } + + updated = entry with { LastDataAt = entry.LastDataAt is { } last && last > seen ? last : seen }; + updated = updated with { Fresh = IsFresh(updated, now) }; + _entries[instrumentId] = updated; } - var updated = entry with { LastDataAt = entry.LastDataAt is { } last && last > seen ? last : seen }; - var fresh = IsFresh(updated, now); - _entries[instrumentId] = updated with { Fresh = fresh }; - if (fresh && !entry.Fresh) + if (updated.Fresh && !entry.Fresh) { bus.Publish(new FreshnessChanged(instrumentId, entry.Symbol, true, updated.LastDataAt, now)); } @@ -55,15 +69,24 @@ public sealed class FreshnessMonitor(IClock clock, EventBus bus, TimeSpan? maxAg public void Check() { var now = clock.UtcNow; - foreach (var (id, entry) in _entries) + var changes = new List(); + lock (_gate) { - var fresh = IsFresh(entry, now); - if (fresh != entry.Fresh) + foreach (var (id, entry) in _entries) { - _entries[id] = entry with { Fresh = fresh }; - bus.Publish(new FreshnessChanged(id, entry.Symbol, fresh, entry.LastDataAt, now)); + var fresh = IsFresh(entry, now); + if (fresh != entry.Fresh) + { + _entries[id] = entry with { Fresh = fresh }; + changes.Add(new FreshnessChanged(id, entry.Symbol, fresh, entry.LastDataAt, now)); + } } } + + foreach (var change in changes) + { + bus.Publish(change); + } } /// Whether new entries may use this instrument's live prices now. diff --git a/src/Anubis/Data/MarketDataService.cs b/src/Anubis/Data/MarketDataService.cs index f193abe..cace228 100644 --- a/src/Anubis/Data/MarketDataService.cs +++ b/src/Anubis/Data/MarketDataService.cs @@ -212,6 +212,20 @@ public sealed partial class MarketDataService : IDisposable { break; } + catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + // A failure outside the single steps (the database during a restart, a secret that cannot be read) ends + // this round only: the worker waits and is ready for the next wake-up. + LogStepFailed(_logger, ex, "history round"); + try + { + await _clock.WaitUntilAsync(_clock.UtcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + } } } @@ -296,10 +310,17 @@ public sealed partial class MarketDataService : IDisposable private async Task EquityHistoryAsync(AlpacaClient alpaca, CancellationToken cancellationToken) { - var collector = new MarketCollector(alpaca, _db, _clock, _bus, Freshness, logger: _logger); + // Trading days come from the broker's calendar; days it does not cover are not checked for gaps. + var collector = new MarketCollector(alpaca, _db, _clock, _bus, Freshness, + expectedDays: (_, from, to) => UsCalendar.Sessions(from, to) is { Count: > 0 } sessions ? sessions.Select(x => x.Date) : null, logger: _logger); var bars = new BarStore(_db, _clock); var eligible = (await _catalog.ListAsync(Venues.Alpaca, cancellationToken).ConfigureAwait(false)).Where(i => EquityUniverse.Eligible(i.Spec)).ToList(); - var yesterday = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime).AddDays(-1); + // The last New York session whose consolidated bar is final (16 minutes after the close), so the run after the + // close already collects that day. Without the calendar, yesterday. + var now = _clock.UtcNow; + var today = DateOnly.FromDateTime(now.UtcDateTime); + var closed = UsCalendar.Sessions(today.AddDays(-10), today.AddDays(1)).Where(s => s.CloseUtc + TimeSpan.FromMinutes(16) <= now).ToList(); + var yesterday = closed.Count > 0 ? closed[^1].Date : today.AddDays(-1); var total = CollectionResult.None; // Grouped by the first missing day, so one request serves up to a hundred symbols. @@ -307,6 +328,11 @@ public sealed partial class MarketDataService : IDisposable foreach (var instrument in eligible) { var last = await bars.LastDayAsync(instrument.Id, cancellationToken).ConfigureAwait(false); + if (!instrument.IsLive && last is not null) + { + continue; // Delisted with its history already here: nothing new will come, and each would cost requests of its own. + } + starts.Add((instrument, last?.AddDays(1) ?? HistoryFrom)); } @@ -314,12 +340,15 @@ public sealed partial class MarketDataService : IDisposable { foreach (var chunk in group.Chunk(100)) { - var fetched = await alpaca.FetchDailyBarsAsync(chunk.Select(c => c.Instrument.Symbol).ToList(), group.Key, yesterday, cancellationToken) + // From the day before the group's first missing day: for symbols with history that is the last day held, + // which the collector compares to find new splits and dividends. + var from = group.Key == HistoryFrom ? group.Key : group.Key.AddDays(-1); + var fetched = await alpaca.FetchDailyBarsAsync(chunk.Select(c => c.Instrument.Symbol).ToList(), from, yesterday, cancellationToken) .ConfigureAwait(false); foreach (var (instrument, first) in chunk) { var list = fetched.TryGetValue(instrument.Symbol, out var b) ? b : []; - total = total.Add(await collector.BackfillDailyAsync(instrument, first, cancellationToken, list).ConfigureAwait(false)); + total = total.Add(await collector.BackfillDailyAsync(instrument, first, cancellationToken, list, yesterday).ConfigureAwait(false)); } } } diff --git a/src/Anubis/Data/ReferenceData.cs b/src/Anubis/Data/ReferenceData.cs index 795ed57..477176d 100644 --- a/src/Anubis/Data/ReferenceData.cs +++ b/src/Anubis/Data/ReferenceData.cs @@ -161,7 +161,9 @@ public sealed class MoodStore(Database db, IClock clock) public async Task UpsertAsync(IEnumerable readings, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(readings); - var list = readings.ToList(); + // An empty field from the source reads as NaN: SQLite would store it as NULL and PostgreSQL as NaN, so it is + // left out on both engines. + var list = readings.Where(r => double.IsFinite(r.Value)).ToList(); var now = clock.UtcNow; await db.WriteAsync(async s => { diff --git a/tests/Anubis.Tests/CollectorTests.cs b/tests/Anubis.Tests/CollectorTests.cs index 7f752fb..c10a6ba 100644 --- a/tests/Anubis.Tests/CollectorTests.cs +++ b/tests/Anubis.Tests/CollectorTests.cs @@ -127,6 +127,91 @@ public sealed class CollectorTests Assert.AreEqual(1, next.Confirmed); Assert.AreEqual(BarQuality.Confirmed, (await bars.DailyAsync(btc.Id, day0.AddDays(120), day0.AddDays(120), CancellationToken.None).ConfigureAwait(false))[0].Quality); Assert.AreEqual(CollectionResult.None, await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false), "nothing new"); + + // The first day after the stored history is missing on the first read: the hole between what is held and + // what arrived is a gap like any other, read again and recovered. + var basis = source.Days[day0.AddDays(120)]; + foreach (var day in new[] { day0.AddDays(121), day0.AddDays(122) }) + { + source.Days[day] = basis with { Day = day, Open = basis.Close, High = basis.Close * 1.002, Low = basis.Close * 0.998, Quality = BarQuality.Ok }; + } + + source.MissingOnFirstRead.Add(day0.AddDays(121)); + clock.Advance(TimeSpan.FromDays(2)); + var after = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual((1, 1), (after.Gaps, after.Recovered), after.ToString()); + Assert.AreEqual(3, (await bars.DailyAsync(btc.Id, day0.AddDays(120), day0.AddDays(122), CancellationToken.None).ConfigureAwait(false)).Count); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task LiveMinuteSkippedByTheFeedIsDownloadedAgain() + { + var clock = new FastClock(new DateTimeOffset(2026, 11, 10, 10, 0, 30, TimeSpan.Zero)); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var bus = new EventBus(); + var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false); + var source = new FakeMarketSource(clock); + var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus)); + var bars = new BarStore(test.Db, clock); + var issues = new DataIssueStore(test.Db, clock); + + using var stop = new CancellationTokenSource(); + var live = collector.RunLiveAsync([btc], stop.Token); + await OkxTests.WaitForAsync(() => source.Connections == 1).ConfigureAwait(false); + Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 9, 59, 0, TimeSpan.Zero), await bars.LastMinuteAsync(btc.Id, CancellationToken.None).ConfigureAwait(false)); + + // The stream was subscribed too late for the 10:00 candle and sends 10:01 first. + var skipped = new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero); + clock.Advance(TimeSpan.FromMinutes(2)); + await source.Live!.Writer.WriteAsync(new BarEvent("BTC-EUR", FakeMarketSource.MinuteBar(skipped.AddMinutes(1)), true)).ConfigureAwait(false); + await OkxTests.WaitForAsync(() => + bars.MinuteAsync(btc.Id, skipped, skipped.AddMinutes(2), CancellationToken.None).GetAwaiter().GetResult().Count == 2).ConfigureAwait(false); + + var gap = (await issues.SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false)).Single(); + Assert.AreEqual((IssueKinds.Gap, IssueStatus.Recovered, skipped), (gap.Kind, gap.Status, gap.StartsAt)); + await stop.CancelAsync().ConfigureAwait(false); + await live.ConfigureAwait(false); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task ASplitAfterTheHistoryRebasesTheStoredAdjustedPrices() + { + var day0 = new DateOnly(2026, 6, 1); + var clock = new FastClock(new DateTimeOffset(day0.AddDays(100).ToDateTime(new TimeOnly(6, 0)), TimeSpan.Zero)); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var bus = new EventBus(); + var stock = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("ACME-EUR"), CancellationToken.None).ConfigureAwait(false); + static DailyBar Adjusted(DailyBar bar, double factor) => + bar with { Adjusted = new Ohlc(bar.Open * factor, bar.High * factor, bar.Low * factor, bar.Close * factor) }; + + var walk = MarketDataLogicTests.Walk(100, day0); + var source = new FakeMarketSource(clock); + foreach (var bar in walk) + { + source.Days[bar.Day] = Adjusted(bar, 1); + } + + var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus)); + Assert.AreEqual(100, (await collector.BackfillDailyAsync(stock, day0, CancellationToken.None).ConfigureAwait(false)).Stored); + + // A 2-for-1 split on day 100: the source now gives every older adjusted price halved, and the real price halves. + foreach (var bar in walk) + { + source.Days[bar.Day] = Adjusted(bar, 0.5); + } + + var close = walk[^1].Close / 2; + source.Days[day0.AddDays(100)] = Adjusted(new DailyBar(day0.AddDays(100), close, close * 1.003, close * 0.997, close * 1.001, 2000), 1); + clock.Advance(TimeSpan.FromDays(1)); + var result = await collector.BackfillDailyAsync(stock, day0, CancellationToken.None).ConfigureAwait(false); + + Assert.AreEqual((1, 0), (result.Stored, result.Suspect), "the split is not a jump"); + var stored = await new BarStore(test.Db, clock).DailyAsync(stock.Id, day0, day0.AddDays(100), CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(walk[50].Close, stored[50].Close, 1e-9, "real prices stay as they were"); + Assert.AreEqual(walk[50].Close / 2, stored[50].Adjusted!.Value.Close, 1e-9, "older adjusted prices are on the new basis"); + Assert.IsEmpty(await new DataIssueStore(test.Db, clock).SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false)); } } diff --git a/tests/Anubis.Tests/OkxTests.cs b/tests/Anubis.Tests/OkxTests.cs index 7a6e998..ae5210c 100644 --- a/tests/Anubis.Tests/OkxTests.cs +++ b/tests/Anubis.Tests/OkxTests.cs @@ -220,6 +220,52 @@ public sealed class OkxTests Assert.IsTrue(sockets.All(s => s.Disposed), "both connections are closed when the feed ends"); } + [TestMethod] + [TestCategory(Category.Integration)] + public async Task OneConnectionDroppingEndsTheFeedWithoutAnError() + { + var sockets = new List(); + var okx = new OkxClient(new FakeHttp().Client(), new ManualClock(_now), sockets: () => + { + var socket = new FakeSocket(); + lock (sockets) + { + sockets.Add(socket); + } + + return socket; + }); + + using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + var reading = Task.Run(async () => + { + await foreach (var item in okx.StreamAsync(["BTC-EUR"], stop.Token).ConfigureAwait(false)) + { + await Task.Delay(1).ConfigureAwait(false); // A slow reader: tickers pile up in the channel. + } + }); + + await WaitForAsync(() => sockets.Count == 2 && sockets.All(s => s.Sent.Count == 1)).ConfigureAwait(false); + var publicSocket = sockets.Single(s => s.Url!.AbsolutePath.EndsWith("/public", StringComparison.Ordinal)); + var businessSocket = sockets.Single(s => s.Url!.AbsolutePath.EndsWith("/business", StringComparison.Ordinal)); + const string ticker = """{"arg":{"channel":"tickers","instId":"BTC-EUR"},"data":[{"instType":"SPOT","instId":"BTC-EUR","last":"66148.9","askPx":"66134.7","askSz":"0.01847","bidPx":"66129.6","bidSz":"0.0193605","ts":"1794318514123"}]}"""; + for (var i = 0; i < 300; i++) + { + publicSocket.Push(ticker); + } + + // The candle connection drops while tickers keep arriving: the feed ends cleanly, so the collector marks it + // down and reconnects, instead of failing on a write to the closed feed. + businessSocket.Close(); + for (var i = 0; i < 300; i++) + { + publicSocket.Push(ticker); + } + + await reading.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); + Assert.IsTrue(sockets.All(s => s.Disposed)); + } + [TestMethod] [TestCategory(Category.Unit)] public void StreamErrorsStopTheFeed() @@ -232,7 +278,8 @@ public sealed class OkxTests internal static async Task WaitForAsync(Func condition) { - var deadline = DateTime.UtcNow.AddSeconds(10); + // Generous: on a busy machine (CI, PostgreSQL in Docker) the first backfill alone can take several seconds. + var deadline = DateTime.UtcNow.AddSeconds(30); while (!condition()) { if (DateTime.UtcNow > deadline) diff --git a/tests/Anubis.Tests/RealHistoryProof.cs b/tests/Anubis.Tests/RealHistoryProof.cs index 7a109c6..ad98a97 100644 --- a/tests/Anubis.Tests/RealHistoryProof.cs +++ b/tests/Anubis.Tests/RealHistoryProof.cs @@ -6,9 +6,9 @@ namespace Anubis.Tests; /// /// The Phase 2 proof on the real public sources (no keys): the whole daily history of the main OKX EUR pairs and the -/// Binance BTCUSDT archives since 2017, then the quality report. Needs the network, so it never runs in the normal -/// suite: dotnet test --filter FullyQualifiedName~RealHistoryProof. With ANUBIS_PROOF_OUT set, the report is -/// written there. +/// Binance BTCUSDT archives since 2017, then the quality report. Needs the network, so it runs only when asked: +/// ANUBIS_LIVE_PROOF=1 and dotnet test --filter FullyQualifiedName~RealHistoryProof. With ANUBIS_PROOF_OUT set, +/// the report is written there. /// [TestClass] public sealed class RealHistoryProof @@ -17,6 +17,11 @@ public sealed class RealHistoryProof [TestCategory(Category.LiveDemo)] public async Task DownloadsTheRealDailyHistoryAndReportsItsQuality() { + if (Environment.GetEnvironmentVariable("ANUBIS_LIVE_PROOF") != "1") + { + Assert.Inconclusive("Needs the network: set ANUBIS_LIVE_PROOF=1 to run it."); + } + var clock = new SystemClock(); await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(60) };