diff --git a/src/Anubis/Data/BarStore.cs b/src/Anubis/Data/BarStore.cs index 6cf8666..dcc1a24 100644 --- a/src/Anubis/Data/BarStore.cs +++ b/src/Anubis/Data/BarStore.cs @@ -117,6 +117,10 @@ public sealed class BarStore(Database db, IClock clock) r => new Bar(r.Moment("opens_at"), r.Measure("open"), r.Measure("high"), r.Measure("low"), r.Measure("close"), r.Measure("volume")), ("i", instrumentId), ("f", from), ("t", to)), cancellationToken).ConfigureAwait(false); + public Task DeleteMinuteAsync(Guid instrumentId, DateTimeOffset opensAt, CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync("DELETE FROM bars_1m WHERE instrument_id = @i AND opens_at = @t", cancellationToken, ("i", instrumentId), ("t", opensAt)), + cancellationToken); + public Task LastMinuteAsync(Guid instrumentId, CancellationToken cancellationToken) => db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT opens_at FROM bars_1m WHERE instrument_id = @i ORDER BY opens_at DESC", r => (DateTimeOffset?)r.Moment("opens_at"), ("i", instrumentId)), cancellationToken); diff --git a/src/Anubis/Data/Collector.cs b/src/Anubis/Data/Collector.cs new file mode 100644 index 0000000..49b3779 --- /dev/null +++ b/src/Anubis/Data/Collector.cs @@ -0,0 +1,406 @@ +using Anubis.Core; +using Anubis.Db; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Anubis.Data; + +/// What a collection pass did, for the log and the quality report. +public sealed record CollectionResult(int Stored, int Suspect, int Confirmed, int Corrected, int Gaps, int Recovered, int Unrecoverable) +{ + public static readonly CollectionResult None = new(0, 0, 0, 0, 0, 0, 0); + + public CollectionResult Add(CollectionResult other) => new(Stored + other.Stored, Suspect + other.Suspect, + Confirmed + other.Confirmed, Corrected + other.Corrected, Gaps + other.Gaps, Recovered + other.Recovered, Unrecoverable + other.Unrecoverable); +} + +/// The live feed of a venue connected, dropped or came back (SPEC 6.2). +public sealed record FeedChanged(string Venue, bool Connected, DateTimeOffset At, string? Reason); + +/// +/// The collector of SPEC 6.2–6.4 for one venue: downloads what is missing at startup and after every drop of the +/// live feed, checks every bar, and never corrects in silence. A suspect bar is stored as suspect, blocks new +/// entries on its instrument (see ) and is read a second +/// time: the same values confirm it, different ones replace it, and an impossible bar is dropped. Gaps are +/// downloaded again once; what the source still lacks is recorded as unrecoverable. +/// +public sealed partial class MarketCollector +{ + /// History of minute bars downloaded the first time an instrument is followed. + public static readonly TimeSpan InitialMinutes = TimeSpan.FromDays(2); + + /// Waits between reconnection attempts: 1, 2, 5, 10, 30 seconds, then 60. + public static readonly IReadOnlyList Backoff = + [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60)]; + + /// Closes used as context for the jump check of new bars. + private const int Context = QualityChecks.JumpWindow + 1; + + private readonly IMarketDataSource _source; + private readonly IClock _clock; + private readonly EventBus _bus; + private readonly FreshnessMonitor _freshness; + private readonly Func?> _expectedMinutes; + private readonly Func?> _expectedDays; + private readonly ILogger _logger; + private readonly BarStore _bars; + private readonly DataIssueStore _issues; + + /// Minutes a bar should exist for; by default every minute for crypto and none (no gap check) for stocks. + /// Days a daily bar should exist for; by default every day for crypto and none for stocks. + public MarketCollector( + IMarketDataSource source, + Database db, + IClock clock, + EventBus bus, + FreshnessMonitor freshness, + Func?>? expectedMinutes = null, + Func?>? expectedDays = null, + ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(source); + _source = source; + _clock = clock; + _bus = bus; + _freshness = freshness; + _expectedMinutes = expectedMinutes ?? ((i, from, to) => i.IsCrypto ? QualityChecks.EveryMinute(from, to) : null); + _expectedDays = expectedDays ?? ((i, from, to) => i.IsCrypto ? QualityChecks.EveryDay(from, to) : null); + _logger = logger ?? NullLogger.Instance; + _bars = new BarStore(db, clock); + _issues = new DataIssueStore(db, clock); + } + + public string Venue => _source.Venue; + + /// True while the live feed is connected. + public bool Connected { get; private set; } + + /// Downloads the daily bars after the last one stored (from the first time) up to yesterday. + public async Task BackfillDailyAsync(Instrument instrument, DateOnly from, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(instrument); + var yesterday = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime).AddDays(-1); + var last = await _bars.LastDayAsync(instrument.Id, cancellationToken).ConfigureAwait(false); + var first = last?.AddDays(1) ?? from; + if (first > yesterday) + { + return CollectionResult.None; + } + + var fetched = await _source.FetchDailyBarsAsync(instrument.Symbol, first, yesterday, cancellationToken).ConfigureAwait(false); + if (fetched.Count == 0) + { + return CollectionResult.None; + } + + var context = await _bars.DailyAsync(instrument.Id, first.AddDays(-Context * 2), first.AddDays(-1), cancellationToken).ConfigureAwait(false); + var all = context.Where(b => b.Day < fetched[0].Day).TakeLast(Context).Concat(fetched).ToList(); + var newFrom = QualityChecks.Midnight(fetched[0].Day); + var findings = QualityChecks.CheckDaily(all, _expectedDays(instrument, all[0].Day, all[^1].Day)) + .Where(f => f.StartsAt >= newFrom || (f.Kind == IssueKinds.Gap && f.EndsAt >= newFrom)).ToList(); + + var result = CollectionResult.None; + var suspectDays = findings.Where(f => IssueKinds.MakesSuspect(f.Kind)).Select(f => DateOnly.FromDateTime(f.StartsAt.UtcDateTime)).ToHashSet(); + var impossible = findings.Where(f => f.Kind is IssueKinds.NonPositive or IssueKinds.HighBelowLow) + .Select(f => DateOnly.FromDateTime(f.StartsAt.UtcDateTime)).ToHashSet(); + var stored = fetched.DistinctBy(b => b.Day) + .Where(b => !impossible.Contains(b.Day)) + .Select(b => suspectDays.Contains(b.Day) ? b with { Quality = BarQuality.Suspect } : b) + .ToList(); + await _bars.UpsertDailyAsync(instrument.Id, stored, Venue, cancellationToken).ConfigureAwait(false); + result = result with { Stored = stored.Count }; + + foreach (var finding in findings) + { + var id = await _issues.RecordAsync(instrument.Id, Timeframes.Day, finding, cancellationToken).ConfigureAwait(false); + if (IssueKinds.MakesSuspect(finding.Kind)) + { + result = result.Add(await SecondDailyReadingAsync(instrument, id, finding, cancellationToken).ConfigureAwait(false)); + } + else if (finding.Kind == IssueKinds.Gap) + { + result = result.Add(await RecoverDailyGapAsync(instrument, id, finding, cancellationToken).ConfigureAwait(false)); + } + } + + return result; + } + + /// Downloads the minute bars after the last one stored, up to the last closed minute. + public async Task BackfillMinutesAsync(Instrument instrument, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(instrument); + var until = Minute(_clock.UtcNow); + var last = await _bars.LastMinuteAsync(instrument.Id, cancellationToken).ConfigureAwait(false); + var start = last?.AddMinutes(1) ?? until - InitialMinutes; + if (start >= until) + { + return CollectionResult.None; + } + + var fetched = await _source.FetchMinuteBarsAsync(instrument.Symbol, start, until, cancellationToken).ConfigureAwait(false); + var context = last is { } l + ? await _bars.MinuteAsync(instrument.Id, l.AddMinutes(-Context), l.AddMinutes(1), cancellationToken).ConfigureAwait(false) + : []; + // Minutes the source has not published yet are not gaps: the next pass starts again from the last bar held, + // and a minute still missing then lies between two bars and is found by the check. + return await AcceptMinutesAsync(instrument, context, fetched, cancellationToken).ConfigureAwait(false); + } + + /// + /// Follows on the live feed until cancelled: every datum refreshes the freshness of + /// its instrument, every closed bar is checked and stored, and every drop is followed by a reconnection and by the + /// download of what was missed meanwhile. + /// + public async Task RunLiveAsync(IReadOnlyList instruments, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(instruments); + var bySymbol = instruments.ToDictionary(i => i.Symbol, StringComparer.Ordinal); + foreach (var instrument in instruments) + { + _freshness.Follow(instrument.Id, instrument.Symbol); + } + + var recent = new Dictionary>(); + var failures = 0; + while (!cancellationToken.IsCancellationRequested) + { + string? reason = null; + try + { + foreach (var instrument in instruments) + { + var recovered = await BackfillMinutesAsync(instrument, cancellationToken).ConfigureAwait(false); + if (recovered.Stored > 0) + { + LogRecovered(_logger, Venue, instrument.Symbol, recovered.Stored); + } + + var last = await _bars.LastMinuteAsync(instrument.Id, cancellationToken).ConfigureAwait(false); + recent[instrument.Id] = last is { } l + ? (await _bars.MinuteAsync(instrument.Id, l.AddMinutes(-Context), l.AddMinutes(1), cancellationToken).ConfigureAwait(false)).ToList() + : []; + } + + await foreach (var item in _source.StreamAsync(bySymbol.Keys, cancellationToken).ConfigureAwait(false)) + { + if (!Connected) + { + Connected = true; + failures = 0; + _bus.Publish(new FeedChanged(Venue, true, _clock.UtcNow, null)); + } + + if (!bySymbol.TryGetValue(item.Symbol, out var instrument)) + { + continue; + } + + if (item is BarEvent { Closed: true } closed) + { + var context = recent[instrument.Id]; + await AcceptMinutesAsync(instrument, context, [closed.Bar], cancellationToken).ConfigureAwait(false); + if (Possible(closed.Bar)) + { + context.RemoveAll(b => b.OpensAt == closed.Bar.OpensAt); + context.Add(closed.Bar); + } + + if (context.Count > Context * 2) + { + context.RemoveRange(0, context.Count - Context); + } + + _freshness.Touch(instrument.Id, closed.Bar.OpensAt.AddMinutes(1)); + } + else if (item is not BarEvent) + { + _freshness.Touch(instrument.Id, item.At); + } + + _bus.Publish(item); + } + + reason = "il flusso si è chiuso"; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) when (ex is MarketDataException or HttpRequestException or System.Net.WebSockets.WebSocketException or IOException + or System.Text.Json.JsonException or TimeoutException or TaskCanceledException) + { + reason = Redactor.Redact(ex.Message); + LogFeedError(_logger, Venue, reason); + } + + if (Connected) + { + Connected = false; + _bus.Publish(new FeedChanged(Venue, false, _clock.UtcNow, reason)); + } + + var wait = Backoff[Math.Min(failures, Backoff.Count - 1)]; + failures++; + try + { + await _clock.WaitUntilAsync(_clock.UtcNow + wait, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + } + + if (Connected) + { + Connected = false; + _bus.Publish(new FeedChanged(Venue, false, _clock.UtcNow, "arresto")); + } + } + + /// Checks new minute bars against their context, stores them, and handles what the checks found. + private async Task AcceptMinutesAsync(Instrument instrument, IReadOnlyList context, IReadOnlyList fresh, CancellationToken cancellationToken) + { + if (fresh.Count == 0) + { + return CollectionResult.None; + } + + var times = fresh.Select(b => b.OpensAt).ToHashSet(); + var all = context.Where(b => !times.Contains(b.OpensAt)).TakeLast(Context).Concat(fresh).ToList(); + var newFrom = fresh.Min(b => b.OpensAt); + var expected = _expectedMinutes(instrument, all.Min(b => b.OpensAt), all.Max(b => b.OpensAt).AddMinutes(1)); + var findings = QualityChecks.CheckMinutes(all, expected) + .Where(f => f.StartsAt >= newFrom || (f.Kind == IssueKinds.Gap && f.EndsAt >= newFrom)).ToList(); + var suspect = findings.Where(f => IssueKinds.MakesSuspect(f.Kind)).Select(f => f.StartsAt).ToHashSet(); + var impossible = findings.Where(f => f.Kind is IssueKinds.NonPositive or IssueKinds.HighBelowLow).Select(f => f.StartsAt).ToHashSet(); + + var good = fresh.DistinctBy(b => b.OpensAt).Where(b => !suspect.Contains(b.OpensAt)).ToList(); + var doubtful = fresh.DistinctBy(b => b.OpensAt).Where(b => suspect.Contains(b.OpensAt) && !impossible.Contains(b.OpensAt)).ToList(); + await _bars.UpsertMinuteAsync(instrument.Id, good, Venue, BarQuality.Ok, cancellationToken).ConfigureAwait(false); + await _bars.UpsertMinuteAsync(instrument.Id, doubtful, Venue, BarQuality.Suspect, cancellationToken).ConfigureAwait(false); + var result = CollectionResult.None with { Stored = good.Count + doubtful.Count }; + + foreach (var finding in findings) + { + if (IssueKinds.MakesSuspect(finding.Kind)) + { + var id = await _issues.RecordAsync(instrument.Id, Timeframes.Minute, finding, cancellationToken).ConfigureAwait(false); + result = result.Add(await SecondMinuteReadingAsync(instrument, id, finding, cancellationToken).ConfigureAwait(false)); + } + else if (finding.Kind == IssueKinds.Gap) + { + result = result.Add(await RecordMinuteGapAsync(instrument, finding, cancellationToken).ConfigureAwait(false)); + } + } + + return result; + } + + private async Task RecordMinuteGapAsync(Instrument instrument, QualityFinding gap, CancellationToken cancellationToken) + { + var id = await _issues.RecordAsync(instrument.Id, Timeframes.Minute, gap, cancellationToken).ConfigureAwait(false); + await _issues.AttemptAsync(id, cancellationToken).ConfigureAwait(false); + var bars = await _source.FetchMinuteBarsAsync(instrument.Symbol, gap.StartsAt, gap.EndsAt.AddMinutes(1), cancellationToken).ConfigureAwait(false); + var found = bars.Where(b => b.OpensAt >= gap.StartsAt && b.OpensAt <= gap.EndsAt && Possible(b)).ToList(); + await _bars.UpsertMinuteAsync(instrument.Id, found, Venue, BarQuality.Ok, cancellationToken).ConfigureAwait(false); + var missing = (int)((gap.EndsAt - gap.StartsAt).TotalMinutes + 1) - found.Count; + if (missing <= 0) + { + await _issues.ResolveAsync(id, IssueStatus.Recovered, $"{gap.Detail}: recuperati {found.Count}", cancellationToken).ConfigureAwait(false); + return CollectionResult.None with { Gaps = 1, Recovered = 1, Stored = found.Count }; + } + + await _issues.ResolveAsync(id, IssueStatus.Unrecoverable, + $"{gap.Detail}: la fonte non ha {missing} barre" + (found.Count > 0 ? $" (recuperate {found.Count})" : "") + ", forse minuti senza scambi", + cancellationToken).ConfigureAwait(false); + return CollectionResult.None with { Gaps = 1, Unrecoverable = 1, Stored = found.Count }; + } + + private async Task SecondMinuteReadingAsync(Instrument instrument, Guid issue, QualityFinding finding, CancellationToken cancellationToken) + { + await _issues.AttemptAsync(issue, cancellationToken).ConfigureAwait(false); + var at = finding.StartsAt; + var stored = FirstOf(await _bars.MinuteAsync(instrument.Id, at, at.AddMinutes(1), cancellationToken).ConfigureAwait(false)); + var again = (await _source.FetchMinuteBarsAsync(instrument.Symbol, at, at.AddMinutes(1), cancellationToken).ConfigureAwait(false)) + .FirstOrDefault(b => b.OpensAt == at); + return await ConcludeAsync(issue, finding, stored is null ? null : Prices(stored), again is null ? null : Prices(again), + q => _bars.UpsertMinuteAsync(instrument.Id, [again!], Venue, q, cancellationToken), + () => _bars.DeleteMinuteAsync(instrument.Id, at, cancellationToken), cancellationToken).ConfigureAwait(false); + } + + private async Task SecondDailyReadingAsync(Instrument instrument, Guid issue, QualityFinding finding, CancellationToken cancellationToken) + { + await _issues.AttemptAsync(issue, cancellationToken).ConfigureAwait(false); + var day = DateOnly.FromDateTime(finding.StartsAt.UtcDateTime); + var stored = FirstOf(await _bars.DailyAsync(instrument.Id, day, day, cancellationToken).ConfigureAwait(false)); + var again = (await _source.FetchDailyBarsAsync(instrument.Symbol, day, day, cancellationToken).ConfigureAwait(false)).FirstOrDefault(b => b.Day == day); + return await ConcludeAsync(issue, finding, stored?.Prices, again?.Prices, + q => _bars.UpsertDailyAsync(instrument.Id, [again! with { Quality = q }], Venue, cancellationToken), + () => _bars.DeleteDailyAsync(instrument.Id, day, cancellationToken), cancellationToken).ConfigureAwait(false); + } + + /// + /// The outcome of a second reading: impossible or absent again → unrecoverable and the bar is dropped; the same + /// prices → confirmed; different prices → the new ones replace the old (corrected). + /// + private async Task ConcludeAsync(Guid issue, QualityFinding finding, Ohlc? stored, Ohlc? again, + Func> save, Func drop, CancellationToken cancellationToken) + { + if (again is not { } second || !Possible(second)) + { + await drop().ConfigureAwait(false); + await _issues.ResolveAsync(issue, IssueStatus.Unrecoverable, finding.Detail + ": la seconda lettura non dà un dato valido, barra scartata", + cancellationToken).ConfigureAwait(false); + return CollectionResult.None with { Suspect = 1, Unrecoverable = 1 }; + } + + if (stored is { } first && Same(first, second)) + { + await save(BarQuality.Confirmed).ConfigureAwait(false); + await _issues.ResolveAsync(issue, IssueStatus.Confirmed, finding.Detail + ": confermato dalla seconda lettura", cancellationToken).ConfigureAwait(false); + return CollectionResult.None with { Suspect = 1, Confirmed = 1 }; + } + + await save(BarQuality.Ok).ConfigureAwait(false); + await _issues.ResolveAsync(issue, IssueStatus.Corrected, + finding.Detail + (stored is { } old ? $": la seconda lettura dà chiusura {second.Close:G6} invece di {old.Close:G6}" : ": la seconda lettura dà una barra valida"), + cancellationToken).ConfigureAwait(false); + return CollectionResult.None with { Suspect = 1, Corrected = 1 }; + } + + private async Task RecoverDailyGapAsync(Instrument instrument, Guid issue, QualityFinding gap, CancellationToken cancellationToken) + { + await _issues.AttemptAsync(issue, cancellationToken).ConfigureAwait(false); + var first = DateOnly.FromDateTime(gap.StartsAt.UtcDateTime); + var last = DateOnly.FromDateTime(gap.EndsAt.UtcDateTime); + var found = (await _source.FetchDailyBarsAsync(instrument.Symbol, first, last, cancellationToken).ConfigureAwait(false)) + .Where(b => b.Day >= first && b.Day <= last && Possible(b.Prices)).ToList(); + await _bars.UpsertDailyAsync(instrument.Id, found, Venue, cancellationToken).ConfigureAwait(false); + var missing = last.DayNumber - first.DayNumber + 1 - found.Count; + await _issues.ResolveAsync(issue, missing <= 0 ? IssueStatus.Recovered : IssueStatus.Unrecoverable, + missing <= 0 ? gap.Detail + ": recuperato" : $"{gap.Detail}: la fonte non ha {missing} giorni", cancellationToken).ConfigureAwait(false); + return CollectionResult.None with { Gaps = 1, Recovered = missing <= 0 ? 1 : 0, Unrecoverable = missing <= 0 ? 0 : 1, Stored = found.Count }; + } + + private static T? FirstOf(IReadOnlyList list) where T : class => list.Count > 0 ? list[0] : null; + + private static Ohlc Prices(Bar b) => new(b.Open, b.High, b.Low, b.Close); + + private static bool Possible(Bar b) => Possible(Prices(b)); + + private static bool Possible(Ohlc p) => p.Open > 0 && p.High > 0 && p.Low > 0 && p.Close > 0 && p.High >= p.Low; + + private static bool Same(Ohlc a, Ohlc b) => Close(a.Open, b.Open) && Close(a.High, b.High) && Close(a.Low, b.Low) && Close(a.Close, b.Close); + + private static bool Close(double a, double b) => Math.Abs(a - b) <= 1e-9 * Math.Max(Math.Abs(a), Math.Abs(b)); + + private static DateTimeOffset Minute(DateTimeOffset t) => new(t.UtcTicks - t.UtcTicks % TimeSpan.TicksPerMinute, TimeSpan.Zero); + + [LoggerMessage(Level = LogLevel.Information, Message = "{Venue} {Symbol}: recovered {Count} minute bars")] + private static partial void LogRecovered(ILogger logger, string venue, string symbol, int count); + + [LoggerMessage(Level = LogLevel.Warning, Message = "{Venue} live feed dropped: {Reason}")] + private static partial void LogFeedError(ILogger logger, string venue, string reason); +} diff --git a/src/Anubis/Data/FreeSources.cs b/src/Anubis/Data/FreeSources.cs index 70d340e..b225f57 100644 --- a/src/Anubis/Data/FreeSources.cs +++ b/src/Anubis/Data/FreeSources.cs @@ -354,6 +354,21 @@ public static partial class NasdaqTraded [GeneratedRegex(@"\b(ETF|ETN|ETNs|Exchange[- ]Traded|Fund|Funds|Trust Units?|Warrants?|Rights?|Units?|Preferred|Notes?|Debentures?|Depositary Shares? representing a fraction|Closed[- ]End|Income Shares|Beneficial Interest|Index|Leveraged|Inverse|SPAC)\b", RegexOptions.IgnoreCase)] internal static partial Regex NotCommonPattern(); + /// + /// Marks as funds the broker's instruments that this list (or their own name) shows are not common shares. Symbols + /// the list does not know keep only the name check. + /// + public static IReadOnlyList MarkFunds(IEnumerable specs, IReadOnlyList listed) + { + ArgumentNullException.ThrowIfNull(specs); + ArgumentNullException.ThrowIfNull(listed); + var bySymbol = listed.GroupBy(l => l.Symbol, StringComparer.Ordinal).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + return specs.Select(s => s with + { + IsFund = s.IsFund == true || (bySymbol.TryGetValue(s.Symbol, out var l) && l.IsExcluded) || NotCommonPattern().IsMatch(s.Name), + }).ToList(); + } + /// Rows of the pipe-separated file (CRLF), header and footer skipped. public static IReadOnlyList Parse(string text) { diff --git a/src/Anubis/Data/Universes.cs b/src/Anubis/Data/Universes.cs new file mode 100644 index 0000000..464ca48 --- /dev/null +++ b/src/Anubis/Data/Universes.cs @@ -0,0 +1,293 @@ +using System.Text.RegularExpressions; +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Data; + +/// A member of a universe on a date, with its rank and the measure it was ranked by. +public sealed record UniverseMember(Guid InstrumentId, string Symbol, int Rank, double Measure); + +public static class Universes +{ + public const string Equity = "equity_us"; + public const string Crypto = "crypto_eur"; +} + +/// An instrument with the daily bars known for it; bars after the date are ignored, never used. +public sealed record UniverseCandidate(Instrument Instrument, IReadOnlyList Bars) +{ + /// Hourly spread samples of the last 30 days (crypto), in basis points. + public IReadOnlyList<(DateTimeOffset At, double Bp)> Spreads { get; init; } = []; + + /// The spread right now (crypto), used until 30 days of samples exist. + public double? CurrentSpreadBp { get; init; } +} + +/// +/// The stock universe "as of a date" (SPEC 6.7, 8.2): common shares on NYSE and Nasdaq, fractionable on Alpaca, +/// raw close above 10 $, at least 300 sessions, the 150 with the highest median traded value over 60 sessions. +/// ETFs, ETNs and funds are out. Only bars up to the date count; instruments delisted later are included if they +/// traded then. The flags the API gives only as of today (fractionable, fund) apply as they are (API-NOTES). +/// +public static class EquityUniverse +{ + public const int Size = 150; + public const double MinPrice = 10; + public const int MinSessions = 300; + public const int Window = 60; + + public static readonly IReadOnlySet Exchanges = new HashSet(StringComparer.Ordinal) { "NYSE", "NASDAQ" }; + + public static bool Eligible(InstrumentSpec spec) + { + ArgumentNullException.ThrowIfNull(spec); + return spec.AssetClass == AssetClasses.Equity && spec.Exchange is { } exchange && Exchanges.Contains(exchange) && spec.Fractionable == true && spec.IsFund != true; + } + + public static IReadOnlyList Select(DateOnly asOf, IEnumerable candidates, int size = Size) + { + ArgumentNullException.ThrowIfNull(candidates); + var ranked = new List<(Instrument Instrument, double Value)>(); + foreach (var candidate in candidates) + { + if (!Eligible(candidate.Instrument.Spec)) + { + continue; + } + + var bars = UpTo(candidate.Bars, asOf); + if (bars.Count < MinSessions || bars[^1].Day < asOf.AddDays(-7) || bars[^1].Close <= MinPrice) + { + continue; // Too young, not trading any more on that date, or too cheap. + } + + ranked.Add((candidate.Instrument, Median(bars.TakeLast(Window).Select(b => b.Close * b.Volume)))); + } + + return Rank(ranked, size); + } + + internal static List UpTo(IReadOnlyList bars, DateOnly asOf) => bars.Where(b => b.Day <= asOf).OrderBy(b => b.Day).ToList(); + + internal static double Median(IEnumerable values) + { + var sorted = values.Order().ToList(); + if (sorted.Count == 0) + { + return 0; + } + + var middle = sorted.Count / 2; + return sorted.Count % 2 == 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; + } + + internal static IReadOnlyList Rank(List<(Instrument Instrument, double Value)> ranked, int size) => + ranked.OrderByDescending(r => r.Value).ThenBy(r => r.Instrument.Symbol, StringComparer.Ordinal).Take(size) + .Select((r, i) => new UniverseMember(r.Instrument.Id, r.Instrument.Symbol, i + 1, r.Value)).ToList(); +} + +/// +/// The crypto universe of SPEC 8.1: EUR spot pairs listed for at least 400 days, 30-day median traded value above +/// 250,000 €, 30-day median spread below 20 basis points (the current spread until 30 days of samples exist), no +/// stablecoins, wrapped or leveraged tokens; the 10 with the highest value. Recomputed on the first day of each month. +/// +public static partial class CryptoUniverse +{ + public const int Size = 10; + public const int MinAgeDays = 400; + public const double MinValueEur = 250_000; + public const double MaxSpreadBp = 20; + public const int Window = 30; + + /// Stablecoins pegged to a currency or to gold, as listed by the main venues. + public static readonly IReadOnlySet Stablecoins = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "USDT", "USDC", "DAI", "EURC", "EUROC", "EURT", "EURS", "EURI", "EURQ", "EURR", "TUSD", "FDUSD", "PYUSD", "USDE", "USDS", "BUSD", "USDP", "GUSD", + "FRAX", "LUSD", "USDD", "USD1", "RLUSD", "USDG", "USDQ", "USDR", "AEUR", "XAUT", "PAXG", + }; + + /// Wrapped or staked receipts of another coin. + public static readonly IReadOnlySet Wrapped = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "WBTC", "WETH", "WBETH", "STETH", "WSTETH", "CBBTC", "CBETH", "RETH", "WEETH", "EETH", "TBTC", "WBNB", "WSOL", "JITOSOL", "MSOL", "BBSOL", + "STSOL", "WTRX", "WAVAX", "WMATIC", "WPOL", "WFTM", "WHBAR", "SAVAX", "OKSOL", "BETH", "LBTC", + }; + + [GeneratedRegex(@"^\w+?(\d+[LS]|UP|DOWN|BULL|BEAR)$", RegexOptions.IgnoreCase)] + private static partial Regex LeveragedPattern(); + + public static bool Excluded(string baseCurrency) => + Stablecoins.Contains(baseCurrency) || Wrapped.Contains(baseCurrency) || LeveragedPattern().IsMatch(baseCurrency); + + public static IReadOnlyList Select(DateOnly asOf, IEnumerable candidates, int size = Size) + { + ArgumentNullException.ThrowIfNull(candidates); + var ranked = new List<(Instrument Instrument, double Value)>(); + var cutoff = QualityChecks.Midnight(asOf.AddDays(1)); + foreach (var candidate in candidates) + { + var spec = candidate.Instrument.Spec; + var baseCurrency = spec.BaseCurrency ?? spec.Symbol.Split('-')[0]; + if (spec.AssetClass != AssetClasses.Crypto || spec.QuoteCurrency != "EUR" || Excluded(baseCurrency)) + { + continue; + } + + var bars = EquityUniverse.UpTo(candidate.Bars, asOf); + var listedOn = spec.ListedAt is { } listed ? DateOnly.FromDateTime(listed.UtcDateTime) : bars.Count > 0 ? bars[0].Day : asOf; + if (bars.Count < Window || listedOn > asOf.AddDays(-MinAgeDays) || bars[^1].Day < asOf.AddDays(-3)) + { + continue; + } + + var value = EquityUniverse.Median(bars.TakeLast(Window).Select(b => b.QuoteVolume ?? b.Close * b.Volume)); + var samples = candidate.Spreads.Where(s => s.At < cutoff && s.At >= cutoff.AddDays(-Window)).ToList(); + var spread = samples.Count > 0 && samples.Min(s => s.At) <= cutoff.AddDays(-Window + 1) + ? EquityUniverse.Median(samples.Select(s => s.Bp)) + : candidate.CurrentSpreadBp; + if (value <= MinValueEur || spread is not { } bp || bp >= MaxSpreadBp) + { + continue; // An unknown spread counts as too wide. + } + + ranked.Add((candidate.Instrument, value)); + } + + return EquityUniverse.Rank(ranked, size); + } +} + +/// Universes as they were chosen, and the hourly spread samples the crypto one needs. +public sealed class UniverseStore(Database db, IClock clock) +{ + public async Task SaveAsync(string universe, DateOnly asOf, IReadOnlyList members, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(members); + var now = clock.UtcNow; + await db.WriteAsync(async s => + { + await s.ExecuteAsync("DELETE FROM universe_members WHERE universe = @u AND as_of = @d", cancellationToken, ("u", universe), ("d", BarStore.DayText(asOf))) + .ConfigureAwait(false); + foreach (var m in members) + { + await s.ExecuteAsync( + "INSERT INTO universe_members (id, universe, as_of, instrument_id, rank, created_at) VALUES (@id, @u, @d, @i, @r, @now)", cancellationToken, + ("id", Ids.NewId(clock)), ("u", universe), ("d", BarStore.DayText(asOf)), ("i", m.InstrumentId), ("r", m.Rank), ("now", now)).ConfigureAwait(false); + } + }, cancellationToken).ConfigureAwait(false); + } + + /// The members of the last computation on or before , by rank. + public async Task> AsOfAsync(string universe, DateOnly asOf, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync( + "SELECT m.instrument_id, i.symbol, m.rank FROM universe_members m JOIN instruments i ON i.id = m.instrument_id " + + "WHERE m.universe = @u AND m.as_of = (SELECT MAX(as_of) FROM universe_members WHERE universe = @u AND as_of <= @d) ORDER BY m.rank", + r => new UniverseMember(r.Id("instrument_id"), r.Text("symbol"), (int)r.Number("rank"), 0), ("u", universe), ("d", BarStore.DayText(asOf))), + cancellationToken).ConfigureAwait(false); + + public Task AddSpreadAsync(Guid instrumentId, DateTimeOffset at, double bp, CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync( + "INSERT INTO spread_samples (id, instrument_id, sampled_at, spread_bp, created_at) VALUES (@id, @i, @t, @bp, @now) " + + "ON CONFLICT (instrument_id, sampled_at) DO UPDATE SET spread_bp = excluded.spread_bp", cancellationToken, + ("id", Ids.NewId(clock)), ("i", instrumentId), ("t", at), ("bp", bp), ("now", clock.UtcNow)), cancellationToken); + + public async Task> SpreadsAsync(Guid instrumentId, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync( + "SELECT sampled_at, spread_bp FROM spread_samples WHERE instrument_id = @i AND sampled_at >= @f AND sampled_at < @t ORDER BY sampled_at", + r => (r.Moment("sampled_at"), r.Measure("spread_bp")), ("i", instrumentId), ("f", from), ("t", to)), cancellationToken).ConfigureAwait(false); + + public Task PruneSpreadsAsync(CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync("DELETE FROM spread_samples WHERE sampled_at < @t", cancellationToken, ("t", clock.UtcNow.AddDays(-90))), + cancellationToken); +} + +/// Computes the universes from what the database holds and saves them (issue #24). +public sealed class UniverseBuilder(Database db, IClock clock) +{ + private readonly InstrumentCatalog _catalog = new(db, clock); + private readonly BarStore _bars = new(db, clock); + private readonly UniverseStore _store = new(db, clock); + + public async Task> EquityAsync(DateOnly asOf, CancellationToken cancellationToken) + { + var candidates = new List(); + foreach (var instrument in (await _catalog.ListAsync(Venues.Alpaca, cancellationToken).ConfigureAwait(false)).Where(i => EquityUniverse.Eligible(i.Spec))) + { + var bars = await _bars.DailyAsync(instrument.Id, asOf.AddDays(-(EquityUniverse.MinSessions * 7 / 5) - 30), asOf, cancellationToken).ConfigureAwait(false); + candidates.Add(new UniverseCandidate(instrument, bars)); + } + + var members = EquityUniverse.Select(asOf, candidates); + await _store.SaveAsync(Universes.Equity, asOf, members, cancellationToken).ConfigureAwait(false); + return members; + } + + /// The spread now, for pairs without 30 days of samples; null when unknown. + public async Task> CryptoAsync(DateOnly asOf, Func> currentSpread, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(currentSpread); + var candidates = new List(); + var end = QualityChecks.Midnight(asOf.AddDays(1)); + foreach (var instrument in (await _catalog.ListAsync(Venues.Okx, cancellationToken).ConfigureAwait(false)) + .Where(i => i.IsCrypto && i.Spec.QuoteCurrency == "EUR" && !CryptoUniverse.Excluded(i.Spec.BaseCurrency ?? ""))) + { + var bars = await _bars.DailyAsync(instrument.Id, asOf.AddDays(-CryptoUniverse.Window * 2), asOf, cancellationToken).ConfigureAwait(false); + var spreads = await _store.SpreadsAsync(instrument.Id, end.AddDays(-CryptoUniverse.Window), end, cancellationToken).ConfigureAwait(false); + candidates.Add(new UniverseCandidate(instrument, bars) + { + Spreads = spreads, + CurrentSpreadBp = spreads.Count > 0 && spreads[0].At <= end.AddDays(-CryptoUniverse.Window + 1) ? null : await currentSpread(instrument).ConfigureAwait(false), + }); + } + + var members = CryptoUniverse.Select(asOf, candidates); + await _store.SaveAsync(Universes.Crypto, asOf, members, cancellationToken).ConfigureAwait(false); + return members; + } +} + +/// Corporate actions as they arrive from Alpaca (SPEC 6.10), kept to adjust series and convert levels. +public sealed class CorporateActionStore(Database db, IClock clock) +{ + private const string Columns = "symbol, kind, ex_date, ratio, cash_amount, currency, new_symbol, source, source_key"; + + public async Task UpsertAsync(IEnumerable actions, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(actions); + var list = actions.DistinctBy(a => a.SourceKey).ToList(); + var now = clock.UtcNow; + await db.WriteAsync(async s => + { + foreach (var a in list) + { + var instrument = await s.FirstOrDefaultAsync("SELECT id FROM instruments WHERE venue = @v AND symbol = @s", r => (Guid?)r.Id("id"), + ("v", Venues.Alpaca), ("s", a.Symbol)).ConfigureAwait(false); + await s.ExecuteAsync( + $"INSERT INTO corporate_actions (id, instrument_id, {Columns}, created_at) VALUES (@id, @i, @sym, @k, @d, @r, @c, @ccy, @n, @src, @key, @now) " + + "ON CONFLICT (source_key) DO UPDATE SET instrument_id = excluded.instrument_id, kind = excluded.kind, ex_date = excluded.ex_date, " + + "ratio = excluded.ratio, cash_amount = excluded.cash_amount, currency = excluded.currency, new_symbol = excluded.new_symbol", + cancellationToken, ("id", Ids.NewId(clock)), ("i", instrument), ("sym", a.Symbol), ("k", a.Kind), ("d", BarStore.DayText(a.ExDate)), + ("r", a.Ratio), ("c", a.CashAmount), ("ccy", a.Currency), ("n", a.NewSymbol), ("src", a.Source), ("key", a.SourceKey), ("now", now)) + .ConfigureAwait(false); + } + }, cancellationToken).ConfigureAwait(false); + return list.Count; + } + + public async Task> ForSymbolAsync(string symbol, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync($"SELECT {Columns} FROM corporate_actions WHERE symbol = @s ORDER BY ex_date", Read, ("s", symbol)), + cancellationToken).ConfigureAwait(false); + + /// Actions effective on : the positions and levels of these symbols change at the open (Phase 3). + public async Task> OnAsync(DateOnly day, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync($"SELECT {Columns} FROM corporate_actions WHERE ex_date = @d ORDER BY symbol", Read, ("d", BarStore.DayText(day))), + cancellationToken).ConfigureAwait(false); + + private static CorporateAction Read(DbRow r) => new(r.Text("symbol"), r.Text("kind"), BarStore.ParseDay(r.Text("ex_date")), r.Text("source"), r.Text("source_key")) + { + Ratio = r.MoneyOrNull("ratio"), + CashAmount = r.MoneyOrNull("cash_amount"), + Currency = r.TextOrNull("currency"), + NewSymbol = r.TextOrNull("new_symbol"), + }; +} diff --git a/tests/Anubis.Tests/CollectorTests.cs b/tests/Anubis.Tests/CollectorTests.cs new file mode 100644 index 0000000..7f752fb --- /dev/null +++ b/tests/Anubis.Tests/CollectorTests.cs @@ -0,0 +1,222 @@ +using System.Threading.Channels; +using Anubis.Core; +using Anubis.Data; + +namespace Anubis.Tests; + +/// The collector of issue #20: backfill, checks, second readings, live feed, drops and gap recovery. +[TestClass] +public sealed class CollectorTests +{ + [TestMethod] + [TestCategory(Category.Integration)] + public async Task FeedDetachedForTenMinutesIsRecoveredOnReconnection() + { + var clock = new FastClock(new DateTimeOffset(2026, 11, 10, 10, 0, 30, TimeSpan.Zero)); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var bus = new EventBus(); + using var feed = bus.Subscribe(); + var freshness = new FreshnessMonitor(clock, bus); + var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false); + var source = new FakeMarketSource(clock); + var quiet = new DateTimeOffset(2026, 11, 10, 9, 30, 0, TimeSpan.Zero); + source.NoTrades.Add(quiet); + var collector = new MarketCollector(source, test.Db, clock, bus, freshness); + var bars = new BarStore(test.Db, clock); + var issues = new DataIssueStore(test.Db, clock); + + using var stop = new CancellationTokenSource(); + var live = collector.RunLiveAsync([btc], stop.Token); + await OkxTests.WaitForAsync(() => source.Connections == 1).ConfigureAwait(false); + + // Startup: two days of minutes downloaded; the minute without trades is recorded, not invented. + var start = clock.UtcNow; + Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 9, 59, 0, TimeSpan.Zero), await bars.LastMinuteAsync(btc.Id, CancellationToken.None).ConfigureAwait(false)); + var startup = await bars.MinuteAsync(btc.Id, start.AddDays(-3), start, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(2 * 24 * 60 - 1, startup.Count); + var unrecoverable = (await issues.SinceAsync(start.AddDays(-1), CancellationToken.None).ConfigureAwait(false)).Single(); + Assert.AreEqual((IssueKinds.Gap, IssueStatus.Unrecoverable, quiet), (unrecoverable.Kind, unrecoverable.Status, unrecoverable.StartsAt)); + StringAssert.Contains(unrecoverable.Detail, "senza scambi"); + + // Live: a closed bar arrives and the instrument is fresh. + clock.Advance(TimeSpan.FromSeconds(31)); + await source.Live!.Writer.WriteAsync(new BarEvent("BTC-EUR", FakeMarketSource.MinuteBar(new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero)), true)) + .ConfigureAwait(false); + await OkxTests.WaitForAsync(() => freshness.IsFresh(btc.Id)).ConfigureAwait(false); + Assert.IsTrue(collector.Connected); + + // The network goes down for ten minutes: the feed drops and nothing answers, not even the REST calls. + var dropAt = clock.UtcNow; + source.DownUntil = dropAt.AddMinutes(10); + source.Live.Writer.Complete(); + await OkxTests.WaitForAsync(() => source.Connections == 2).ConfigureAwait(false); + Assert.IsTrue(clock.UtcNow >= dropAt.AddMinutes(10), "reconnected only after the outage"); + Assert.IsFalse(freshness.IsFresh(btc.Id), "no live datum since the drop: no new entries"); + + // Back: every minute missed is downloaded before the feed resumes. + var resumed = new DateTimeOffset(clock.UtcNow.UtcTicks - (clock.UtcNow.UtcTicks % TimeSpan.TicksPerMinute), TimeSpan.Zero); + var recovered = await bars.MinuteAsync(btc.Id, new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero), resumed, CancellationToken.None).ConfigureAwait(false); + Assert.IsTrue(recovered.Count >= 11, $"at least the ten minutes of the outage, got {recovered.Count}"); + CollectionAssert.AreEqual( + QualityChecks.EveryMinute(new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero), resumed).ToList(), + recovered.Select(b => b.OpensAt).ToList(), "no minute is missing"); + Assert.AreEqual(1, (await issues.SinceAsync(start.AddDays(-1), CancellationToken.None).ConfigureAwait(false)).Count, "no new issue"); + + await source.Live!.Writer.WriteAsync(new TradeEvent("BTC-EUR", clock.UtcNow, 50_010m, 0.01m)).ConfigureAwait(false); + await OkxTests.WaitForAsync(() => freshness.IsFresh(btc.Id)).ConfigureAwait(false); + + await stop.CancelAsync().ConfigureAwait(false); + await live.ConfigureAwait(false); + var changes = new List(); + while (feed.TryRead(out var change)) + { + changes.Add(change!.Connected); + } + + CollectionAssert.AreEqual(new[] { true, false, true, false }, changes); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task SuspectBarsAreReadAgainAndNeverFixedInSilence() + { + var day0 = new DateOnly(2026, 6, 1); + var clock = new FastClock(new DateTimeOffset(day0.AddDays(120).ToDateTime(new TimeOnly(6, 0)), TimeSpan.Zero)); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var bus = new EventBus(); + var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false); + var walk = MarketDataLogicTests.Walk(120, day0); + var source = new FakeMarketSource(clock); + foreach (var bar in walk) + { + source.Days[bar.Day] = bar; + } + + var spike = day0.AddDays(80); + var broken = day0.AddDays(90); + var lost = day0.AddDays(50); + source.FirstReadings[spike] = new Queue([walk[80] with { High = walk[80].High * 3, Close = walk[80].Close * 3 }]); + source.Days[broken] = walk[90] with { Low = -1 }; + source.MissingOnFirstRead.Add(lost); + + var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus)); + var result = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false); + + var bars = new BarStore(test.Db, clock); + var stored = await bars.DailyAsync(btc.Id, day0, day0.AddDays(200), CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(119, stored.Count, "the impossible day is dropped, the lost one recovered"); + Assert.AreEqual(walk[80].Close, stored.Single(b => b.Day == spike).Close, "the second reading replaced the spike"); + Assert.IsFalse(stored.Any(b => b.Day == broken)); + Assert.IsTrue(stored.Any(b => b.Day == lost)); + + var issues = await new DataIssueStore(test.Db, clock).SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false); + var byDay = issues.ToLookup(i => DateOnly.FromDateTime(i.StartsAt.UtcDateTime)); + Assert.AreEqual(IssueStatus.Corrected, byDay[spike].Single(i => i.Kind == IssueKinds.Jump).Status); + StringAssert.Contains(byDay[spike].Single(i => i.Kind == IssueKinds.Jump).Detail, "invece di"); + Assert.AreEqual(IssueStatus.Unrecoverable, byDay[broken].Single(i => i.Kind == IssueKinds.NonPositive).Status); + Assert.AreEqual(IssueStatus.Recovered, byDay[lost].Single(i => i.Kind == IssueKinds.Gap).Status); + Assert.IsTrue(issues.All(i => i.Attempts == 1), "each problem was tried once more"); + Assert.AreEqual(0, (await new DataIssueStore(test.Db, clock).BlockedInstrumentsAsync(CancellationToken.None).ConfigureAwait(false)).Count, + "once read again, nothing blocks new entries"); + Assert.IsTrue(result.Suspect >= 2 && result.Corrected >= 1 && result.Unrecoverable == 1 && result.Recovered == 1, result.ToString()); + + // A suspect bar the second reading confirms stays, marked as confirmed. + source.Days[day0.AddDays(120)] = walk[^1] with { Day = day0.AddDays(120), Open = walk[^1].Close, Low = walk[^1].Close * 0.5, Close = walk[^1].Close * 0.5 }; + clock.Advance(TimeSpan.FromDays(1)); + var next = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(1, next.Confirmed); + Assert.AreEqual(BarQuality.Confirmed, (await bars.DailyAsync(btc.Id, day0.AddDays(120), day0.AddDays(120), CancellationToken.None).ConfigureAwait(false))[0].Quality); + Assert.AreEqual(CollectionResult.None, await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false), "nothing new"); + } +} + +/// +/// A venue for the collector tests: minute bars exist for every past minute except , daily +/// bars come from , and the whole venue can be down until . +/// +internal sealed class FakeMarketSource(IClock clock) : IMarketDataSource +{ + public string Venue => Venues.Okx; + + public HashSet NoTrades { get; } = []; + + public Dictionary Days { get; } = []; + + /// What the first reading of a day returns instead of . + public Dictionary> FirstReadings { get; } = []; + + public HashSet MissingOnFirstRead { get; } = []; + + public DateTimeOffset? DownUntil { get; set; } + + public Channel? Live { get; private set; } + + public int Connections; + + public static Bar MinuteBar(DateTimeOffset t) + { + var price = 50_000 + (20 * Math.Sin(t.ToUnixTimeSeconds() / 60.0 / 15)); + return new Bar(t, price, price + 5, price - 5, price + 1, 0.5); + } + + public Task> ListInstrumentsAsync(CancellationToken cancellationToken) => + Task.FromResult>([MarketDataStoreTests.Okx("BTC-EUR")]); + + public Task> FetchDailyBarsAsync(string symbol, DateOnly first, DateOnly last, CancellationToken cancellationToken) + { + ThrowIfDown(); + var list = new List(); + for (var day = first; day <= last; day = day.AddDays(1)) + { + if (FirstReadings.TryGetValue(day, out var queue) && queue.Count > 0) + { + list.Add(queue.Dequeue()); + } + else if (!MissingOnFirstRead.Remove(day) && Days.TryGetValue(day, out var bar)) + { + list.Add(bar); + } + } + + return Task.FromResult>(list); + } + + public Task> FetchMinuteBarsAsync(string symbol, DateTimeOffset start, DateTimeOffset until, CancellationToken cancellationToken) + { + ThrowIfDown(); + var now = clock.UtcNow; + var list = new List(); + for (var t = start; t < until && t.AddMinutes(1) <= now; t = t.AddMinutes(1)) + { + if (!NoTrades.Contains(t)) + { + list.Add(MinuteBar(t)); + } + } + + return Task.FromResult>(list); + } + + public Task FetchQuoteAsync(string symbol, CancellationToken cancellationToken) => + Task.FromResult(new Quote(symbol, 49_999m, 50_001m, clock.UtcNow)); + + public async IAsyncEnumerable StreamAsync(IReadOnlyCollection symbols, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + ThrowIfDown(); + Live = Channel.CreateUnbounded(); + Interlocked.Increment(ref Connections); + await foreach (var item in Live.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return item; + } + } + + private void ThrowIfDown() + { + if (DownUntil is { } until && clock.UtcNow < until) + { + throw new MarketDataException("rete assente", retryable: true); + } + } +} diff --git a/tests/Anubis.Tests/UniverseTests.cs b/tests/Anubis.Tests/UniverseTests.cs new file mode 100644 index 0000000..37494be --- /dev/null +++ b/tests/Anubis.Tests/UniverseTests.cs @@ -0,0 +1,237 @@ +using Anubis.Core; +using Anubis.Data; + +namespace Anubis.Tests; + +/// Universes "as of a date" (issue #24) and the corporate action store (issue #23). +[TestClass] +public sealed class UniverseTests +{ + private static readonly DateOnly[] _pastDates = [new(2019, 12, 31), new(2021, 6, 30), new(2024, 3, 28)]; + + private static readonly DateTimeOffset _now = new(2026, 11, 10, 8, 0, 0, TimeSpan.Zero); + + /// + /// The securities of the recorded nasdaqtraded.txt plus 40 common shares, a stock delisted in 2020, a cheap one + /// and one listed in 2021. Funds and ETFs trade the most, so they would lead the ranking if they got through. + /// + private static List Market() + { + var listed = NasdaqTraded.Parse(Fixture.Text("free/nasdaqtraded.txt")); + var specs = listed.Select(l => Stock(l.Symbol, l.Name, l.Exchange)).ToList(); + specs.AddRange(Enumerable.Range(1, 40).Select(i => Stock($"S{i:D3}", $"Synthetic {i} Inc. Common Stock", i % 2 == 0 ? "NYSE" : "NASDAQ"))); + specs.Add(Stock("DEAD", "Gone Corp. Common Stock", "NYSE")); + specs.Add(Stock("CHEAP", "Penny Corp. Common Stock", "NASDAQ")); + specs.Add(Stock("YOUNG", "New Listing Inc. Common Stock", "NASDAQ")); + specs.Add(Stock("NOFRAC", "Whole Shares Only Inc.", "NYSE") with { Fractionable = false }); + var marked = NasdaqTraded.MarkFunds(specs, listed); + + var candidates = new List(); + foreach (var spec in marked) + { + var symbol = spec.Symbol; + var first = symbol == "YOUNG" ? new DateOnly(2021, 1, 4) : new DateOnly(2018, 1, 2); + var last = symbol == "DEAD" ? new DateOnly(2020, 6, 30) : new DateOnly(2026, 11, 9); + var price = symbol == "CHEAP" ? 5.0 : 20.0 + (symbol.GetHashCode(StringComparison.Ordinal) & 0x3f); + var volume = spec.IsFund == true || symbol.StartsWith("ZX", StringComparison.Ordinal) ? 50_000_000.0 : 100_000.0 * (1 + (symbol.Length * 7 % 13)); + if (symbol is "DEAD" or "YOUNG") + { + volume = 10_000_000; // Large enough to rank first whenever they are eligible. + } + + candidates.Add(new UniverseCandidate(new Instrument(Guid.NewGuid(), spec, null), Sessions(first, last, price, volume))); + } + + return candidates; + } + + private static InstrumentSpec Stock(string symbol, string name, string exchange) => + new(Venues.Alpaca, symbol, name, AssetClasses.Equity, "USD") { Exchange = exchange, Fractionable = true, Status = "active" }; + + private static List Sessions(DateOnly first, DateOnly last, double price, double volume) + { + var bars = new List(); + for (var day = first; day <= last; day = day.AddDays(1)) + { + if (day.DayOfWeek is not (DayOfWeek.Saturday or DayOfWeek.Sunday)) + { + bars.Add(new DailyBar(day, price, price * 1.01, price * 0.99, price, volume)); + } + } + + return bars; + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void EquityUniverseAtThreePastDatesHasNoEtfsNorFunds() + { + var market = Market(); + var excluded = new HashSet { "SPY", "BULZ", "AMUB", "PDI", "ABR$D", "AACIW", "ZXIET", "CHEAP", "NOFRAC" }; + foreach (var date in _pastDates) + { + var members = EquityUniverse.Select(date, market); + var symbols = members.Select(m => m.Symbol).ToList(); + Assert.IsFalse(symbols.Any(excluded.Contains), $"{date}: {string.Join(", ", symbols.Where(excluded.Contains))}"); + Assert.IsFalse(symbols.Contains("CECO"), "Texas Stock Exchange is neither NYSE nor Nasdaq"); + CollectionAssert.IsSubsetOf(new[] { "AAPL", "KO", "AACG", "S001", "S040" }, symbols, BarStore.DayText(date)); + CollectionAssert.AreEqual(Enumerable.Range(1, members.Count).ToList(), members.Select(m => m.Rank).ToList()); + } + + // Survivorship: the stock delisted in 2020 belongs to the 2019 universe and to no later one. + Assert.AreEqual(1, EquityUniverse.Select(_pastDates[0], market)[0].Rank); + Assert.AreEqual("DEAD", EquityUniverse.Select(_pastDates[0], market)[0].Symbol); + Assert.IsFalse(EquityUniverse.Select(_pastDates[1], market).Any(m => m.Symbol == "DEAD")); + + // History: a stock listed in January 2021 has fewer than 300 sessions in June 2021, enough in 2024. + Assert.IsFalse(EquityUniverse.Select(_pastDates[1], market).Any(m => m.Symbol == "YOUNG")); + Assert.AreEqual("YOUNG", EquityUniverse.Select(_pastDates[2], market)[0].Symbol); + + // The size cut keeps the most traded. + var top = EquityUniverse.Select(_pastDates[2], market, size: 5); + Assert.AreEqual(5, top.Count); + Assert.IsTrue(top.Zip(top.Skip(1)).All(p => p.First.Measure >= p.Second.Measure)); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void UniverseOnADateIgnoresEverythingAfterIt() + { + var market = Market(); + var date = _pastDates[1]; + var truncated = market.Select(c => c with { Bars = c.Bars.Where(b => b.Day <= date).ToList() }).ToList(); + + // A later crash of AAPL to 1 $ and a later volume explosion of S002 must not change the June 2021 universe. + var future = market.Select(c => c.Instrument.Symbol switch + { + "AAPL" => c with { Bars = [.. c.Bars.Select(b => b.Day > date ? b with { Close = 1, Low = 0.9, Open = 1, High = 1.1 } : b)] }, + "S002" => c with { Bars = [.. c.Bars.Select(b => b.Day > date ? b with { Volume = 1e12 } : b)] }, + _ => c, + }).ToList(); + + var expected = EquityUniverse.Select(date, truncated); + CollectionAssert.AreEqual(expected.ToList(), EquityUniverse.Select(date, market).ToList()); + CollectionAssert.AreEqual(expected.ToList(), EquityUniverse.Select(date, future).ToList()); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void CryptoUniverseFiltersAgeValueSpreadAndExcludedTokens() + { + var asOf = new DateOnly(2026, 11, 1); + var end = QualityChecks.Midnight(asOf.AddDays(1)); + UniverseCandidate Pair(string coin, double value, double? current, int ageDays = 900, double? sampled = null) => + new(new Instrument(Guid.NewGuid(), new InstrumentSpec(Venues.Okx, coin + "-EUR", coin + "/EUR", AssetClasses.Crypto, "EUR") + { + BaseCurrency = coin, + ListedAt = end.AddDays(-ageDays), + }, null), Sessions(asOf.AddDays(-60), asOf, 100, value / 100).Concat([]).ToList()) + { + CurrentSpreadBp = current, + Spreads = sampled is { } bp ? [.. Enumerable.Range(1, 30 * 24).Select(h => (end.AddHours(-h), bp))] : [], + }; + + // Weekends are missing from Sessions: give crypto every day. + UniverseCandidate Daily(UniverseCandidate c) => c with + { + Bars = [.. QualityChecks.EveryDay(asOf.AddDays(-60), asOf).Select(d => c.Bars[0] with { Day = d })], + }; + + var candidates = new[] + { + Pair("BTC", 50_000_000, 2), Pair("ETH", 20_000_000, 3), Pair("SOL", 5_000_000, null, sampled: 8), + Pair("XRP", 3_000_000, 50, sampled: 10), // 30 days of samples win over a wide spread right now. + Pair("ADA", 2_000_000, 25), Pair("DOT", 200_000, 5), Pair("NEW", 9_000_000, 5, ageDays: 200), + Pair("USDC", 90_000_000, 1), Pair("WBTC", 9_000_000, 5), Pair("BTC3L", 9_000_000, 5), Pair("LINK", 1_000_000, null), + }.Select(Daily).ToList(); + var members = CryptoUniverse.Select(asOf, candidates); + CollectionAssert.AreEqual(new[] { "BTC-EUR", "ETH-EUR", "SOL-EUR", "XRP-EUR" }, members.Select(m => m.Symbol).ToList(), + "ADA spread too wide, DOT too small, NEW too young, stable/wrapped/leveraged out, LINK spread unknown"); + Assert.AreEqual(2, CryptoUniverse.Select(asOf, candidates, size: 2).Count); + Assert.IsTrue(CryptoUniverse.Excluded("ETHUP") && CryptoUniverse.Excluded("eurc") && !CryptoUniverse.Excluded("UNI")); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task UniversesAreComputedFromTheDatabaseAndKeptByDate() + { + var clock = new ManualClock(_now); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var catalog = new InstrumentCatalog(test.Db, clock); + var bars = new BarStore(test.Db, clock); + var listed = NasdaqTraded.Parse(Fixture.Text("free/nasdaqtraded.txt")); + var specs = NasdaqTraded.MarkFunds( + [Stock("AAPL", "Apple Inc. - Common Stock", "NASDAQ"), Stock("SPY", "SPDR S&P 500 ETF Trust", "ARCA") with { Exchange = "NYSE" }, + Stock("PDI", "PIMCO Dynamic Income Fund", "NYSE"), Stock("KO", "Coca-Cola Company (The) Common Stock", "NYSE")], listed); + await catalog.SyncAsync(Venues.Alpaca, specs, complete: true, CancellationToken.None).ConfigureAwait(false); + foreach (var instrument in await catalog.ListAsync(Venues.Alpaca, CancellationToken.None).ConfigureAwait(false)) + { + var volume = instrument.Symbol == "SPY" ? 1e9 : instrument.Symbol == "AAPL" ? 2e6 : 1e6; + await bars.UpsertDailyAsync(instrument.Id, Sessions(new DateOnly(2024, 1, 2), new DateOnly(2026, 11, 6), 50, volume), Venues.Alpaca, CancellationToken.None) + .ConfigureAwait(false); + } + + var builder = new UniverseBuilder(test.Db, clock); + var equity = await builder.EquityAsync(new DateOnly(2026, 11, 6), CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEqual(new[] { "AAPL", "KO" }, equity.Select(m => m.Symbol).ToList(), "SPY is an ETF even when listed on NYSE; PDI a fund"); + await builder.EquityAsync(new DateOnly(2025, 3, 3), CancellationToken.None).ConfigureAwait(false); + + var store = new UniverseStore(test.Db, clock); + Assert.AreEqual(2, (await store.AsOfAsync(Universes.Equity, new DateOnly(2026, 11, 9), CancellationToken.None).ConfigureAwait(false)).Count); + Assert.AreEqual(2, (await store.AsOfAsync(Universes.Equity, new DateOnly(2025, 6, 1), CancellationToken.None).ConfigureAwait(false)).Count); + Assert.AreEqual(0, (await store.AsOfAsync(Universes.Equity, new DateOnly(2025, 1, 1), CancellationToken.None).ConfigureAwait(false)).Count, + "before the first computation there is no universe"); + + // Crypto: spread samples come from the hourly job; without 30 days of them the current spread decides. + var btc = await catalog.EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR") with { ListedAt = _now.AddYears(-3) }, CancellationToken.None).ConfigureAwait(false); + var eth = await catalog.EnsureAsync(MarketDataStoreTests.Okx("ETH-EUR") with { ListedAt = _now.AddYears(-3) }, CancellationToken.None).ConfigureAwait(false); + foreach (var pair in new[] { btc, eth }) + { + await bars.UpsertDailyAsync(pair.Id, [.. QualityChecks.EveryDay(new DateOnly(2026, 9, 1), new DateOnly(2026, 11, 9)) + .Select(d => new DailyBar(d, 60_000, 61_000, 59_000, 60_000, 100) { QuoteVolume = 6_000_000 })], Venues.Okx, CancellationToken.None).ConfigureAwait(false); + } + + for (var h = 1; h <= 30 * 24; h++) + { + await store.AddSpreadAsync(eth.Id, QualityChecks.Midnight(new DateOnly(2026, 11, 10)).AddHours(-h), 30, CancellationToken.None).ConfigureAwait(false); + } + + var asked = new List(); + var crypto = await builder.CryptoAsync(new DateOnly(2026, 11, 9), i => + { + asked.Add(i.Symbol); + return Task.FromResult(1.5); + }, CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEqual(new[] { "BTC-EUR" }, crypto.Select(m => m.Symbol).ToList(), "ETH's 30-day median spread is 30 bp"); + CollectionAssert.AreEqual(new[] { "BTC-EUR" }, asked, "the current spread is asked only where samples are missing"); + Assert.AreEqual(0, await store.PruneSpreadsAsync(CancellationToken.None).ConfigureAwait(false)); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task CorporateActionsAreKeptOnceAndFoundByDay() + { + var clock = new ManualClock(_now); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var aapl = await new InstrumentCatalog(test.Db, clock).EnsureAsync(Stock("AAPL", "Apple Inc.", "NASDAQ"), CancellationToken.None).ConfigureAwait(false); + var store = new CorporateActionStore(test.Db, clock); + var split = new CorporateAction("AAPL", CorporateActionKinds.Split, new DateOnly(2020, 8, 31), Venues.Alpaca, "alpaca:split:1") { Ratio = 4m }; + var dividend = new CorporateAction("AAPL", CorporateActionKinds.CashDividend, new DateOnly(2026, 11, 10), Venues.Alpaca, "alpaca:div:2") + { + CashAmount = 0.26m, + Currency = "USD", + }; + var other = new CorporateAction("ZZZ", CorporateActionKinds.SymbolChange, new DateOnly(2026, 11, 10), Venues.Alpaca, "alpaca:sc:3") { NewSymbol = "ZZY" }; + + Assert.AreEqual(3, await store.UpsertAsync([split, dividend, other, split], CancellationToken.None).ConfigureAwait(false)); + await store.UpsertAsync([dividend with { CashAmount = 0.27m }], CancellationToken.None).ConfigureAwait(false); + var forApple = await store.ForSymbolAsync("AAPL", CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEqual(new[] { split, dividend with { CashAmount = 0.27m } }, forApple.ToList()); + var today = await store.OnAsync(new DateOnly(2026, 11, 10), CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEqual(new[] { "AAPL", "ZZZ" }, today.Select(a => a.Symbol).ToList()); + + var linked = await test.Db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM corporate_actions WHERE instrument_id = @i", ("i", aapl.Id)), CancellationToken.None) + .ConfigureAwait(false); + Assert.AreEqual(2L, linked, "actions of a known symbol point to its instrument"); + } +}