MarketCollector downloads what is missing at startup and after every drop of the live feed, checks every bar and never corrects in silence: suspect bars are stored as suspect and read a second time (same values confirm, different ones replace, impossible ones are dropped), gaps are downloaded again once and what the source still lacks is recorded as unrecoverable. A test detaches the feed for ten minutes and finds every minute recovered. The stock and crypto universes are chosen "as of a date" from the bars up to that date only: funds, ETFs and ETNs are excluded through nasdaqtraded.txt and the security name, stocks delisted later still count when they traded. Corporate actions are kept once per source key and found by ex-date. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b3fb69e696
commit
49ed73e6f6
6 files changed
+1177
No files matched your search
@@ -117,6 +117,10 @@ public sealed class BarStore(Database db, IClock clock)
|
||||
r => new Bar(r.Moment("opens_at"), r.Measure("open"), r.Measure("high"), r.Measure("low"), r.Measure("close"), r.Measure("volume")),
|
||||
("i", instrumentId), ("f", from), ("t", to)), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
public Task DeleteMinuteAsync(Guid instrumentId, DateTimeOffset opensAt, CancellationToken cancellationToken) =>
|
||||
db.WriteAsync(s => s.ExecuteAsync("DELETE FROM bars_1m WHERE instrument_id = @i AND opens_at = @t", cancellationToken, ("i", instrumentId), ("t", opensAt)),
|
||||
cancellationToken);
|
||||
|
||||
public Task<DateTimeOffset?> LastMinuteAsync(Guid instrumentId, CancellationToken cancellationToken) =>
|
||||
db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT opens_at FROM bars_1m WHERE instrument_id = @i ORDER BY opens_at DESC",
|
||||
r => (DateTimeOffset?)r.Moment("opens_at"), ("i", instrumentId)), cancellationToken);
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
using Anubis.Core;
|
||||
using Anubis.Db;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace Anubis.Data;
|
||||
|
||||
/// <summary>What a collection pass did, for the log and the quality report.</summary>
|
||||
public sealed record CollectionResult(int Stored, int Suspect, int Confirmed, int Corrected, int Gaps, int Recovered, int Unrecoverable)
|
||||
{
|
||||
public static readonly CollectionResult None = new(0, 0, 0, 0, 0, 0, 0);
|
||||
|
||||
public CollectionResult Add(CollectionResult other) => new(Stored + other.Stored, Suspect + other.Suspect,
|
||||
Confirmed + other.Confirmed, Corrected + other.Corrected, Gaps + other.Gaps, Recovered + other.Recovered, Unrecoverable + other.Unrecoverable);
|
||||
}
|
||||
|
||||
/// <summary>The live feed of a venue connected, dropped or came back (SPEC 6.2).</summary>
|
||||
public sealed record FeedChanged(string Venue, bool Connected, DateTimeOffset At, string? Reason);
|
||||
|
||||
/// <summary>
|
||||
/// The collector of SPEC 6.2–6.4 for one venue: downloads what is missing at startup and after every drop of the
|
||||
/// live feed, checks every bar, and never corrects in silence. A suspect bar is stored as suspect, blocks new
|
||||
/// entries on its instrument (see <see cref="DataIssueStore.BlockedInstrumentsAsync"/>) and is read a second
|
||||
/// time: the same values confirm it, different ones replace it, and an impossible bar is dropped. Gaps are
|
||||
/// downloaded again once; what the source still lacks is recorded as unrecoverable.
|
||||
/// </summary>
|
||||
public sealed partial class MarketCollector
|
||||
{
|
||||
/// <summary>History of minute bars downloaded the first time an instrument is followed.</summary>
|
||||
public static readonly TimeSpan InitialMinutes = TimeSpan.FromDays(2);
|
||||
|
||||
/// <summary>Waits between reconnection attempts: 1, 2, 5, 10, 30 seconds, then 60.</summary>
|
||||
public static readonly IReadOnlyList<TimeSpan> Backoff =
|
||||
[TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60)];
|
||||
|
||||
/// <summary>Closes used as context for the jump check of new bars.</summary>
|
||||
private const int Context = QualityChecks.JumpWindow + 1;
|
||||
|
||||
private readonly IMarketDataSource _source;
|
||||
private readonly IClock _clock;
|
||||
private readonly EventBus _bus;
|
||||
private readonly FreshnessMonitor _freshness;
|
||||
private readonly Func<Instrument, DateTimeOffset, DateTimeOffset, IEnumerable<DateTimeOffset>?> _expectedMinutes;
|
||||
private readonly Func<Instrument, DateOnly, DateOnly, IEnumerable<DateOnly>?> _expectedDays;
|
||||
private readonly ILogger _logger;
|
||||
private readonly BarStore _bars;
|
||||
private readonly DataIssueStore _issues;
|
||||
|
||||
/// <param name="expectedMinutes">Minutes a bar should exist for; by default every minute for crypto and none (no gap check) for stocks.</param>
|
||||
/// <param name="expectedDays">Days a daily bar should exist for; by default every day for crypto and none for stocks.</param>
|
||||
public MarketCollector(
|
||||
IMarketDataSource source,
|
||||
Database db,
|
||||
IClock clock,
|
||||
EventBus bus,
|
||||
FreshnessMonitor freshness,
|
||||
Func<Instrument, DateTimeOffset, DateTimeOffset, IEnumerable<DateTimeOffset>?>? expectedMinutes = null,
|
||||
Func<Instrument, DateOnly, DateOnly, IEnumerable<DateOnly>?>? expectedDays = null,
|
||||
ILogger? logger = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(source);
|
||||
_source = source;
|
||||
_clock = clock;
|
||||
_bus = bus;
|
||||
_freshness = freshness;
|
||||
_expectedMinutes = expectedMinutes ?? ((i, from, to) => i.IsCrypto ? QualityChecks.EveryMinute(from, to) : null);
|
||||
_expectedDays = expectedDays ?? ((i, from, to) => i.IsCrypto ? QualityChecks.EveryDay(from, to) : null);
|
||||
_logger = logger ?? NullLogger.Instance;
|
||||
_bars = new BarStore(db, clock);
|
||||
_issues = new DataIssueStore(db, clock);
|
||||
}
|
||||
|
||||
public string Venue => _source.Venue;
|
||||
|
||||
/// <summary>True while the live feed is connected.</summary>
|
||||
public bool Connected { get; private set; }
|
||||
|
||||
/// <summary>Downloads the daily bars after the last one stored (from <paramref name="from"/> the first time) up to yesterday.</summary>
|
||||
public async Task<CollectionResult> BackfillDailyAsync(Instrument instrument, DateOnly from, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(instrument);
|
||||
var yesterday = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime).AddDays(-1);
|
||||
var last = await _bars.LastDayAsync(instrument.Id, cancellationToken).ConfigureAwait(false);
|
||||
var first = last?.AddDays(1) ?? from;
|
||||
if (first > yesterday)
|
||||
{
|
||||
return CollectionResult.None;
|
||||
}
|
||||
|
||||
var fetched = await _source.FetchDailyBarsAsync(instrument.Symbol, first, yesterday, cancellationToken).ConfigureAwait(false);
|
||||
if (fetched.Count == 0)
|
||||
{
|
||||
return CollectionResult.None;
|
||||
}
|
||||
|
||||
var context = await _bars.DailyAsync(instrument.Id, first.AddDays(-Context * 2), first.AddDays(-1), cancellationToken).ConfigureAwait(false);
|
||||
var all = context.Where(b => b.Day < fetched[0].Day).TakeLast(Context).Concat(fetched).ToList();
|
||||
var newFrom = QualityChecks.Midnight(fetched[0].Day);
|
||||
var findings = QualityChecks.CheckDaily(all, _expectedDays(instrument, all[0].Day, all[^1].Day))
|
||||
.Where(f => f.StartsAt >= newFrom || (f.Kind == IssueKinds.Gap && f.EndsAt >= newFrom)).ToList();
|
||||
|
||||
var result = CollectionResult.None;
|
||||
var suspectDays = findings.Where(f => IssueKinds.MakesSuspect(f.Kind)).Select(f => DateOnly.FromDateTime(f.StartsAt.UtcDateTime)).ToHashSet();
|
||||
var impossible = findings.Where(f => f.Kind is IssueKinds.NonPositive or IssueKinds.HighBelowLow)
|
||||
.Select(f => DateOnly.FromDateTime(f.StartsAt.UtcDateTime)).ToHashSet();
|
||||
var stored = fetched.DistinctBy(b => b.Day)
|
||||
.Where(b => !impossible.Contains(b.Day))
|
||||
.Select(b => suspectDays.Contains(b.Day) ? b with { Quality = BarQuality.Suspect } : b)
|
||||
.ToList();
|
||||
await _bars.UpsertDailyAsync(instrument.Id, stored, Venue, cancellationToken).ConfigureAwait(false);
|
||||
result = result with { Stored = stored.Count };
|
||||
|
||||
foreach (var finding in findings)
|
||||
{
|
||||
var id = await _issues.RecordAsync(instrument.Id, Timeframes.Day, finding, cancellationToken).ConfigureAwait(false);
|
||||
if (IssueKinds.MakesSuspect(finding.Kind))
|
||||
{
|
||||
result = result.Add(await SecondDailyReadingAsync(instrument, id, finding, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
else if (finding.Kind == IssueKinds.Gap)
|
||||
{
|
||||
result = result.Add(await RecoverDailyGapAsync(instrument, id, finding, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>Downloads the minute bars after the last one stored, up to the last closed minute.</summary>
|
||||
public async Task<CollectionResult> BackfillMinutesAsync(Instrument instrument, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(instrument);
|
||||
var until = Minute(_clock.UtcNow);
|
||||
var last = await _bars.LastMinuteAsync(instrument.Id, cancellationToken).ConfigureAwait(false);
|
||||
var start = last?.AddMinutes(1) ?? until - InitialMinutes;
|
||||
if (start >= until)
|
||||
{
|
||||
return CollectionResult.None;
|
||||
}
|
||||
|
||||
var fetched = await _source.FetchMinuteBarsAsync(instrument.Symbol, start, until, cancellationToken).ConfigureAwait(false);
|
||||
var context = last is { } l
|
||||
? await _bars.MinuteAsync(instrument.Id, l.AddMinutes(-Context), l.AddMinutes(1), cancellationToken).ConfigureAwait(false)
|
||||
: [];
|
||||
// Minutes the source has not published yet are not gaps: the next pass starts again from the last bar held,
|
||||
// and a minute still missing then lies between two bars and is found by the check.
|
||||
return await AcceptMinutesAsync(instrument, context, fetched, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Follows <paramref name="instruments"/> on the live feed until cancelled: every datum refreshes the freshness of
|
||||
/// its instrument, every closed bar is checked and stored, and every drop is followed by a reconnection and by the
|
||||
/// download of what was missed meanwhile.
|
||||
/// </summary>
|
||||
public async Task RunLiveAsync(IReadOnlyList<Instrument> instruments, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(instruments);
|
||||
var bySymbol = instruments.ToDictionary(i => i.Symbol, StringComparer.Ordinal);
|
||||
foreach (var instrument in instruments)
|
||||
{
|
||||
_freshness.Follow(instrument.Id, instrument.Symbol);
|
||||
}
|
||||
|
||||
var recent = new Dictionary<Guid, List<Bar>>();
|
||||
var failures = 0;
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
string? reason = null;
|
||||
try
|
||||
{
|
||||
foreach (var instrument in instruments)
|
||||
{
|
||||
var recovered = await BackfillMinutesAsync(instrument, cancellationToken).ConfigureAwait(false);
|
||||
if (recovered.Stored > 0)
|
||||
{
|
||||
LogRecovered(_logger, Venue, instrument.Symbol, recovered.Stored);
|
||||
}
|
||||
|
||||
var last = await _bars.LastMinuteAsync(instrument.Id, cancellationToken).ConfigureAwait(false);
|
||||
recent[instrument.Id] = last is { } l
|
||||
? (await _bars.MinuteAsync(instrument.Id, l.AddMinutes(-Context), l.AddMinutes(1), cancellationToken).ConfigureAwait(false)).ToList()
|
||||
: [];
|
||||
}
|
||||
|
||||
await foreach (var item in _source.StreamAsync(bySymbol.Keys, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!Connected)
|
||||
{
|
||||
Connected = true;
|
||||
failures = 0;
|
||||
_bus.Publish(new FeedChanged(Venue, true, _clock.UtcNow, null));
|
||||
}
|
||||
|
||||
if (!bySymbol.TryGetValue(item.Symbol, out var instrument))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (item is BarEvent { Closed: true } closed)
|
||||
{
|
||||
var context = recent[instrument.Id];
|
||||
await AcceptMinutesAsync(instrument, context, [closed.Bar], cancellationToken).ConfigureAwait(false);
|
||||
if (Possible(closed.Bar))
|
||||
{
|
||||
context.RemoveAll(b => b.OpensAt == closed.Bar.OpensAt);
|
||||
context.Add(closed.Bar);
|
||||
}
|
||||
|
||||
if (context.Count > Context * 2)
|
||||
{
|
||||
context.RemoveRange(0, context.Count - Context);
|
||||
}
|
||||
|
||||
_freshness.Touch(instrument.Id, closed.Bar.OpensAt.AddMinutes(1));
|
||||
}
|
||||
else if (item is not BarEvent)
|
||||
{
|
||||
_freshness.Touch(instrument.Id, item.At);
|
||||
}
|
||||
|
||||
_bus.Publish(item);
|
||||
}
|
||||
|
||||
reason = "il flusso si è chiuso";
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex) when (ex is MarketDataException or HttpRequestException or System.Net.WebSockets.WebSocketException or IOException
|
||||
or System.Text.Json.JsonException or TimeoutException or TaskCanceledException)
|
||||
{
|
||||
reason = Redactor.Redact(ex.Message);
|
||||
LogFeedError(_logger, Venue, reason);
|
||||
}
|
||||
|
||||
if (Connected)
|
||||
{
|
||||
Connected = false;
|
||||
_bus.Publish(new FeedChanged(Venue, false, _clock.UtcNow, reason));
|
||||
}
|
||||
|
||||
var wait = Backoff[Math.Min(failures, Backoff.Count - 1)];
|
||||
failures++;
|
||||
try
|
||||
{
|
||||
await _clock.WaitUntilAsync(_clock.UtcNow + wait, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (Connected)
|
||||
{
|
||||
Connected = false;
|
||||
_bus.Publish(new FeedChanged(Venue, false, _clock.UtcNow, "arresto"));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Checks new minute bars against their context, stores them, and handles what the checks found.</summary>
|
||||
private async Task<CollectionResult> AcceptMinutesAsync(Instrument instrument, IReadOnlyList<Bar> context, IReadOnlyList<Bar> fresh, CancellationToken cancellationToken)
|
||||
{
|
||||
if (fresh.Count == 0)
|
||||
{
|
||||
return CollectionResult.None;
|
||||
}
|
||||
|
||||
var times = fresh.Select(b => b.OpensAt).ToHashSet();
|
||||
var all = context.Where(b => !times.Contains(b.OpensAt)).TakeLast(Context).Concat(fresh).ToList();
|
||||
var newFrom = fresh.Min(b => b.OpensAt);
|
||||
var expected = _expectedMinutes(instrument, all.Min(b => b.OpensAt), all.Max(b => b.OpensAt).AddMinutes(1));
|
||||
var findings = QualityChecks.CheckMinutes(all, expected)
|
||||
.Where(f => f.StartsAt >= newFrom || (f.Kind == IssueKinds.Gap && f.EndsAt >= newFrom)).ToList();
|
||||
var suspect = findings.Where(f => IssueKinds.MakesSuspect(f.Kind)).Select(f => f.StartsAt).ToHashSet();
|
||||
var impossible = findings.Where(f => f.Kind is IssueKinds.NonPositive or IssueKinds.HighBelowLow).Select(f => f.StartsAt).ToHashSet();
|
||||
|
||||
var good = fresh.DistinctBy(b => b.OpensAt).Where(b => !suspect.Contains(b.OpensAt)).ToList();
|
||||
var doubtful = fresh.DistinctBy(b => b.OpensAt).Where(b => suspect.Contains(b.OpensAt) && !impossible.Contains(b.OpensAt)).ToList();
|
||||
await _bars.UpsertMinuteAsync(instrument.Id, good, Venue, BarQuality.Ok, cancellationToken).ConfigureAwait(false);
|
||||
await _bars.UpsertMinuteAsync(instrument.Id, doubtful, Venue, BarQuality.Suspect, cancellationToken).ConfigureAwait(false);
|
||||
var result = CollectionResult.None with { Stored = good.Count + doubtful.Count };
|
||||
|
||||
foreach (var finding in findings)
|
||||
{
|
||||
if (IssueKinds.MakesSuspect(finding.Kind))
|
||||
{
|
||||
var id = await _issues.RecordAsync(instrument.Id, Timeframes.Minute, finding, cancellationToken).ConfigureAwait(false);
|
||||
result = result.Add(await SecondMinuteReadingAsync(instrument, id, finding, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
else if (finding.Kind == IssueKinds.Gap)
|
||||
{
|
||||
result = result.Add(await RecordMinuteGapAsync(instrument, finding, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private async Task<CollectionResult> RecordMinuteGapAsync(Instrument instrument, QualityFinding gap, CancellationToken cancellationToken)
|
||||
{
|
||||
var id = await _issues.RecordAsync(instrument.Id, Timeframes.Minute, gap, cancellationToken).ConfigureAwait(false);
|
||||
await _issues.AttemptAsync(id, cancellationToken).ConfigureAwait(false);
|
||||
var bars = await _source.FetchMinuteBarsAsync(instrument.Symbol, gap.StartsAt, gap.EndsAt.AddMinutes(1), cancellationToken).ConfigureAwait(false);
|
||||
var found = bars.Where(b => b.OpensAt >= gap.StartsAt && b.OpensAt <= gap.EndsAt && Possible(b)).ToList();
|
||||
await _bars.UpsertMinuteAsync(instrument.Id, found, Venue, BarQuality.Ok, cancellationToken).ConfigureAwait(false);
|
||||
var missing = (int)((gap.EndsAt - gap.StartsAt).TotalMinutes + 1) - found.Count;
|
||||
if (missing <= 0)
|
||||
{
|
||||
await _issues.ResolveAsync(id, IssueStatus.Recovered, $"{gap.Detail}: recuperati {found.Count}", cancellationToken).ConfigureAwait(false);
|
||||
return CollectionResult.None with { Gaps = 1, Recovered = 1, Stored = found.Count };
|
||||
}
|
||||
|
||||
await _issues.ResolveAsync(id, IssueStatus.Unrecoverable,
|
||||
$"{gap.Detail}: la fonte non ha {missing} barre" + (found.Count > 0 ? $" (recuperate {found.Count})" : "") + ", forse minuti senza scambi",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return CollectionResult.None with { Gaps = 1, Unrecoverable = 1, Stored = found.Count };
|
||||
}
|
||||
|
||||
private async Task<CollectionResult> SecondMinuteReadingAsync(Instrument instrument, Guid issue, QualityFinding finding, CancellationToken cancellationToken)
|
||||
{
|
||||
await _issues.AttemptAsync(issue, cancellationToken).ConfigureAwait(false);
|
||||
var at = finding.StartsAt;
|
||||
var stored = FirstOf(await _bars.MinuteAsync(instrument.Id, at, at.AddMinutes(1), cancellationToken).ConfigureAwait(false));
|
||||
var again = (await _source.FetchMinuteBarsAsync(instrument.Symbol, at, at.AddMinutes(1), cancellationToken).ConfigureAwait(false))
|
||||
.FirstOrDefault(b => b.OpensAt == at);
|
||||
return await ConcludeAsync(issue, finding, stored is null ? null : Prices(stored), again is null ? null : Prices(again),
|
||||
q => _bars.UpsertMinuteAsync(instrument.Id, [again!], Venue, q, cancellationToken),
|
||||
() => _bars.DeleteMinuteAsync(instrument.Id, at, cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<CollectionResult> SecondDailyReadingAsync(Instrument instrument, Guid issue, QualityFinding finding, CancellationToken cancellationToken)
|
||||
{
|
||||
await _issues.AttemptAsync(issue, cancellationToken).ConfigureAwait(false);
|
||||
var day = DateOnly.FromDateTime(finding.StartsAt.UtcDateTime);
|
||||
var stored = FirstOf(await _bars.DailyAsync(instrument.Id, day, day, cancellationToken).ConfigureAwait(false));
|
||||
var again = (await _source.FetchDailyBarsAsync(instrument.Symbol, day, day, cancellationToken).ConfigureAwait(false)).FirstOrDefault(b => b.Day == day);
|
||||
return await ConcludeAsync(issue, finding, stored?.Prices, again?.Prices,
|
||||
q => _bars.UpsertDailyAsync(instrument.Id, [again! with { Quality = q }], Venue, cancellationToken),
|
||||
() => _bars.DeleteDailyAsync(instrument.Id, day, cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The outcome of a second reading: impossible or absent again → unrecoverable and the bar is dropped; the same
|
||||
/// prices → confirmed; different prices → the new ones replace the old (corrected).
|
||||
/// </summary>
|
||||
private async Task<CollectionResult> ConcludeAsync(Guid issue, QualityFinding finding, Ohlc? stored, Ohlc? again,
|
||||
Func<string, Task<int>> save, Func<Task> drop, CancellationToken cancellationToken)
|
||||
{
|
||||
if (again is not { } second || !Possible(second))
|
||||
{
|
||||
await drop().ConfigureAwait(false);
|
||||
await _issues.ResolveAsync(issue, IssueStatus.Unrecoverable, finding.Detail + ": la seconda lettura non dà un dato valido, barra scartata",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return CollectionResult.None with { Suspect = 1, Unrecoverable = 1 };
|
||||
}
|
||||
|
||||
if (stored is { } first && Same(first, second))
|
||||
{
|
||||
await save(BarQuality.Confirmed).ConfigureAwait(false);
|
||||
await _issues.ResolveAsync(issue, IssueStatus.Confirmed, finding.Detail + ": confermato dalla seconda lettura", cancellationToken).ConfigureAwait(false);
|
||||
return CollectionResult.None with { Suspect = 1, Confirmed = 1 };
|
||||
}
|
||||
|
||||
await save(BarQuality.Ok).ConfigureAwait(false);
|
||||
await _issues.ResolveAsync(issue, IssueStatus.Corrected,
|
||||
finding.Detail + (stored is { } old ? $": la seconda lettura dà chiusura {second.Close:G6} invece di {old.Close:G6}" : ": la seconda lettura dà una barra valida"),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return CollectionResult.None with { Suspect = 1, Corrected = 1 };
|
||||
}
|
||||
|
||||
private async Task<CollectionResult> RecoverDailyGapAsync(Instrument instrument, Guid issue, QualityFinding gap, CancellationToken cancellationToken)
|
||||
{
|
||||
await _issues.AttemptAsync(issue, cancellationToken).ConfigureAwait(false);
|
||||
var first = DateOnly.FromDateTime(gap.StartsAt.UtcDateTime);
|
||||
var last = DateOnly.FromDateTime(gap.EndsAt.UtcDateTime);
|
||||
var found = (await _source.FetchDailyBarsAsync(instrument.Symbol, first, last, cancellationToken).ConfigureAwait(false))
|
||||
.Where(b => b.Day >= first && b.Day <= last && Possible(b.Prices)).ToList();
|
||||
await _bars.UpsertDailyAsync(instrument.Id, found, Venue, cancellationToken).ConfigureAwait(false);
|
||||
var missing = last.DayNumber - first.DayNumber + 1 - found.Count;
|
||||
await _issues.ResolveAsync(issue, missing <= 0 ? IssueStatus.Recovered : IssueStatus.Unrecoverable,
|
||||
missing <= 0 ? gap.Detail + ": recuperato" : $"{gap.Detail}: la fonte non ha {missing} giorni", cancellationToken).ConfigureAwait(false);
|
||||
return CollectionResult.None with { Gaps = 1, Recovered = missing <= 0 ? 1 : 0, Unrecoverable = missing <= 0 ? 0 : 1, Stored = found.Count };
|
||||
}
|
||||
|
||||
private static T? FirstOf<T>(IReadOnlyList<T> list) where T : class => list.Count > 0 ? list[0] : null;
|
||||
|
||||
private static Ohlc Prices(Bar b) => new(b.Open, b.High, b.Low, b.Close);
|
||||
|
||||
private static bool Possible(Bar b) => Possible(Prices(b));
|
||||
|
||||
private static bool Possible(Ohlc p) => p.Open > 0 && p.High > 0 && p.Low > 0 && p.Close > 0 && p.High >= p.Low;
|
||||
|
||||
private static bool Same(Ohlc a, Ohlc b) => Close(a.Open, b.Open) && Close(a.High, b.High) && Close(a.Low, b.Low) && Close(a.Close, b.Close);
|
||||
|
||||
private static bool Close(double a, double b) => Math.Abs(a - b) <= 1e-9 * Math.Max(Math.Abs(a), Math.Abs(b));
|
||||
|
||||
private static DateTimeOffset Minute(DateTimeOffset t) => new(t.UtcTicks - t.UtcTicks % TimeSpan.TicksPerMinute, TimeSpan.Zero);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Information, Message = "{Venue} {Symbol}: recovered {Count} minute bars")]
|
||||
private static partial void LogRecovered(ILogger logger, string venue, string symbol, int count);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "{Venue} live feed dropped: {Reason}")]
|
||||
private static partial void LogFeedError(ILogger logger, string venue, string reason);
|
||||
}
|
||||
@@ -354,6 +354,21 @@ public static partial class NasdaqTraded
|
||||
[GeneratedRegex(@"\b(ETF|ETN|ETNs|Exchange[- ]Traded|Fund|Funds|Trust Units?|Warrants?|Rights?|Units?|Preferred|Notes?|Debentures?|Depositary Shares? representing a fraction|Closed[- ]End|Income Shares|Beneficial Interest|Index|Leveraged|Inverse|SPAC)\b", RegexOptions.IgnoreCase)]
|
||||
internal static partial Regex NotCommonPattern();
|
||||
|
||||
/// <summary>
|
||||
/// Marks as funds the broker's instruments that this list (or their own name) shows are not common shares. Symbols
|
||||
/// the list does not know keep only the name check.
|
||||
/// </summary>
|
||||
public static IReadOnlyList<InstrumentSpec> MarkFunds(IEnumerable<InstrumentSpec> specs, IReadOnlyList<ListedSecurity> listed)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(specs);
|
||||
ArgumentNullException.ThrowIfNull(listed);
|
||||
var bySymbol = listed.GroupBy(l => l.Symbol, StringComparer.Ordinal).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal);
|
||||
return specs.Select(s => s with
|
||||
{
|
||||
IsFund = s.IsFund == true || (bySymbol.TryGetValue(s.Symbol, out var l) && l.IsExcluded) || NotCommonPattern().IsMatch(s.Name),
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
/// <summary>Rows of the pipe-separated file (CRLF), header and footer skipped.</summary>
|
||||
public static IReadOnlyList<ListedSecurity> Parse(string text)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using Anubis.Core;
|
||||
using Anubis.Db;
|
||||
|
||||
namespace Anubis.Data;
|
||||
|
||||
/// <summary>A member of a universe on a date, with its rank and the measure it was ranked by.</summary>
|
||||
public sealed record UniverseMember(Guid InstrumentId, string Symbol, int Rank, double Measure);
|
||||
|
||||
public static class Universes
|
||||
{
|
||||
public const string Equity = "equity_us";
|
||||
public const string Crypto = "crypto_eur";
|
||||
}
|
||||
|
||||
/// <summary>An instrument with the daily bars known for it; bars after the date are ignored, never used.</summary>
|
||||
public sealed record UniverseCandidate(Instrument Instrument, IReadOnlyList<DailyBar> Bars)
|
||||
{
|
||||
/// <summary>Hourly spread samples of the last 30 days (crypto), in basis points.</summary>
|
||||
public IReadOnlyList<(DateTimeOffset At, double Bp)> Spreads { get; init; } = [];
|
||||
|
||||
/// <summary>The spread right now (crypto), used until 30 days of samples exist.</summary>
|
||||
public double? CurrentSpreadBp { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The stock universe "as of a date" (SPEC 6.7, 8.2): common shares on NYSE and Nasdaq, fractionable on Alpaca,
|
||||
/// raw close above 10 $, at least 300 sessions, the 150 with the highest median traded value over 60 sessions.
|
||||
/// ETFs, ETNs and funds are out. Only bars up to the date count; instruments delisted later are included if they
|
||||
/// traded then. The flags the API gives only as of today (fractionable, fund) apply as they are (API-NOTES).
|
||||
/// </summary>
|
||||
public static class EquityUniverse
|
||||
{
|
||||
public const int Size = 150;
|
||||
public const double MinPrice = 10;
|
||||
public const int MinSessions = 300;
|
||||
public const int Window = 60;
|
||||
|
||||
public static readonly IReadOnlySet<string> Exchanges = new HashSet<string>(StringComparer.Ordinal) { "NYSE", "NASDAQ" };
|
||||
|
||||
public static bool Eligible(InstrumentSpec spec)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(spec);
|
||||
return spec.AssetClass == AssetClasses.Equity && spec.Exchange is { } exchange && Exchanges.Contains(exchange) && spec.Fractionable == true && spec.IsFund != true;
|
||||
}
|
||||
|
||||
public static IReadOnlyList<UniverseMember> Select(DateOnly asOf, IEnumerable<UniverseCandidate> candidates, int size = Size)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(candidates);
|
||||
var ranked = new List<(Instrument Instrument, double Value)>();
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (!Eligible(candidate.Instrument.Spec))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var bars = UpTo(candidate.Bars, asOf);
|
||||
if (bars.Count < MinSessions || bars[^1].Day < asOf.AddDays(-7) || bars[^1].Close <= MinPrice)
|
||||
{
|
||||
continue; // Too young, not trading any more on that date, or too cheap.
|
||||
}
|
||||
|
||||
ranked.Add((candidate.Instrument, Median(bars.TakeLast(Window).Select(b => b.Close * b.Volume))));
|
||||
}
|
||||
|
||||
return Rank(ranked, size);
|
||||
}
|
||||
|
||||
internal static List<DailyBar> UpTo(IReadOnlyList<DailyBar> bars, DateOnly asOf) => bars.Where(b => b.Day <= asOf).OrderBy(b => b.Day).ToList();
|
||||
|
||||
internal static double Median(IEnumerable<double> values)
|
||||
{
|
||||
var sorted = values.Order().ToList();
|
||||
if (sorted.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var middle = sorted.Count / 2;
|
||||
return sorted.Count % 2 == 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
|
||||
}
|
||||
|
||||
internal static IReadOnlyList<UniverseMember> Rank(List<(Instrument Instrument, double Value)> ranked, int size) =>
|
||||
ranked.OrderByDescending(r => r.Value).ThenBy(r => r.Instrument.Symbol, StringComparer.Ordinal).Take(size)
|
||||
.Select((r, i) => new UniverseMember(r.Instrument.Id, r.Instrument.Symbol, i + 1, r.Value)).ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The crypto universe of SPEC 8.1: EUR spot pairs listed for at least 400 days, 30-day median traded value above
|
||||
/// 250,000 €, 30-day median spread below 20 basis points (the current spread until 30 days of samples exist), no
|
||||
/// stablecoins, wrapped or leveraged tokens; the 10 with the highest value. Recomputed on the first day of each month.
|
||||
/// </summary>
|
||||
public static partial class CryptoUniverse
|
||||
{
|
||||
public const int Size = 10;
|
||||
public const int MinAgeDays = 400;
|
||||
public const double MinValueEur = 250_000;
|
||||
public const double MaxSpreadBp = 20;
|
||||
public const int Window = 30;
|
||||
|
||||
/// <summary>Stablecoins pegged to a currency or to gold, as listed by the main venues.</summary>
|
||||
public static readonly IReadOnlySet<string> Stablecoins = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"USDT", "USDC", "DAI", "EURC", "EUROC", "EURT", "EURS", "EURI", "EURQ", "EURR", "TUSD", "FDUSD", "PYUSD", "USDE", "USDS", "BUSD", "USDP", "GUSD",
|
||||
"FRAX", "LUSD", "USDD", "USD1", "RLUSD", "USDG", "USDQ", "USDR", "AEUR", "XAUT", "PAXG",
|
||||
};
|
||||
|
||||
/// <summary>Wrapped or staked receipts of another coin.</summary>
|
||||
public static readonly IReadOnlySet<string> Wrapped = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"WBTC", "WETH", "WBETH", "STETH", "WSTETH", "CBBTC", "CBETH", "RETH", "WEETH", "EETH", "TBTC", "WBNB", "WSOL", "JITOSOL", "MSOL", "BBSOL",
|
||||
"STSOL", "WTRX", "WAVAX", "WMATIC", "WPOL", "WFTM", "WHBAR", "SAVAX", "OKSOL", "BETH", "LBTC",
|
||||
};
|
||||
|
||||
[GeneratedRegex(@"^\w+?(\d+[LS]|UP|DOWN|BULL|BEAR)$", RegexOptions.IgnoreCase)]
|
||||
private static partial Regex LeveragedPattern();
|
||||
|
||||
public static bool Excluded(string baseCurrency) =>
|
||||
Stablecoins.Contains(baseCurrency) || Wrapped.Contains(baseCurrency) || LeveragedPattern().IsMatch(baseCurrency);
|
||||
|
||||
public static IReadOnlyList<UniverseMember> Select(DateOnly asOf, IEnumerable<UniverseCandidate> candidates, int size = Size)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(candidates);
|
||||
var ranked = new List<(Instrument Instrument, double Value)>();
|
||||
var cutoff = QualityChecks.Midnight(asOf.AddDays(1));
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
var spec = candidate.Instrument.Spec;
|
||||
var baseCurrency = spec.BaseCurrency ?? spec.Symbol.Split('-')[0];
|
||||
if (spec.AssetClass != AssetClasses.Crypto || spec.QuoteCurrency != "EUR" || Excluded(baseCurrency))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var bars = EquityUniverse.UpTo(candidate.Bars, asOf);
|
||||
var listedOn = spec.ListedAt is { } listed ? DateOnly.FromDateTime(listed.UtcDateTime) : bars.Count > 0 ? bars[0].Day : asOf;
|
||||
if (bars.Count < Window || listedOn > asOf.AddDays(-MinAgeDays) || bars[^1].Day < asOf.AddDays(-3))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var value = EquityUniverse.Median(bars.TakeLast(Window).Select(b => b.QuoteVolume ?? b.Close * b.Volume));
|
||||
var samples = candidate.Spreads.Where(s => s.At < cutoff && s.At >= cutoff.AddDays(-Window)).ToList();
|
||||
var spread = samples.Count > 0 && samples.Min(s => s.At) <= cutoff.AddDays(-Window + 1)
|
||||
? EquityUniverse.Median(samples.Select(s => s.Bp))
|
||||
: candidate.CurrentSpreadBp;
|
||||
if (value <= MinValueEur || spread is not { } bp || bp >= MaxSpreadBp)
|
||||
{
|
||||
continue; // An unknown spread counts as too wide.
|
||||
}
|
||||
|
||||
ranked.Add((candidate.Instrument, value));
|
||||
}
|
||||
|
||||
return EquityUniverse.Rank(ranked, size);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Universes as they were chosen, and the hourly spread samples the crypto one needs.</summary>
|
||||
public sealed class UniverseStore(Database db, IClock clock)
|
||||
{
|
||||
public async Task SaveAsync(string universe, DateOnly asOf, IReadOnlyList<UniverseMember> members, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(members);
|
||||
var now = clock.UtcNow;
|
||||
await db.WriteAsync(async s =>
|
||||
{
|
||||
await s.ExecuteAsync("DELETE FROM universe_members WHERE universe = @u AND as_of = @d", cancellationToken, ("u", universe), ("d", BarStore.DayText(asOf)))
|
||||
.ConfigureAwait(false);
|
||||
foreach (var m in members)
|
||||
{
|
||||
await s.ExecuteAsync(
|
||||
"INSERT INTO universe_members (id, universe, as_of, instrument_id, rank, created_at) VALUES (@id, @u, @d, @i, @r, @now)", cancellationToken,
|
||||
("id", Ids.NewId(clock)), ("u", universe), ("d", BarStore.DayText(asOf)), ("i", m.InstrumentId), ("r", m.Rank), ("now", now)).ConfigureAwait(false);
|
||||
}
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>The members of the last computation on or before <paramref name="asOf"/>, by rank.</summary>
|
||||
public async Task<IReadOnlyList<UniverseMember>> AsOfAsync(string universe, DateOnly asOf, CancellationToken cancellationToken) =>
|
||||
await db.ReadAsync(s => s.QueryAsync(
|
||||
"SELECT m.instrument_id, i.symbol, m.rank FROM universe_members m JOIN instruments i ON i.id = m.instrument_id " +
|
||||
"WHERE m.universe = @u AND m.as_of = (SELECT MAX(as_of) FROM universe_members WHERE universe = @u AND as_of <= @d) ORDER BY m.rank",
|
||||
r => new UniverseMember(r.Id("instrument_id"), r.Text("symbol"), (int)r.Number("rank"), 0), ("u", universe), ("d", BarStore.DayText(asOf))),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
public Task AddSpreadAsync(Guid instrumentId, DateTimeOffset at, double bp, CancellationToken cancellationToken) =>
|
||||
db.WriteAsync(s => s.ExecuteAsync(
|
||||
"INSERT INTO spread_samples (id, instrument_id, sampled_at, spread_bp, created_at) VALUES (@id, @i, @t, @bp, @now) " +
|
||||
"ON CONFLICT (instrument_id, sampled_at) DO UPDATE SET spread_bp = excluded.spread_bp", cancellationToken,
|
||||
("id", Ids.NewId(clock)), ("i", instrumentId), ("t", at), ("bp", bp), ("now", clock.UtcNow)), cancellationToken);
|
||||
|
||||
public async Task<IReadOnlyList<(DateTimeOffset At, double Bp)>> SpreadsAsync(Guid instrumentId, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken) =>
|
||||
await db.ReadAsync(s => s.QueryAsync(
|
||||
"SELECT sampled_at, spread_bp FROM spread_samples WHERE instrument_id = @i AND sampled_at >= @f AND sampled_at < @t ORDER BY sampled_at",
|
||||
r => (r.Moment("sampled_at"), r.Measure("spread_bp")), ("i", instrumentId), ("f", from), ("t", to)), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
public Task<int> PruneSpreadsAsync(CancellationToken cancellationToken) =>
|
||||
db.WriteAsync(s => s.ExecuteAsync("DELETE FROM spread_samples WHERE sampled_at < @t", cancellationToken, ("t", clock.UtcNow.AddDays(-90))),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Computes the universes from what the database holds and saves them (issue #24).</summary>
|
||||
public sealed class UniverseBuilder(Database db, IClock clock)
|
||||
{
|
||||
private readonly InstrumentCatalog _catalog = new(db, clock);
|
||||
private readonly BarStore _bars = new(db, clock);
|
||||
private readonly UniverseStore _store = new(db, clock);
|
||||
|
||||
public async Task<IReadOnlyList<UniverseMember>> EquityAsync(DateOnly asOf, CancellationToken cancellationToken)
|
||||
{
|
||||
var candidates = new List<UniverseCandidate>();
|
||||
foreach (var instrument in (await _catalog.ListAsync(Venues.Alpaca, cancellationToken).ConfigureAwait(false)).Where(i => EquityUniverse.Eligible(i.Spec)))
|
||||
{
|
||||
var bars = await _bars.DailyAsync(instrument.Id, asOf.AddDays(-(EquityUniverse.MinSessions * 7 / 5) - 30), asOf, cancellationToken).ConfigureAwait(false);
|
||||
candidates.Add(new UniverseCandidate(instrument, bars));
|
||||
}
|
||||
|
||||
var members = EquityUniverse.Select(asOf, candidates);
|
||||
await _store.SaveAsync(Universes.Equity, asOf, members, cancellationToken).ConfigureAwait(false);
|
||||
return members;
|
||||
}
|
||||
|
||||
/// <param name="currentSpread">The spread now, for pairs without 30 days of samples; null when unknown.</param>
|
||||
public async Task<IReadOnlyList<UniverseMember>> CryptoAsync(DateOnly asOf, Func<Instrument, Task<double?>> currentSpread, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(currentSpread);
|
||||
var candidates = new List<UniverseCandidate>();
|
||||
var end = QualityChecks.Midnight(asOf.AddDays(1));
|
||||
foreach (var instrument in (await _catalog.ListAsync(Venues.Okx, cancellationToken).ConfigureAwait(false))
|
||||
.Where(i => i.IsCrypto && i.Spec.QuoteCurrency == "EUR" && !CryptoUniverse.Excluded(i.Spec.BaseCurrency ?? "")))
|
||||
{
|
||||
var bars = await _bars.DailyAsync(instrument.Id, asOf.AddDays(-CryptoUniverse.Window * 2), asOf, cancellationToken).ConfigureAwait(false);
|
||||
var spreads = await _store.SpreadsAsync(instrument.Id, end.AddDays(-CryptoUniverse.Window), end, cancellationToken).ConfigureAwait(false);
|
||||
candidates.Add(new UniverseCandidate(instrument, bars)
|
||||
{
|
||||
Spreads = spreads,
|
||||
CurrentSpreadBp = spreads.Count > 0 && spreads[0].At <= end.AddDays(-CryptoUniverse.Window + 1) ? null : await currentSpread(instrument).ConfigureAwait(false),
|
||||
});
|
||||
}
|
||||
|
||||
var members = CryptoUniverse.Select(asOf, candidates);
|
||||
await _store.SaveAsync(Universes.Crypto, asOf, members, cancellationToken).ConfigureAwait(false);
|
||||
return members;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Corporate actions as they arrive from Alpaca (SPEC 6.10), kept to adjust series and convert levels.</summary>
|
||||
public sealed class CorporateActionStore(Database db, IClock clock)
|
||||
{
|
||||
private const string Columns = "symbol, kind, ex_date, ratio, cash_amount, currency, new_symbol, source, source_key";
|
||||
|
||||
public async Task<int> UpsertAsync(IEnumerable<CorporateAction> actions, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(actions);
|
||||
var list = actions.DistinctBy(a => a.SourceKey).ToList();
|
||||
var now = clock.UtcNow;
|
||||
await db.WriteAsync(async s =>
|
||||
{
|
||||
foreach (var a in list)
|
||||
{
|
||||
var instrument = await s.FirstOrDefaultAsync("SELECT id FROM instruments WHERE venue = @v AND symbol = @s", r => (Guid?)r.Id("id"),
|
||||
("v", Venues.Alpaca), ("s", a.Symbol)).ConfigureAwait(false);
|
||||
await s.ExecuteAsync(
|
||||
$"INSERT INTO corporate_actions (id, instrument_id, {Columns}, created_at) VALUES (@id, @i, @sym, @k, @d, @r, @c, @ccy, @n, @src, @key, @now) " +
|
||||
"ON CONFLICT (source_key) DO UPDATE SET instrument_id = excluded.instrument_id, kind = excluded.kind, ex_date = excluded.ex_date, " +
|
||||
"ratio = excluded.ratio, cash_amount = excluded.cash_amount, currency = excluded.currency, new_symbol = excluded.new_symbol",
|
||||
cancellationToken, ("id", Ids.NewId(clock)), ("i", instrument), ("sym", a.Symbol), ("k", a.Kind), ("d", BarStore.DayText(a.ExDate)),
|
||||
("r", a.Ratio), ("c", a.CashAmount), ("ccy", a.Currency), ("n", a.NewSymbol), ("src", a.Source), ("key", a.SourceKey), ("now", now))
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
return list.Count;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CorporateAction>> ForSymbolAsync(string symbol, CancellationToken cancellationToken) =>
|
||||
await db.ReadAsync(s => s.QueryAsync($"SELECT {Columns} FROM corporate_actions WHERE symbol = @s ORDER BY ex_date", Read, ("s", symbol)),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
/// <summary>Actions effective on <paramref name="day"/>: the positions and levels of these symbols change at the open (Phase 3).</summary>
|
||||
public async Task<IReadOnlyList<CorporateAction>> OnAsync(DateOnly day, CancellationToken cancellationToken) =>
|
||||
await db.ReadAsync(s => s.QueryAsync($"SELECT {Columns} FROM corporate_actions WHERE ex_date = @d ORDER BY symbol", Read, ("d", BarStore.DayText(day))),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
private static CorporateAction Read(DbRow r) => new(r.Text("symbol"), r.Text("kind"), BarStore.ParseDay(r.Text("ex_date")), r.Text("source"), r.Text("source_key"))
|
||||
{
|
||||
Ratio = r.MoneyOrNull("ratio"),
|
||||
CashAmount = r.MoneyOrNull("cash_amount"),
|
||||
Currency = r.TextOrNull("currency"),
|
||||
NewSymbol = r.TextOrNull("new_symbol"),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
using System.Threading.Channels;
|
||||
using Anubis.Core;
|
||||
using Anubis.Data;
|
||||
|
||||
namespace Anubis.Tests;
|
||||
|
||||
/// <summary>The collector of issue #20: backfill, checks, second readings, live feed, drops and gap recovery.</summary>
|
||||
[TestClass]
|
||||
public sealed class CollectorTests
|
||||
{
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Integration)]
|
||||
public async Task FeedDetachedForTenMinutesIsRecoveredOnReconnection()
|
||||
{
|
||||
var clock = new FastClock(new DateTimeOffset(2026, 11, 10, 10, 0, 30, TimeSpan.Zero));
|
||||
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
||||
var bus = new EventBus();
|
||||
using var feed = bus.Subscribe<FeedChanged>();
|
||||
var freshness = new FreshnessMonitor(clock, bus);
|
||||
var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false);
|
||||
var source = new FakeMarketSource(clock);
|
||||
var quiet = new DateTimeOffset(2026, 11, 10, 9, 30, 0, TimeSpan.Zero);
|
||||
source.NoTrades.Add(quiet);
|
||||
var collector = new MarketCollector(source, test.Db, clock, bus, freshness);
|
||||
var bars = new BarStore(test.Db, clock);
|
||||
var issues = new DataIssueStore(test.Db, clock);
|
||||
|
||||
using var stop = new CancellationTokenSource();
|
||||
var live = collector.RunLiveAsync([btc], stop.Token);
|
||||
await OkxTests.WaitForAsync(() => source.Connections == 1).ConfigureAwait(false);
|
||||
|
||||
// Startup: two days of minutes downloaded; the minute without trades is recorded, not invented.
|
||||
var start = clock.UtcNow;
|
||||
Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 9, 59, 0, TimeSpan.Zero), await bars.LastMinuteAsync(btc.Id, CancellationToken.None).ConfigureAwait(false));
|
||||
var startup = await bars.MinuteAsync(btc.Id, start.AddDays(-3), start, CancellationToken.None).ConfigureAwait(false);
|
||||
Assert.AreEqual(2 * 24 * 60 - 1, startup.Count);
|
||||
var unrecoverable = (await issues.SinceAsync(start.AddDays(-1), CancellationToken.None).ConfigureAwait(false)).Single();
|
||||
Assert.AreEqual((IssueKinds.Gap, IssueStatus.Unrecoverable, quiet), (unrecoverable.Kind, unrecoverable.Status, unrecoverable.StartsAt));
|
||||
StringAssert.Contains(unrecoverable.Detail, "senza scambi");
|
||||
|
||||
// Live: a closed bar arrives and the instrument is fresh.
|
||||
clock.Advance(TimeSpan.FromSeconds(31));
|
||||
await source.Live!.Writer.WriteAsync(new BarEvent("BTC-EUR", FakeMarketSource.MinuteBar(new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero)), true))
|
||||
.ConfigureAwait(false);
|
||||
await OkxTests.WaitForAsync(() => freshness.IsFresh(btc.Id)).ConfigureAwait(false);
|
||||
Assert.IsTrue(collector.Connected);
|
||||
|
||||
// The network goes down for ten minutes: the feed drops and nothing answers, not even the REST calls.
|
||||
var dropAt = clock.UtcNow;
|
||||
source.DownUntil = dropAt.AddMinutes(10);
|
||||
source.Live.Writer.Complete();
|
||||
await OkxTests.WaitForAsync(() => source.Connections == 2).ConfigureAwait(false);
|
||||
Assert.IsTrue(clock.UtcNow >= dropAt.AddMinutes(10), "reconnected only after the outage");
|
||||
Assert.IsFalse(freshness.IsFresh(btc.Id), "no live datum since the drop: no new entries");
|
||||
|
||||
// Back: every minute missed is downloaded before the feed resumes.
|
||||
var resumed = new DateTimeOffset(clock.UtcNow.UtcTicks - (clock.UtcNow.UtcTicks % TimeSpan.TicksPerMinute), TimeSpan.Zero);
|
||||
var recovered = await bars.MinuteAsync(btc.Id, new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero), resumed, CancellationToken.None).ConfigureAwait(false);
|
||||
Assert.IsTrue(recovered.Count >= 11, $"at least the ten minutes of the outage, got {recovered.Count}");
|
||||
CollectionAssert.AreEqual(
|
||||
QualityChecks.EveryMinute(new DateTimeOffset(2026, 11, 10, 10, 0, 0, TimeSpan.Zero), resumed).ToList(),
|
||||
recovered.Select(b => b.OpensAt).ToList(), "no minute is missing");
|
||||
Assert.AreEqual(1, (await issues.SinceAsync(start.AddDays(-1), CancellationToken.None).ConfigureAwait(false)).Count, "no new issue");
|
||||
|
||||
await source.Live!.Writer.WriteAsync(new TradeEvent("BTC-EUR", clock.UtcNow, 50_010m, 0.01m)).ConfigureAwait(false);
|
||||
await OkxTests.WaitForAsync(() => freshness.IsFresh(btc.Id)).ConfigureAwait(false);
|
||||
|
||||
await stop.CancelAsync().ConfigureAwait(false);
|
||||
await live.ConfigureAwait(false);
|
||||
var changes = new List<bool>();
|
||||
while (feed.TryRead(out var change))
|
||||
{
|
||||
changes.Add(change!.Connected);
|
||||
}
|
||||
|
||||
CollectionAssert.AreEqual(new[] { true, false, true, false }, changes);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Integration)]
|
||||
public async Task SuspectBarsAreReadAgainAndNeverFixedInSilence()
|
||||
{
|
||||
var day0 = new DateOnly(2026, 6, 1);
|
||||
var clock = new FastClock(new DateTimeOffset(day0.AddDays(120).ToDateTime(new TimeOnly(6, 0)), TimeSpan.Zero));
|
||||
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
||||
var bus = new EventBus();
|
||||
var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false);
|
||||
var walk = MarketDataLogicTests.Walk(120, day0);
|
||||
var source = new FakeMarketSource(clock);
|
||||
foreach (var bar in walk)
|
||||
{
|
||||
source.Days[bar.Day] = bar;
|
||||
}
|
||||
|
||||
var spike = day0.AddDays(80);
|
||||
var broken = day0.AddDays(90);
|
||||
var lost = day0.AddDays(50);
|
||||
source.FirstReadings[spike] = new Queue<DailyBar>([walk[80] with { High = walk[80].High * 3, Close = walk[80].Close * 3 }]);
|
||||
source.Days[broken] = walk[90] with { Low = -1 };
|
||||
source.MissingOnFirstRead.Add(lost);
|
||||
|
||||
var collector = new MarketCollector(source, test.Db, clock, bus, new FreshnessMonitor(clock, bus));
|
||||
var result = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
var bars = new BarStore(test.Db, clock);
|
||||
var stored = await bars.DailyAsync(btc.Id, day0, day0.AddDays(200), CancellationToken.None).ConfigureAwait(false);
|
||||
Assert.AreEqual(119, stored.Count, "the impossible day is dropped, the lost one recovered");
|
||||
Assert.AreEqual(walk[80].Close, stored.Single(b => b.Day == spike).Close, "the second reading replaced the spike");
|
||||
Assert.IsFalse(stored.Any(b => b.Day == broken));
|
||||
Assert.IsTrue(stored.Any(b => b.Day == lost));
|
||||
|
||||
var issues = await new DataIssueStore(test.Db, clock).SinceAsync(DateTimeOffset.MinValue, CancellationToken.None).ConfigureAwait(false);
|
||||
var byDay = issues.ToLookup(i => DateOnly.FromDateTime(i.StartsAt.UtcDateTime));
|
||||
Assert.AreEqual(IssueStatus.Corrected, byDay[spike].Single(i => i.Kind == IssueKinds.Jump).Status);
|
||||
StringAssert.Contains(byDay[spike].Single(i => i.Kind == IssueKinds.Jump).Detail, "invece di");
|
||||
Assert.AreEqual(IssueStatus.Unrecoverable, byDay[broken].Single(i => i.Kind == IssueKinds.NonPositive).Status);
|
||||
Assert.AreEqual(IssueStatus.Recovered, byDay[lost].Single(i => i.Kind == IssueKinds.Gap).Status);
|
||||
Assert.IsTrue(issues.All(i => i.Attempts == 1), "each problem was tried once more");
|
||||
Assert.AreEqual(0, (await new DataIssueStore(test.Db, clock).BlockedInstrumentsAsync(CancellationToken.None).ConfigureAwait(false)).Count,
|
||||
"once read again, nothing blocks new entries");
|
||||
Assert.IsTrue(result.Suspect >= 2 && result.Corrected >= 1 && result.Unrecoverable == 1 && result.Recovered == 1, result.ToString());
|
||||
|
||||
// A suspect bar the second reading confirms stays, marked as confirmed.
|
||||
source.Days[day0.AddDays(120)] = walk[^1] with { Day = day0.AddDays(120), Open = walk[^1].Close, Low = walk[^1].Close * 0.5, Close = walk[^1].Close * 0.5 };
|
||||
clock.Advance(TimeSpan.FromDays(1));
|
||||
var next = await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false);
|
||||
Assert.AreEqual(1, next.Confirmed);
|
||||
Assert.AreEqual(BarQuality.Confirmed, (await bars.DailyAsync(btc.Id, day0.AddDays(120), day0.AddDays(120), CancellationToken.None).ConfigureAwait(false))[0].Quality);
|
||||
Assert.AreEqual(CollectionResult.None, await collector.BackfillDailyAsync(btc, day0, CancellationToken.None).ConfigureAwait(false), "nothing new");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A venue for the collector tests: minute bars exist for every past minute except <see cref="NoTrades"/>, daily
|
||||
/// bars come from <see cref="Days"/>, and the whole venue can be down until <see cref="DownUntil"/>.
|
||||
/// </summary>
|
||||
internal sealed class FakeMarketSource(IClock clock) : IMarketDataSource
|
||||
{
|
||||
public string Venue => Venues.Okx;
|
||||
|
||||
public HashSet<DateTimeOffset> NoTrades { get; } = [];
|
||||
|
||||
public Dictionary<DateOnly, DailyBar> Days { get; } = [];
|
||||
|
||||
/// <summary>What the first reading of a day returns instead of <see cref="Days"/>.</summary>
|
||||
public Dictionary<DateOnly, Queue<DailyBar>> FirstReadings { get; } = [];
|
||||
|
||||
public HashSet<DateOnly> MissingOnFirstRead { get; } = [];
|
||||
|
||||
public DateTimeOffset? DownUntil { get; set; }
|
||||
|
||||
public Channel<MarketEvent>? Live { get; private set; }
|
||||
|
||||
public int Connections;
|
||||
|
||||
public static Bar MinuteBar(DateTimeOffset t)
|
||||
{
|
||||
var price = 50_000 + (20 * Math.Sin(t.ToUnixTimeSeconds() / 60.0 / 15));
|
||||
return new Bar(t, price, price + 5, price - 5, price + 1, 0.5);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<InstrumentSpec>> ListInstrumentsAsync(CancellationToken cancellationToken) =>
|
||||
Task.FromResult<IReadOnlyList<InstrumentSpec>>([MarketDataStoreTests.Okx("BTC-EUR")]);
|
||||
|
||||
public Task<IReadOnlyList<DailyBar>> FetchDailyBarsAsync(string symbol, DateOnly first, DateOnly last, CancellationToken cancellationToken)
|
||||
{
|
||||
ThrowIfDown();
|
||||
var list = new List<DailyBar>();
|
||||
for (var day = first; day <= last; day = day.AddDays(1))
|
||||
{
|
||||
if (FirstReadings.TryGetValue(day, out var queue) && queue.Count > 0)
|
||||
{
|
||||
list.Add(queue.Dequeue());
|
||||
}
|
||||
else if (!MissingOnFirstRead.Remove(day) && Days.TryGetValue(day, out var bar))
|
||||
{
|
||||
list.Add(bar);
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult<IReadOnlyList<DailyBar>>(list);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<Bar>> FetchMinuteBarsAsync(string symbol, DateTimeOffset start, DateTimeOffset until, CancellationToken cancellationToken)
|
||||
{
|
||||
ThrowIfDown();
|
||||
var now = clock.UtcNow;
|
||||
var list = new List<Bar>();
|
||||
for (var t = start; t < until && t.AddMinutes(1) <= now; t = t.AddMinutes(1))
|
||||
{
|
||||
if (!NoTrades.Contains(t))
|
||||
{
|
||||
list.Add(MinuteBar(t));
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult<IReadOnlyList<Bar>>(list);
|
||||
}
|
||||
|
||||
public Task<Quote> FetchQuoteAsync(string symbol, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new Quote(symbol, 49_999m, 50_001m, clock.UtcNow));
|
||||
|
||||
public async IAsyncEnumerable<MarketEvent> StreamAsync(IReadOnlyCollection<string> symbols,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
ThrowIfDown();
|
||||
Live = Channel.CreateUnbounded<MarketEvent>();
|
||||
Interlocked.Increment(ref Connections);
|
||||
await foreach (var item in Live.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return item;
|
||||
}
|
||||
}
|
||||
|
||||
private void ThrowIfDown()
|
||||
{
|
||||
if (DownUntil is { } until && clock.UtcNow < until)
|
||||
{
|
||||
throw new MarketDataException("rete assente", retryable: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
using Anubis.Core;
|
||||
using Anubis.Data;
|
||||
|
||||
namespace Anubis.Tests;
|
||||
|
||||
/// <summary>Universes "as of a date" (issue #24) and the corporate action store (issue #23).</summary>
|
||||
[TestClass]
|
||||
public sealed class UniverseTests
|
||||
{
|
||||
private static readonly DateOnly[] _pastDates = [new(2019, 12, 31), new(2021, 6, 30), new(2024, 3, 28)];
|
||||
|
||||
private static readonly DateTimeOffset _now = new(2026, 11, 10, 8, 0, 0, TimeSpan.Zero);
|
||||
|
||||
/// <summary>
|
||||
/// The securities of the recorded nasdaqtraded.txt plus 40 common shares, a stock delisted in 2020, a cheap one
|
||||
/// and one listed in 2021. Funds and ETFs trade the most, so they would lead the ranking if they got through.
|
||||
/// </summary>
|
||||
private static List<UniverseCandidate> Market()
|
||||
{
|
||||
var listed = NasdaqTraded.Parse(Fixture.Text("free/nasdaqtraded.txt"));
|
||||
var specs = listed.Select(l => Stock(l.Symbol, l.Name, l.Exchange)).ToList();
|
||||
specs.AddRange(Enumerable.Range(1, 40).Select(i => Stock($"S{i:D3}", $"Synthetic {i} Inc. Common Stock", i % 2 == 0 ? "NYSE" : "NASDAQ")));
|
||||
specs.Add(Stock("DEAD", "Gone Corp. Common Stock", "NYSE"));
|
||||
specs.Add(Stock("CHEAP", "Penny Corp. Common Stock", "NASDAQ"));
|
||||
specs.Add(Stock("YOUNG", "New Listing Inc. Common Stock", "NASDAQ"));
|
||||
specs.Add(Stock("NOFRAC", "Whole Shares Only Inc.", "NYSE") with { Fractionable = false });
|
||||
var marked = NasdaqTraded.MarkFunds(specs, listed);
|
||||
|
||||
var candidates = new List<UniverseCandidate>();
|
||||
foreach (var spec in marked)
|
||||
{
|
||||
var symbol = spec.Symbol;
|
||||
var first = symbol == "YOUNG" ? new DateOnly(2021, 1, 4) : new DateOnly(2018, 1, 2);
|
||||
var last = symbol == "DEAD" ? new DateOnly(2020, 6, 30) : new DateOnly(2026, 11, 9);
|
||||
var price = symbol == "CHEAP" ? 5.0 : 20.0 + (symbol.GetHashCode(StringComparison.Ordinal) & 0x3f);
|
||||
var volume = spec.IsFund == true || symbol.StartsWith("ZX", StringComparison.Ordinal) ? 50_000_000.0 : 100_000.0 * (1 + (symbol.Length * 7 % 13));
|
||||
if (symbol is "DEAD" or "YOUNG")
|
||||
{
|
||||
volume = 10_000_000; // Large enough to rank first whenever they are eligible.
|
||||
}
|
||||
|
||||
candidates.Add(new UniverseCandidate(new Instrument(Guid.NewGuid(), spec, null), Sessions(first, last, price, volume)));
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
private static InstrumentSpec Stock(string symbol, string name, string exchange) =>
|
||||
new(Venues.Alpaca, symbol, name, AssetClasses.Equity, "USD") { Exchange = exchange, Fractionable = true, Status = "active" };
|
||||
|
||||
private static List<DailyBar> Sessions(DateOnly first, DateOnly last, double price, double volume)
|
||||
{
|
||||
var bars = new List<DailyBar>();
|
||||
for (var day = first; day <= last; day = day.AddDays(1))
|
||||
{
|
||||
if (day.DayOfWeek is not (DayOfWeek.Saturday or DayOfWeek.Sunday))
|
||||
{
|
||||
bars.Add(new DailyBar(day, price, price * 1.01, price * 0.99, price, volume));
|
||||
}
|
||||
}
|
||||
|
||||
return bars;
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void EquityUniverseAtThreePastDatesHasNoEtfsNorFunds()
|
||||
{
|
||||
var market = Market();
|
||||
var excluded = new HashSet<string> { "SPY", "BULZ", "AMUB", "PDI", "ABR$D", "AACIW", "ZXIET", "CHEAP", "NOFRAC" };
|
||||
foreach (var date in _pastDates)
|
||||
{
|
||||
var members = EquityUniverse.Select(date, market);
|
||||
var symbols = members.Select(m => m.Symbol).ToList();
|
||||
Assert.IsFalse(symbols.Any(excluded.Contains), $"{date}: {string.Join(", ", symbols.Where(excluded.Contains))}");
|
||||
Assert.IsFalse(symbols.Contains("CECO"), "Texas Stock Exchange is neither NYSE nor Nasdaq");
|
||||
CollectionAssert.IsSubsetOf(new[] { "AAPL", "KO", "AACG", "S001", "S040" }, symbols, BarStore.DayText(date));
|
||||
CollectionAssert.AreEqual(Enumerable.Range(1, members.Count).ToList(), members.Select(m => m.Rank).ToList());
|
||||
}
|
||||
|
||||
// Survivorship: the stock delisted in 2020 belongs to the 2019 universe and to no later one.
|
||||
Assert.AreEqual(1, EquityUniverse.Select(_pastDates[0], market)[0].Rank);
|
||||
Assert.AreEqual("DEAD", EquityUniverse.Select(_pastDates[0], market)[0].Symbol);
|
||||
Assert.IsFalse(EquityUniverse.Select(_pastDates[1], market).Any(m => m.Symbol == "DEAD"));
|
||||
|
||||
// History: a stock listed in January 2021 has fewer than 300 sessions in June 2021, enough in 2024.
|
||||
Assert.IsFalse(EquityUniverse.Select(_pastDates[1], market).Any(m => m.Symbol == "YOUNG"));
|
||||
Assert.AreEqual("YOUNG", EquityUniverse.Select(_pastDates[2], market)[0].Symbol);
|
||||
|
||||
// The size cut keeps the most traded.
|
||||
var top = EquityUniverse.Select(_pastDates[2], market, size: 5);
|
||||
Assert.AreEqual(5, top.Count);
|
||||
Assert.IsTrue(top.Zip(top.Skip(1)).All(p => p.First.Measure >= p.Second.Measure));
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void UniverseOnADateIgnoresEverythingAfterIt()
|
||||
{
|
||||
var market = Market();
|
||||
var date = _pastDates[1];
|
||||
var truncated = market.Select(c => c with { Bars = c.Bars.Where(b => b.Day <= date).ToList() }).ToList();
|
||||
|
||||
// A later crash of AAPL to 1 $ and a later volume explosion of S002 must not change the June 2021 universe.
|
||||
var future = market.Select(c => c.Instrument.Symbol switch
|
||||
{
|
||||
"AAPL" => c with { Bars = [.. c.Bars.Select(b => b.Day > date ? b with { Close = 1, Low = 0.9, Open = 1, High = 1.1 } : b)] },
|
||||
"S002" => c with { Bars = [.. c.Bars.Select(b => b.Day > date ? b with { Volume = 1e12 } : b)] },
|
||||
_ => c,
|
||||
}).ToList();
|
||||
|
||||
var expected = EquityUniverse.Select(date, truncated);
|
||||
CollectionAssert.AreEqual(expected.ToList(), EquityUniverse.Select(date, market).ToList());
|
||||
CollectionAssert.AreEqual(expected.ToList(), EquityUniverse.Select(date, future).ToList());
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void CryptoUniverseFiltersAgeValueSpreadAndExcludedTokens()
|
||||
{
|
||||
var asOf = new DateOnly(2026, 11, 1);
|
||||
var end = QualityChecks.Midnight(asOf.AddDays(1));
|
||||
UniverseCandidate Pair(string coin, double value, double? current, int ageDays = 900, double? sampled = null) =>
|
||||
new(new Instrument(Guid.NewGuid(), new InstrumentSpec(Venues.Okx, coin + "-EUR", coin + "/EUR", AssetClasses.Crypto, "EUR")
|
||||
{
|
||||
BaseCurrency = coin,
|
||||
ListedAt = end.AddDays(-ageDays),
|
||||
}, null), Sessions(asOf.AddDays(-60), asOf, 100, value / 100).Concat([]).ToList())
|
||||
{
|
||||
CurrentSpreadBp = current,
|
||||
Spreads = sampled is { } bp ? [.. Enumerable.Range(1, 30 * 24).Select(h => (end.AddHours(-h), bp))] : [],
|
||||
};
|
||||
|
||||
// Weekends are missing from Sessions: give crypto every day.
|
||||
UniverseCandidate Daily(UniverseCandidate c) => c with
|
||||
{
|
||||
Bars = [.. QualityChecks.EveryDay(asOf.AddDays(-60), asOf).Select(d => c.Bars[0] with { Day = d })],
|
||||
};
|
||||
|
||||
var candidates = new[]
|
||||
{
|
||||
Pair("BTC", 50_000_000, 2), Pair("ETH", 20_000_000, 3), Pair("SOL", 5_000_000, null, sampled: 8),
|
||||
Pair("XRP", 3_000_000, 50, sampled: 10), // 30 days of samples win over a wide spread right now.
|
||||
Pair("ADA", 2_000_000, 25), Pair("DOT", 200_000, 5), Pair("NEW", 9_000_000, 5, ageDays: 200),
|
||||
Pair("USDC", 90_000_000, 1), Pair("WBTC", 9_000_000, 5), Pair("BTC3L", 9_000_000, 5), Pair("LINK", 1_000_000, null),
|
||||
}.Select(Daily).ToList();
|
||||
var members = CryptoUniverse.Select(asOf, candidates);
|
||||
CollectionAssert.AreEqual(new[] { "BTC-EUR", "ETH-EUR", "SOL-EUR", "XRP-EUR" }, members.Select(m => m.Symbol).ToList(),
|
||||
"ADA spread too wide, DOT too small, NEW too young, stable/wrapped/leveraged out, LINK spread unknown");
|
||||
Assert.AreEqual(2, CryptoUniverse.Select(asOf, candidates, size: 2).Count);
|
||||
Assert.IsTrue(CryptoUniverse.Excluded("ETHUP") && CryptoUniverse.Excluded("eurc") && !CryptoUniverse.Excluded("UNI"));
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Integration)]
|
||||
public async Task UniversesAreComputedFromTheDatabaseAndKeptByDate()
|
||||
{
|
||||
var clock = new ManualClock(_now);
|
||||
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
||||
var catalog = new InstrumentCatalog(test.Db, clock);
|
||||
var bars = new BarStore(test.Db, clock);
|
||||
var listed = NasdaqTraded.Parse(Fixture.Text("free/nasdaqtraded.txt"));
|
||||
var specs = NasdaqTraded.MarkFunds(
|
||||
[Stock("AAPL", "Apple Inc. - Common Stock", "NASDAQ"), Stock("SPY", "SPDR S&P 500 ETF Trust", "ARCA") with { Exchange = "NYSE" },
|
||||
Stock("PDI", "PIMCO Dynamic Income Fund", "NYSE"), Stock("KO", "Coca-Cola Company (The) Common Stock", "NYSE")], listed);
|
||||
await catalog.SyncAsync(Venues.Alpaca, specs, complete: true, CancellationToken.None).ConfigureAwait(false);
|
||||
foreach (var instrument in await catalog.ListAsync(Venues.Alpaca, CancellationToken.None).ConfigureAwait(false))
|
||||
{
|
||||
var volume = instrument.Symbol == "SPY" ? 1e9 : instrument.Symbol == "AAPL" ? 2e6 : 1e6;
|
||||
await bars.UpsertDailyAsync(instrument.Id, Sessions(new DateOnly(2024, 1, 2), new DateOnly(2026, 11, 6), 50, volume), Venues.Alpaca, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var builder = new UniverseBuilder(test.Db, clock);
|
||||
var equity = await builder.EquityAsync(new DateOnly(2026, 11, 6), CancellationToken.None).ConfigureAwait(false);
|
||||
CollectionAssert.AreEqual(new[] { "AAPL", "KO" }, equity.Select(m => m.Symbol).ToList(), "SPY is an ETF even when listed on NYSE; PDI a fund");
|
||||
await builder.EquityAsync(new DateOnly(2025, 3, 3), CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
var store = new UniverseStore(test.Db, clock);
|
||||
Assert.AreEqual(2, (await store.AsOfAsync(Universes.Equity, new DateOnly(2026, 11, 9), CancellationToken.None).ConfigureAwait(false)).Count);
|
||||
Assert.AreEqual(2, (await store.AsOfAsync(Universes.Equity, new DateOnly(2025, 6, 1), CancellationToken.None).ConfigureAwait(false)).Count);
|
||||
Assert.AreEqual(0, (await store.AsOfAsync(Universes.Equity, new DateOnly(2025, 1, 1), CancellationToken.None).ConfigureAwait(false)).Count,
|
||||
"before the first computation there is no universe");
|
||||
|
||||
// Crypto: spread samples come from the hourly job; without 30 days of them the current spread decides.
|
||||
var btc = await catalog.EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR") with { ListedAt = _now.AddYears(-3) }, CancellationToken.None).ConfigureAwait(false);
|
||||
var eth = await catalog.EnsureAsync(MarketDataStoreTests.Okx("ETH-EUR") with { ListedAt = _now.AddYears(-3) }, CancellationToken.None).ConfigureAwait(false);
|
||||
foreach (var pair in new[] { btc, eth })
|
||||
{
|
||||
await bars.UpsertDailyAsync(pair.Id, [.. QualityChecks.EveryDay(new DateOnly(2026, 9, 1), new DateOnly(2026, 11, 9))
|
||||
.Select(d => new DailyBar(d, 60_000, 61_000, 59_000, 60_000, 100) { QuoteVolume = 6_000_000 })], Venues.Okx, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
for (var h = 1; h <= 30 * 24; h++)
|
||||
{
|
||||
await store.AddSpreadAsync(eth.Id, QualityChecks.Midnight(new DateOnly(2026, 11, 10)).AddHours(-h), 30, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var asked = new List<string>();
|
||||
var crypto = await builder.CryptoAsync(new DateOnly(2026, 11, 9), i =>
|
||||
{
|
||||
asked.Add(i.Symbol);
|
||||
return Task.FromResult<double?>(1.5);
|
||||
}, CancellationToken.None).ConfigureAwait(false);
|
||||
CollectionAssert.AreEqual(new[] { "BTC-EUR" }, crypto.Select(m => m.Symbol).ToList(), "ETH's 30-day median spread is 30 bp");
|
||||
CollectionAssert.AreEqual(new[] { "BTC-EUR" }, asked, "the current spread is asked only where samples are missing");
|
||||
Assert.AreEqual(0, await store.PruneSpreadsAsync(CancellationToken.None).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Integration)]
|
||||
public async Task CorporateActionsAreKeptOnceAndFoundByDay()
|
||||
{
|
||||
var clock = new ManualClock(_now);
|
||||
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
||||
var aapl = await new InstrumentCatalog(test.Db, clock).EnsureAsync(Stock("AAPL", "Apple Inc.", "NASDAQ"), CancellationToken.None).ConfigureAwait(false);
|
||||
var store = new CorporateActionStore(test.Db, clock);
|
||||
var split = new CorporateAction("AAPL", CorporateActionKinds.Split, new DateOnly(2020, 8, 31), Venues.Alpaca, "alpaca:split:1") { Ratio = 4m };
|
||||
var dividend = new CorporateAction("AAPL", CorporateActionKinds.CashDividend, new DateOnly(2026, 11, 10), Venues.Alpaca, "alpaca:div:2")
|
||||
{
|
||||
CashAmount = 0.26m,
|
||||
Currency = "USD",
|
||||
};
|
||||
var other = new CorporateAction("ZZZ", CorporateActionKinds.SymbolChange, new DateOnly(2026, 11, 10), Venues.Alpaca, "alpaca:sc:3") { NewSymbol = "ZZY" };
|
||||
|
||||
Assert.AreEqual(3, await store.UpsertAsync([split, dividend, other, split], CancellationToken.None).ConfigureAwait(false));
|
||||
await store.UpsertAsync([dividend with { CashAmount = 0.27m }], CancellationToken.None).ConfigureAwait(false);
|
||||
var forApple = await store.ForSymbolAsync("AAPL", CancellationToken.None).ConfigureAwait(false);
|
||||
CollectionAssert.AreEqual(new[] { split, dividend with { CashAmount = 0.27m } }, forApple.ToList());
|
||||
var today = await store.OnAsync(new DateOnly(2026, 11, 10), CancellationToken.None).ConfigureAwait(false);
|
||||
CollectionAssert.AreEqual(new[] { "AAPL", "ZZZ" }, today.Select(a => a.Symbol).ToList());
|
||||
|
||||
var linked = await test.Db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM corporate_actions WHERE instrument_id = @i", ("i", aapl.Id)), CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
Assert.AreEqual(2L, linked, "actions of a known symbol point to its instrument");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user