Add free reference sources and research importers (refs #21 #22)

Readers for the ECB rates, the Forex Factory calendar, FRED release dates,
Finnhub earnings, the Fed/ECB meeting list, fear and greed, the Cboe VIX
history, nasdaqtraded.txt and the Binance kline archives, each tested on
recorded responses. ReferenceSources downloads them within their limits
and records every attempt in source_state, so limits hold across restarts
and the interface can say which source is off and why (FRED and Finnhub
without a key). The central-bank list is an editable, validated setting.

BinanceImporter checks every monthly zip against its SHA-256 file before
writing it to the research history; CsvImporter reads user files with a
configurable column mapping and reports unreadable lines and gaps instead
of fixing them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alby96andClaude Opus 5.5 committed 2026-11-10 15:20:35 +01:00
1 parent 956306a8fa
commit b3fb69e696
7 files changed
+1666 -3

No files matched your search

+15 -3
View File
@@ -3,8 +3,11 @@ using Anubis.Db;
namespace Anubis.Core;
/// <summary>A setting the interface may change: key, default and allowed values.</summary>
public sealed record SettingDefinition(string Key, string Label, string Default, IReadOnlyList<string>? Allowed = null);
/// <summary>
/// A setting the interface may change: key, default and allowed values. <see cref="Check"/>, when present, returns
/// the reason a free-text value is refused, in the words the interface shows, or null when it is valid.
/// </summary>
public sealed record SettingDefinition(string Key, string Label, string Default, IReadOnlyList<string>? Allowed = null, Func<string, string?>? Check = null);
/// <summary>A value fixed by the startup configuration: it wins and is shown read-only with where to change it (SPEC 13.5).</summary>
public sealed record StartupOverride(string Value, string Source);
@@ -34,7 +37,11 @@ public sealed class SettingsStore(Database db, IClock clock, EventBus bus, IRead
/// <summary>Bot state behind the top bar buttons: active, paused (no new entries) or stopped (kill switch, SPEC 9.3).</summary>
public static readonly SettingDefinition BotState = new("bot.state", "Stato del bot", "active", ["active", "paused", "stopped"]);
public static readonly IReadOnlyList<SettingDefinition> All = [Theme, Currency, BotState];
/// <summary>Fed and ECB decision days, one per line (SPEC 4.4: a yearly list the user edits).</summary>
public static readonly SettingDefinition CentralBankMeetings = new("calendar.central_banks", "Riunioni di Fed e BCE",
Data.CentralBanks.DefaultList, Check: Data.CentralBanks.Check);
public static readonly IReadOnlyList<SettingDefinition> All = [Theme, Currency, BotState, CentralBankMeetings];
private readonly IReadOnlyDictionary<string, StartupOverride> _overrides = overrides ?? new Dictionary<string, StartupOverride>();
@@ -68,6 +75,11 @@ public sealed class SettingsStore(Database db, IClock clock, EventBus bus, IRead
throw new ArgumentException($"Valore non ammesso per {definition.Label}: {value}.", nameof(value));
}
if (definition.Check?.Invoke(value) is { } problem)
{
throw new ArgumentException(problem, nameof(value));
}
var now = clock.UtcNow;
var json = new DbJson(JsonSerializer.Serialize(value));
var changed = await db.WriteAsync(async s =>
+467
View File
@@ -0,0 +1,467 @@
using System.Globalization;
using System.IO.Compression;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Xml.Linq;
namespace Anubis.Data;
/// <summary>
/// Readers of the free sources of SPEC 4.4. Each is a pure function from the published text to Anubis records, tested
/// on recorded responses; <see cref="ReferenceSources"/> downloads them within their limits.
/// </summary>
public static class Ecb
{
public const string Source = "ecb";
public static readonly Uri Daily = new("https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml");
public static readonly Uri Last90Days = new("https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml");
public static readonly Uri History = new("https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.zip");
private static readonly XNamespace _ns = "http://www.ecb.int/vocabulary/2002-08-01/eurofxref";
/// <summary>Rates of the daily or 90-day XML for the <paramref name="currencies"/> asked, EUR as base.</summary>
public static IReadOnlyList<FxRate> ParseXml(string xml, IReadOnlyCollection<string> currencies)
{
var document = XDocument.Parse(xml);
return document.Descendants(_ns + "Cube")
.Where(day => day.Attribute("time") is not null)
.SelectMany(day => day.Elements(_ns + "Cube").Select(rate => (Day: day, Rate: rate)))
.Where(x => currencies.Contains((string?)x.Rate.Attribute("currency") ?? ""))
.Select(x => new FxRate("EUR", (string)x.Rate.Attribute("currency")!,
DateOnly.ParseExact((string)x.Day.Attribute("time")!, "yyyy-MM-dd", CultureInfo.InvariantCulture),
decimal.Parse((string)x.Rate.Attribute("rate")!, NumberStyles.Float, CultureInfo.InvariantCulture), Source))
.OrderBy(r => r.Date)
.ToList();
}
/// <summary>The full history CSV (inside eurofxref-hist.zip): newest first, a trailing comma on every line, N/A where missing.</summary>
public static IReadOnlyList<FxRate> ParseHistoryCsv(string csv, IReadOnlyCollection<string> currencies)
{
ArgumentNullException.ThrowIfNull(csv);
var lines = csv.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var header = lines[0].Split(',');
var rates = new List<FxRate>();
foreach (var line in lines.Skip(1))
{
var fields = line.Split(',');
var date = DateOnly.ParseExact(fields[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
for (var i = 1; i < Math.Min(header.Length, fields.Length); i++)
{
if (currencies.Contains(header[i]) && decimal.TryParse(fields[i], NumberStyles.Float, CultureInfo.InvariantCulture, out var rate))
{
rates.Add(new FxRate("EUR", header[i], date, rate, Source));
}
}
}
return rates.OrderBy(r => r.Date).ToList();
}
public static string ExtractHistoryCsv(byte[] zip)
{
using var archive = new ZipArchive(new MemoryStream(zip), ZipArchiveMode.Read);
var entry = archive.Entries.Single(e => e.Name.EndsWith(".csv", StringComparison.OrdinalIgnoreCase));
using var reader = new StreamReader(entry.Open());
return reader.ReadToEnd();
}
}
/// <summary>The weekly economic calendar of Forex Factory (faireconomy.media). It has no actual values.</summary>
public static class ForexFactory
{
public const string Source = "forexfactory";
public static readonly Uri ThisWeek = new("https://nfs.faireconomy.media/ff_calendar_thisweek.json");
public static IReadOnlyList<CalendarEvent> Parse(string json)
{
using var document = JsonDocument.Parse(json);
var events = new List<CalendarEvent>();
foreach (var item in document.RootElement.EnumerateArray())
{
var title = Text(item, "title");
var country = Text(item, "country");
var at = DateTimeOffset.Parse(Text(item, "date"), CultureInfo.InvariantCulture).ToUniversalTime();
var impact = Text(item, "impact");
var holiday = impact == "Holiday";
events.Add(new CalendarEvent(holiday ? CalendarEventKinds.Holiday : CalendarEventKinds.Economic, title, at, TimeKnown: !holiday, Source,
$"ff:{country}:{at:yyyy-MM-ddTHH:mm}:{title}")
{
Currency = country == "All" ? null : country,
Impact = holiday ? null : impact,
Forecast = Empty(Text(item, "forecast")),
Previous = Empty(Text(item, "previous")),
});
}
return events;
}
private static string? Empty(string text) => text.Length == 0 ? null : text;
internal static string Text(JsonElement item, string name) =>
item.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString()! : "";
}
/// <summary>
/// Future release dates of the main US data from FRED (optional key). FRED gives only the day: the time is
/// 08:30 New York and the impact high (SPEC 4.4).
/// </summary>
public static class Fred
{
public const string Source = "fred";
/// <summary>Releases followed, with the title the calendar shows (ids checked on fred.stlouisfed.org).</summary>
public static readonly IReadOnlyList<(int Id, string Title)> Releases =
[
(10, "CPI (inflazione USA)"),
(50, "Employment Situation (occupazione USA)"),
(53, "PIL USA"),
(54, "Personal Income and Outlays (PCE)"),
];
public static Uri ReleaseDates(int releaseId, string apiKey, DateOnly from) => new(
"https://api.stlouisfed.org/fred/release/dates?release_id=" + releaseId.ToString(CultureInfo.InvariantCulture) +
"&api_key=" + Uri.EscapeDataString(apiKey) + "&file_type=json&include_release_dates_with_no_data=true&sort_order=asc" +
"&realtime_start=" + from.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
public static IReadOnlyList<CalendarEvent> Parse(string json, string title, DateOnly from)
{
using var document = JsonDocument.Parse(json);
var events = new List<CalendarEvent>();
foreach (var item in document.RootElement.GetProperty("release_dates").EnumerateArray())
{
var day = DateOnly.ParseExact(ForexFactory.Text(item, "date"), "yyyy-MM-dd", CultureInfo.InvariantCulture);
if (day < from)
{
continue;
}
var id = item.GetProperty("release_id").GetInt32();
events.Add(new CalendarEvent(CalendarEventKinds.Economic, title, Zones.At(day, new TimeOnly(8, 30), Zones.NewYork), TimeKnown: true, Source,
$"fred:{id}:{day:yyyy-MM-dd}")
{
Currency = "USD",
Impact = "High",
});
}
return events;
}
}
/// <summary>Earnings dates from Finnhub (optional key, free plan: 60 calls a minute).</summary>
public static class Finnhub
{
public const string Source = "finnhub";
public static Uri Earnings(DateOnly from, DateOnly to, string apiKey) => new(
$"https://finnhub.io/api/v1/calendar/earnings?from={from:yyyy-MM-dd}&to={to:yyyy-MM-dd}&token={Uri.EscapeDataString(apiKey)}");
/// <summary>
/// <c>hour</c>: bmo before the open (08:00 New York), amc after the close (16:05), dmh during market hours
/// (12:00); empty means unknown and the event keeps only its day.
/// </summary>
public static IReadOnlyList<CalendarEvent> Parse(string json, IReadOnlySet<string>? symbols = null)
{
using var document = JsonDocument.Parse(json);
var events = new List<CalendarEvent>();
foreach (var item in document.RootElement.GetProperty("earningsCalendar").EnumerateArray())
{
var symbol = ForexFactory.Text(item, "symbol");
if (symbols is not null && !symbols.Contains(symbol))
{
continue;
}
var day = DateOnly.ParseExact(ForexFactory.Text(item, "date"), "yyyy-MM-dd", CultureInfo.InvariantCulture);
var hour = ForexFactory.Text(item, "hour");
TimeOnly? time = hour switch
{
"bmo" => new TimeOnly(8, 0),
"amc" => new TimeOnly(16, 5),
"dmh" => new TimeOnly(12, 0),
_ => null,
};
events.Add(new CalendarEvent(CalendarEventKinds.Earnings, "Trimestrale " + symbol, Zones.At(day, time ?? TimeOnly.MinValue, Zones.NewYork),
time is not null, Source, $"finnhub:{symbol}:{day:yyyy-MM-dd}")
{
Symbol = symbol,
Currency = "USD",
TimeHint = hour.Length == 0 ? null : hour,
Impact = "High",
});
}
return events;
}
}
/// <summary>
/// Meetings of the Fed and the ECB from the yearly list in the settings (SPEC 4.4): one per line,
/// <c>FED 2026-12-09</c> or <c>BCE 2026-12-17</c>. Fed decisions come at 14:00 New York, ECB ones at 14:15 Frankfurt.
/// </summary>
public static class CentralBanks
{
public const string Source = "central_banks";
/// <summary>Decision days from the official calendars, read on 10/11/2026 (API-NOTES).</summary>
public const string DefaultList =
"FED 2026-12-09\nFED 2027-01-27\nFED 2027-03-17\nFED 2027-04-28\nFED 2027-06-09\nFED 2027-07-28\nFED 2027-09-15\nFED 2027-10-27\nFED 2027-12-08\n" +
"BCE 2026-12-17\nBCE 2027-02-04\nBCE 2027-03-18\nBCE 2027-04-29\nBCE 2027-06-10\nBCE 2027-07-22\nBCE 2027-09-09\nBCE 2027-10-28\nBCE 2027-12-16";
/// <summary>Null when the list is valid, otherwise what is wrong with it.</summary>
public static string? Check(string list)
{
try
{
Parse(list);
return null;
}
catch (FormatException ex)
{
return ex.Message;
}
}
public static IReadOnlyList<CalendarEvent> Parse(string list)
{
ArgumentNullException.ThrowIfNull(list);
var events = new List<CalendarEvent>();
var number = 0;
foreach (var raw in list.Split('\n'))
{
number++;
var line = raw.Trim();
if (line.Length == 0 || line.StartsWith('#'))
{
continue;
}
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length != 2 || !DateOnly.TryParseExact(parts[1], "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var day))
{
throw new FormatException($"Riga {number} dell'elenco delle banche centrali non valida: «{line}». Scrivi per esempio «FED 2026-12-09».");
}
var bank = parts[0].ToUpperInvariant();
var (title, currency, at) = bank switch
{
"FED" => ("Decisione sui tassi della Fed", "USD", Zones.At(day, new TimeOnly(14, 0), Zones.NewYork)),
"BCE" or "ECB" => ("Decisione sui tassi della BCE", "EUR", Zones.At(day, new TimeOnly(14, 15), Zones.Frankfurt)),
_ => throw new FormatException($"Riga {number}: banca «{parts[0]}» sconosciuta, usa FED o BCE."),
};
events.Add(new CalendarEvent(CalendarEventKinds.CentralBank, title, at, TimeKnown: true, Source, $"cb:{currency}:{day:yyyy-MM-dd}")
{
Currency = currency,
Impact = "High",
});
}
return events;
}
}
/// <summary>Crypto Fear and Greed index of alternative.me (attribution required: the interface names the source).</summary>
public static class FearGreed
{
public const string Source = "alternative.me";
public const string Attribution = "Indice di paura e avidità: alternative.me";
public static Uri Latest(int days) => new($"https://api.alternative.me/fng/?limit={days.ToString(CultureInfo.InvariantCulture)}&format=json");
public static IReadOnlyList<MoodReading> Parse(string json)
{
using var document = JsonDocument.Parse(json);
return document.RootElement.GetProperty("data").EnumerateArray()
.Select(item =>
{
var value = double.Parse(ForexFactory.Text(item, "value"), CultureInfo.InvariantCulture);
var at = DateTimeOffset.FromUnixTimeSeconds(long.Parse(ForexFactory.Text(item, "timestamp"), CultureInfo.InvariantCulture));
return new MoodReading(MoodIndicators.FearGreed, at, value, MoodLabels.FearGreed(value), Source);
})
.OrderBy(m => m.AsOf)
.ToList();
}
}
/// <summary>Daily VIX history from Cboe (no key). The file answers with a redirect to cdn-api.cboe.com.</summary>
public static class VixHistory
{
public const string Source = "cboe";
public static readonly Uri Csv = new("https://cdn.cboe.com/api/global/us_indices/daily_prices/VIX_History.csv");
/// <summary>Closes on or after <paramref name="from"/>, dated at midnight UTC of the session day.</summary>
public static IReadOnlyList<MoodReading> Parse(string csv, DateOnly from)
{
ArgumentNullException.ThrowIfNull(csv);
var readings = new List<MoodReading>();
foreach (var line in csv.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Skip(1))
{
var f = line.Split(',');
var day = DateOnly.ParseExact(f[0], "MM/dd/yyyy", CultureInfo.InvariantCulture);
if (day < from || f.Length < 5)
{
continue;
}
var close = double.Parse(f[4], CultureInfo.InvariantCulture);
readings.Add(new MoodReading(MoodIndicators.Vix, QualityChecks.Midnight(day), close, MoodLabels.Vix(close), Source));
}
return readings;
}
}
/// <summary>A row of nasdaqtraded.txt: every security traded in the USA with its ETF flag.</summary>
public sealed record ListedSecurity(string Symbol, string Name, string Exchange, bool IsEtf, bool IsTest, string FinancialStatus)
{
/// <summary>
/// ETF, ETN, fund, or not a common share (preferred, warrant, right, unit, note): excluded from the stock universe
/// (SPEC 4.6, 8.2). The ETF flag misses many ETNs and every closed-end fund, so the name is read too.
/// </summary>
public bool IsExcluded => IsEtf || IsTest || NotCommon().IsMatch(Name);
private static Regex NotCommon() => NasdaqTraded.NotCommonPattern();
}
/// <summary>The public list of US securities (SPEC 4.1): https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqtraded.txt.</summary>
public static partial class NasdaqTraded
{
public const string Source = "nasdaqtrader";
public static readonly Uri File = new("https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqtraded.txt");
/// <summary>Listing exchange codes: Q Nasdaq, N NYSE, A NYSE American, P NYSE Arca, Z Cboe BZX, V IEX, F Texas Stock Exchange, M NYSE Texas.</summary>
public static readonly IReadOnlyDictionary<string, string> Exchanges = new Dictionary<string, string>
{
["Q"] = "NASDAQ",
["N"] = "NYSE",
["A"] = "AMEX",
["P"] = "ARCA",
["Z"] = "BATS",
["V"] = "IEX",
["F"] = "TXSE",
["M"] = "NYSETX",
};
[GeneratedRegex(@"\b(ETF|ETN|ETNs|Exchange[- ]Traded|Fund|Funds|Trust Units?|Warrants?|Rights?|Units?|Preferred|Notes?|Debentures?|Depositary Shares? representing a fraction|Closed[- ]End|Income Shares|Beneficial Interest|Index|Leveraged|Inverse|SPAC)\b", RegexOptions.IgnoreCase)]
internal static partial Regex NotCommonPattern();
/// <summary>Rows of the pipe-separated file (CRLF), header and footer skipped.</summary>
public static IReadOnlyList<ListedSecurity> Parse(string text)
{
ArgumentNullException.ThrowIfNull(text);
var lines = text.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var header = lines[0].Split('|');
int Column(string name) => Array.IndexOf(header, name) is var i and >= 0 ? i : throw new FormatException("nasdaqtraded.txt: manca la colonna " + name);
int symbol = Column("Symbol"), name = Column("Security Name"), exchange = Column("Listing Exchange"), etf = Column("ETF"),
test = Column("Test Issue"), status = Column("Financial Status");
var list = new List<ListedSecurity>();
foreach (var line in lines.Skip(1))
{
if (line.StartsWith("File Creation Time", StringComparison.Ordinal))
{
continue;
}
var f = line.Split('|');
if (f.Length < header.Length)
{
continue;
}
list.Add(new ListedSecurity(f[symbol], f[name], Exchanges.GetValueOrDefault(f[exchange], f[exchange]), f[etf] == "Y", f[test] == "Y", f[status]));
}
return list;
}
}
/// <summary>
/// The public archives of data.binance.vision (SPEC 4.2, 6.6): monthly zips of klines with a SHA-256 checksum file.
/// Research only: Binance does not serve Italian residents since July 2026.
/// </summary>
public static class BinanceArchive
{
public const string Host = "https://data.binance.vision";
public static Uri Monthly(string symbol, string interval, int year, int month) =>
new($"{Host}/data/spot/monthly/klines/{symbol}/{interval}/{symbol}-{interval}-{year:D4}-{month:D2}.zip");
public static Uri Daily(string symbol, string interval, DateOnly day) =>
new($"{Host}/data/spot/daily/klines/{symbol}/{interval}/{symbol}-{interval}-{day:yyyy-MM-dd}.zip");
public static Uri ChecksumOf(Uri zip) => new(zip + ".CHECKSUM");
/// <summary>Checks the zip against its CHECKSUM file: "&lt;sha256 hex&gt; &lt;file name&gt;".</summary>
public static bool Verify(byte[] zip, string checksumFile)
{
ArgumentNullException.ThrowIfNull(checksumFile);
var expected = checksumFile.Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries)[0];
return string.Equals(Convert.ToHexStringLower(SHA256.HashData(zip)), expected, StringComparison.OrdinalIgnoreCase);
}
public static IReadOnlyList<ResearchBar> ReadZip(byte[] zip)
{
using var archive = new ZipArchive(new MemoryStream(zip), ZipArchiveMode.Read);
var entry = archive.Entries.Single(e => e.Name.EndsWith(".csv", StringComparison.OrdinalIgnoreCase));
using var reader = new StreamReader(entry.Open());
return ParseCsv(reader.ReadToEnd());
}
/// <summary>
/// Klines without header: open_time, open, high, low, close, volume, close_time, quote_volume, trades, … The
/// open time is in milliseconds up to 2024 and in microseconds from 2025; the last column is ignored.
/// </summary>
public static IReadOnlyList<ResearchBar> ParseCsv(string csv)
{
ArgumentNullException.ThrowIfNull(csv);
var bars = new List<ResearchBar>();
foreach (var line in csv.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
{
var f = line.Split(',');
if (f.Length < 9 || !long.TryParse(f[0], NumberStyles.None, CultureInfo.InvariantCulture, out var time))
{
continue; // A header line some archives carry.
}
bars.Add(new ResearchBar(FromEpoch(time), Number(f[1]), Number(f[2]), Number(f[3]), Number(f[4]), Number(f[5]))
{
QuoteVolume = Number(f[7]),
Trades = long.Parse(f[8], NumberStyles.None, CultureInfo.InvariantCulture),
});
}
return bars;
}
/// <summary>Microseconds when the number is too large to be milliseconds (after year 5138).</summary>
public static DateTimeOffset FromEpoch(long value) =>
value >= 100_000_000_000_000 ? DateTimeOffset.UnixEpoch.AddTicks(value * 10) : DateTimeOffset.FromUnixTimeMilliseconds(value);
private static decimal Number(string text) => decimal.Parse(text, NumberStyles.Float, CultureInfo.InvariantCulture);
}
/// <summary>Time zones of the exchanges and central banks, from the system's database (both Windows and Linux know the IANA ids).</summary>
public static class Zones
{
public static readonly TimeZoneInfo NewYork = TimeZoneInfo.FindSystemTimeZoneById("America/New_York");
public static readonly TimeZoneInfo Frankfurt = TimeZoneInfo.FindSystemTimeZoneById("Europe/Berlin");
public static readonly TimeZoneInfo Rome = TimeZoneInfo.FindSystemTimeZoneById("Europe/Rome");
public static DateTimeOffset At(DateOnly day, TimeOnly time, TimeZoneInfo zone)
{
ArgumentNullException.ThrowIfNull(zone);
var local = day.ToDateTime(time);
return new DateTimeOffset(local, zone.GetUtcOffset(local)).ToUniversalTime();
}
}
+246
View File
@@ -0,0 +1,246 @@
using System.Globalization;
using System.Text.Json;
using System.Text.Json.Serialization;
using Anubis.Brokers;
using Anubis.Core;
namespace Anubis.Data;
/// <summary>What an import did, for the log and the interface.</summary>
public sealed record ImportResult(string Venue, string Symbol, string Timeframe, int Bars, DateTimeOffset? First, DateTimeOffset? Last, IReadOnlyList<string> Notes);
/// <summary>
/// Downloads the monthly kline archives of data.binance.vision into the research history (issue #21). Every zip
/// is checked against its SHA-256 file before use; a month not published yet (404) ends the run without error.
/// Years already complete on disk are skipped.
/// </summary>
public sealed class BinanceImporter(ApiHttp http, ResearchStore store, IClock clock)
{
/// <summary>The archive serves files from S3: a few requests a second is far below any limit.</summary>
private readonly RateLimiter _limiter = new(clock, 5, TimeSpan.FromSeconds(1));
public async Task<ImportResult> ImportAsync(string symbol, string interval, int fromYear, int fromMonth, CancellationToken cancellationToken)
{
var timeframe = interval == "1m" ? Timeframes.Minute : interval == "1d" ? Timeframes.Day : interval;
var notes = new List<string>();
var total = 0;
DateTimeOffset? first = null, last = null;
var today = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime);
var have = store.Years(Venues.Binance, symbol, timeframe).ToHashSet();
for (var month = new DateOnly(fromYear, fromMonth, 1); month <= today; month = month.AddMonths(1))
{
// Months go in order, so a year is complete once the next one has a file. A year interrupted halfway is
// read again: writing merges with what is on disk.
if (have.Contains(month.Year) && have.Contains(month.Year + 1))
{
continue;
}
var url = BinanceArchive.Monthly(symbol, interval, month.Year, month.Month);
var zip = await http.GetBytesAsync(url, _limiter, null, cancellationToken).ConfigureAwait(false);
if (zip is null)
{
if (total > 0 || month >= new DateOnly(today.Year, today.Month, 1).AddMonths(-1))
{
notes.Add($"{month:yyyy-MM}: archivio non ancora pubblicato");
break;
}
continue; // Before the pair was listed.
}
var checksum = await http.GetStringAsync(BinanceArchive.ChecksumOf(url), _limiter, null, cancellationToken).ConfigureAwait(false);
if (!BinanceArchive.Verify(zip, checksum))
{
throw new MarketDataException($"Binance: l'archivio {month:yyyy-MM} di {symbol} non corrisponde alla sua impronta SHA-256.", retryable: true);
}
var bars = BinanceArchive.ReadZip(zip);
if (bars.Count == 0)
{
continue;
}
await store.WriteAsync(Venues.Binance, symbol, timeframe, bars, cancellationToken).ConfigureAwait(false);
total += bars.Count;
first ??= bars[0].OpensAt;
last = bars[^1].OpensAt;
}
return new ImportResult(Venues.Binance, symbol, timeframe, total, first, last, notes);
}
}
/// <summary>
/// Column mapping of a user CSV in <c>/data/import</c> (SPEC 6.6): a JSON file next to the CSV with the same name and
/// <c>.mapping.json</c>, or <c>mapping.json</c> for the whole folder. Columns are named by header or by 0-based number.
/// </summary>
public sealed record CsvMapping
{
/// <summary>Symbol of the series, e.g. BTCEUR; when empty, the part of the file name before the first '-' or '_'.</summary>
public string? Symbol { get; init; }
public string Venue { get; init; } = "import";
public string Timeframe { get; init; } = Timeframes.Minute;
public string Delimiter { get; init; } = ",";
public bool HasHeader { get; init; } = true;
public string Time { get; init; } = "0";
/// <summary><c>unix_ms</c>, <c>unix_s</c>, <c>unix_us</c>, <c>auto</c> (milliseconds or microseconds by size), or a .NET date format such as <c>yyyy-MM-dd HH:mm:ss</c>.</summary>
public string TimeFormat { get; init; } = "auto";
/// <summary>Time zone of text times (IANA id); unix times are always UTC.</summary>
public string TimeZone { get; init; } = "UTC";
public string Open { get; init; } = "1";
public string High { get; init; } = "2";
public string Low { get; init; } = "3";
public string Close { get; init; } = "4";
public string Volume { get; init; } = "5";
public string? QuoteVolume { get; init; }
/// <summary>Decimal separator of the numbers: "." or ","; with "," a "." is read as the thousands separator.</summary>
public string DecimalSeparator { get; init; } = ".";
/// <summary>True for a market that never closes (crypto): every missing minute or day is reported as a gap.</summary>
public bool Continuous { get; init; } = true;
public static readonly JsonSerializerOptions Json = new()
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
}
/// <summary>
/// The generic importer of user CSV files with one-minute or daily bars into the research history (issue #21).
/// Lines that cannot be read are counted and reported, never silently fixed.
/// </summary>
public sealed class CsvImporter(ResearchStore store)
{
public const int MaxReportedErrors = 5;
public static CsvMapping MappingFor(string csvPath)
{
ArgumentNullException.ThrowIfNull(csvPath);
var own = Path.ChangeExtension(csvPath, ".mapping.json");
var shared = Path.Combine(Path.GetDirectoryName(csvPath) ?? ".", "mapping.json");
var path = File.Exists(own) ? own : File.Exists(shared) ? shared : null;
return path is null ? new CsvMapping() : JsonSerializer.Deserialize<CsvMapping>(File.ReadAllText(path), CsvMapping.Json) ?? new CsvMapping();
}
public async Task<ImportResult> ImportAsync(string csvPath, CsvMapping mapping, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(mapping);
var symbol = string.IsNullOrWhiteSpace(mapping.Symbol)
? Path.GetFileNameWithoutExtension(csvPath).Split('-', '_')[0].ToUpperInvariant()
: mapping.Symbol;
var zone = TimeZoneInfo.FindSystemTimeZoneById(mapping.TimeZone);
var numbers = new NumberFormatInfo { NumberDecimalSeparator = mapping.DecimalSeparator, NumberGroupSeparator = mapping.DecimalSeparator == "," ? "." : "," };
var style = mapping.DecimalSeparator == "," ? NumberStyles.Float | NumberStyles.AllowThousands : NumberStyles.Float;
var bars = new List<ResearchBar>();
var notes = new List<string>();
var errors = 0;
string[]? header = null;
var lineNumber = 0;
foreach (var line in await File.ReadAllLinesAsync(csvPath, cancellationToken).ConfigureAwait(false))
{
lineNumber++;
if (line.Length == 0)
{
continue;
}
var fields = line.Split(mapping.Delimiter);
if (header is null && mapping.HasHeader)
{
header = fields.Select(f => f.Trim().Trim('"')).ToArray();
continue;
}
try
{
string Field(string column) => fields[Index(column, header)].Trim().Trim('"');
decimal Number(string column) => decimal.Parse(Field(column), style, numbers);
bars.Add(new ResearchBar(ParseTime(Field(mapping.Time), mapping.TimeFormat, zone), Number(mapping.Open), Number(mapping.High),
Number(mapping.Low), Number(mapping.Close), Number(mapping.Volume))
{
QuoteVolume = mapping.QuoteVolume is { Length: > 0 } q ? Number(q) : null,
});
}
catch (Exception ex) when (ex is FormatException or IndexOutOfRangeException or OverflowException or ArgumentException)
{
if (++errors <= MaxReportedErrors)
{
notes.Add($"riga {lineNumber}: {ex.Message}");
}
}
}
if (errors > 0)
{
notes.Add($"{errors} righe non lette su {lineNumber}");
}
var ordered = bars.OrderBy(b => b.OpensAt).ToList();
IReadOnlyList<QualityFinding> issues = [];
if (ordered.Count > 0 && mapping.Timeframe == Timeframes.Minute)
{
issues = QualityChecks.CheckMinutes(ordered.Select(b => b.ToBar()).ToList(),
mapping.Continuous ? QualityChecks.EveryMinute(ordered[0].OpensAt, ordered[^1].OpensAt.AddMinutes(1)) : null);
}
else if (ordered.Count > 0)
{
var days = ordered.Select(b => new DailyBar(DateOnly.FromDateTime(b.OpensAt.UtcDateTime), (double)b.Open, (double)b.High, (double)b.Low,
(double)b.Close, (double)b.Volume)).ToList();
issues = QualityChecks.CheckDaily(days, mapping.Continuous ? QualityChecks.EveryDay(days[0].Day, days[^1].Day) : null);
}
notes.AddRange(issues.Take(MaxReportedErrors).Select(i => $"{i.Kind} {i.StartsAt:yyyy-MM-dd HH:mm}: {i.Detail}"));
await store.WriteAsync(mapping.Venue, symbol, mapping.Timeframe, bars, cancellationToken).ConfigureAwait(false);
return new ImportResult(mapping.Venue, symbol, mapping.Timeframe, bars.Count, ordered.FirstOrDefault()?.OpensAt, ordered.LastOrDefault()?.OpensAt, notes);
}
private static int Index(string column, string[]? header)
{
if (int.TryParse(column, NumberStyles.None, CultureInfo.InvariantCulture, out var index))
{
return index;
}
var found = header is null ? -1 : Array.FindIndex(header, h => string.Equals(h, column, StringComparison.OrdinalIgnoreCase));
return found >= 0 ? found : throw new ArgumentException($"colonna «{column}» assente");
}
private static DateTimeOffset ParseTime(string text, string format, TimeZoneInfo zone)
{
switch (format)
{
case "unix_s":
return DateTimeOffset.FromUnixTimeSeconds(long.Parse(text, CultureInfo.InvariantCulture));
case "unix_ms":
return DateTimeOffset.FromUnixTimeMilliseconds(long.Parse(text, CultureInfo.InvariantCulture));
case "unix_us":
return DateTimeOffset.UnixEpoch.AddTicks(long.Parse(text, CultureInfo.InvariantCulture) * 10);
case "auto" when long.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out var number):
return number < 100_000_000_000 ? DateTimeOffset.FromUnixTimeSeconds(number) : BinanceArchive.FromEpoch(number);
case "auto":
return DateTimeOffset.Parse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
default:
var local = DateTime.ParseExact(text, format, CultureInfo.InvariantCulture, DateTimeStyles.None);
return new DateTimeOffset(local, zone.GetUtcOffset(local)).ToUniversalTime();
}
}
}
+278
View File
@@ -0,0 +1,278 @@
using System.Collections.Concurrent;
using Anubis.Core;
using Anubis.Db;
namespace Anubis.Data;
/// <summary>A reference exchange rate: <see cref="Rate"/> units of <see cref="Quote"/> for one <see cref="Base"/>, as published for <see cref="Date"/>.</summary>
public sealed record FxRate(string Base, string Quote, DateOnly Date, decimal Rate, string Source);
/// <summary>
/// Reference exchange rates (SPEC 6.8): one per day as the ECB publishes them. A day without a rate uses the last
/// one before it, as the tax export requires (SPEC 4.6).
/// </summary>
public sealed class FxStore(Database db, IClock clock)
{
public async Task<int> UpsertAsync(IEnumerable<FxRate> rates, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(rates);
var list = rates.ToList();
var now = clock.UtcNow;
await db.WriteAsync(async s =>
{
foreach (var rate in list)
{
await s.ExecuteAsync(
"INSERT INTO fx_rates (id, base, quote, rate_date, rate, source, created_at) VALUES (@id, @b, @q, @d, @r, @s, @now) " +
"ON CONFLICT (base, quote, rate_date) DO UPDATE SET rate = excluded.rate, source = excluded.source",
cancellationToken, ("id", Ids.NewId(clock)), ("b", rate.Base), ("q", rate.Quote), ("d", BarStore.DayText(rate.Date)),
("r", rate.Rate), ("s", rate.Source), ("now", now)).ConfigureAwait(false);
}
}, cancellationToken).ConfigureAwait(false);
return list.Count;
}
/// <summary>The rate published for <paramref name="day"/>, or the last one before it; null if none is known.</summary>
public Task<FxRate?> RateOnAsync(string baseCurrency, string quote, DateOnly day, CancellationToken cancellationToken) =>
db.ReadAsync(s => s.FirstOrDefaultAsync(
"SELECT base, quote, rate_date, rate, source FROM fx_rates WHERE base = @b AND quote = @q AND rate_date <= @d ORDER BY rate_date DESC",
Read, ("b", baseCurrency), ("q", quote), ("d", BarStore.DayText(day))), cancellationToken);
public Task<FxRate?> LatestAsync(string baseCurrency, string quote, CancellationToken cancellationToken) =>
RateOnAsync(baseCurrency, quote, DateOnly.MaxValue, cancellationToken);
private static FxRate Read(DbRow r) =>
new(r.Text("base"), r.Text("quote"), BarStore.ParseDay(r.Text("rate_date")), r.Money("rate"), r.Text("source"));
}
public static class CalendarEventKinds
{
public const string Economic = "economic";
public const string CentralBank = "central_bank";
public const string Earnings = "earnings";
public const string Holiday = "holiday";
}
/// <summary>
/// An event of the calendar (SPEC 10.4). <see cref="TimeKnown"/> is false when the source gives only the day
/// (earnings without an hour, holidays): <see cref="OccursAt"/> is then midnight of that day in the exchange's zone.
/// </summary>
public sealed record CalendarEvent(string Kind, string Title, DateTimeOffset OccursAt, bool TimeKnown, string Source, string SourceKey)
{
public string? Currency { get; init; }
public string? Symbol { get; init; }
/// <summary>For earnings: <c>bmo</c> before the open, <c>amc</c> after the close, <c>dmh</c> during market hours.</summary>
public string? TimeHint { get; init; }
/// <summary>Low, Medium or High.</summary>
public string? Impact { get; init; }
public string? Forecast { get; init; }
public string? Previous { get; init; }
}
public sealed class CalendarEventStore(Database db, IClock clock)
{
private const string Columns = "kind, title, currency, symbol, occurs_at, time_known, time_hint, impact, forecast, previous, source, source_key";
/// <summary>Adds new events and updates known ones (a moved date, a new forecast). Returns how many arrived.</summary>
public async Task<int> UpsertAsync(IEnumerable<CalendarEvent> events, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(events);
var list = events.DistinctBy(e => e.SourceKey).ToList();
var now = clock.UtcNow;
await db.WriteAsync(async s =>
{
foreach (var e in list)
{
await s.ExecuteAsync(
$"INSERT INTO calendar_events (id, {Columns}, created_at, updated_at) VALUES (@id, @kind, @title, @ccy, @sym, @at, @known, @hint, @impact, " +
"@forecast, @previous, @source, @key, @now, @now) ON CONFLICT (source_key) DO UPDATE SET kind = excluded.kind, title = excluded.title, " +
"currency = excluded.currency, symbol = excluded.symbol, occurs_at = excluded.occurs_at, time_known = excluded.time_known, " +
"time_hint = excluded.time_hint, impact = excluded.impact, forecast = excluded.forecast, previous = excluded.previous, updated_at = excluded.updated_at",
cancellationToken, ("id", Ids.NewId(clock)), ("kind", e.Kind), ("title", e.Title), ("ccy", e.Currency), ("sym", e.Symbol), ("at", e.OccursAt),
("known", e.TimeKnown), ("hint", e.TimeHint), ("impact", e.Impact), ("forecast", e.Forecast), ("previous", e.Previous), ("source", e.Source),
("key", e.SourceKey), ("now", now)).ConfigureAwait(false);
}
}, cancellationToken).ConfigureAwait(false);
return list.Count;
}
public async Task<IReadOnlyList<CalendarEvent>> BetweenAsync(DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken) =>
await db.ReadAsync(s => s.QueryAsync($"SELECT {Columns} FROM calendar_events WHERE occurs_at >= @f AND occurs_at < @t ORDER BY occurs_at, title",
r => new CalendarEvent(r.Text("kind"), r.Text("title"), r.Moment("occurs_at"), r.Bool("time_known"), r.Text("source"), r.Text("source_key"))
{
Currency = r.TextOrNull("currency"),
Symbol = r.TextOrNull("symbol"),
TimeHint = r.TextOrNull("time_hint"),
Impact = r.TextOrNull("impact"),
Forecast = r.TextOrNull("forecast"),
Previous = r.TextOrNull("previous"),
}, ("f", from), ("t", to)), cancellationToken).ConfigureAwait(false);
}
public static class MoodIndicators
{
public const string FearGreed = "fear_greed";
public const string Vix = "vix";
public const string BtcFunding = "btc_funding";
public const string BtcOpenInterest = "btc_open_interest";
}
public sealed record MoodReading(string Indicator, DateTimeOffset AsOf, double Value, string? Label, string Source);
/// <summary>The labels of SPEC 10.4, in the words the interface shows.</summary>
public static class MoodLabels
{
public static string FearGreed(double value) => value switch
{
<= 24 => "paura estrema",
<= 44 => "paura",
<= 55 => "neutro",
<= 75 => "avidità",
_ => "avidità estrema",
};
public static string Vix(double value) => value switch
{
< 20 => "calmo",
<= 30 => "nervoso",
_ => "panico",
};
/// <summary><paramref name="rate"/> is the funding of one period; <paramref name="interval"/> its length (usually 8 hours).</summary>
public static string Funding(double rate, TimeSpan interval)
{
var perEightHours = interval > TimeSpan.Zero ? rate * (TimeSpan.FromHours(8) / interval) : rate;
return perEightHours switch
{
> 0.0002 => "troppi al rialzo",
< -0.0002 => "troppi al ribasso",
_ => "nella norma",
};
}
}
public sealed class MoodStore(Database db, IClock clock)
{
public async Task<int> UpsertAsync(IEnumerable<MoodReading> readings, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(readings);
var list = readings.ToList();
var now = clock.UtcNow;
await db.WriteAsync(async s =>
{
foreach (var m in list)
{
await s.ExecuteAsync(
"INSERT INTO market_mood (id, indicator, as_of, value, label, source, created_at) VALUES (@id, @i, @a, @v, @l, @s, @now) " +
"ON CONFLICT (indicator, as_of) DO UPDATE SET value = excluded.value, label = excluded.label, source = excluded.source",
cancellationToken, ("id", Ids.NewId(clock)), ("i", m.Indicator), ("a", m.AsOf), ("v", m.Value), ("l", m.Label), ("s", m.Source), ("now", now))
.ConfigureAwait(false);
}
}, cancellationToken).ConfigureAwait(false);
return list.Count;
}
public Task<MoodReading?> LatestAsync(string indicator, CancellationToken cancellationToken) =>
db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT indicator, as_of, value, label, source FROM market_mood WHERE indicator = @i ORDER BY as_of DESC",
r => new MoodReading(r.Text("indicator"), r.Moment("as_of"), r.Measure("value"), r.TextOrNull("label"), r.Text("source")), ("i", indicator)),
cancellationToken);
}
/// <summary>
/// The exchange calendar as the broker gives it, saved day by day so the scheduler can read it without network
/// (SPEC 5.5: market times come from the broker's calendar, never from Italian time).
/// </summary>
public sealed class StoredMarketCalendar(Database db, IClock clock, string market) : IMarketCalendar
{
private readonly ConcurrentDictionary<DateOnly, MarketSession?> _days = new();
public string Market { get; } = market;
/// <summary>First and last day the stored calendar covers; null when it is empty.</summary>
public (DateOnly First, DateOnly Last)? Coverage => _days.IsEmpty ? null : (_days.Keys.Min(), _days.Keys.Max());
/// <summary>The session of <paramref name="day"/>; null when the market is closed or the day is not covered yet.</summary>
public MarketSession? SessionOn(DateOnly day) => _days.TryGetValue(day, out var session) ? session : null;
public bool Covers(DateOnly day) => _days.ContainsKey(day);
/// <summary>Saves the sessions of [first, last]; days without a session are saved as closed.</summary>
public async Task SaveAsync(IReadOnlyCollection<MarketSession> sessions, DateOnly first, DateOnly last, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(sessions);
var byDay = sessions.Where(x => x.Date >= first && x.Date <= last).ToDictionary(x => x.Date);
var now = clock.UtcNow;
await db.WriteAsync(async s =>
{
for (var day = first; day <= last; day = day.AddDays(1))
{
var session = byDay.GetValueOrDefault(day);
await s.ExecuteAsync(
"INSERT INTO market_calendar (id, market, session_date, is_open, open_at, close_at, created_at) VALUES (@id, @m, @d, @o, @oa, @ca, @now) " +
"ON CONFLICT (market, session_date) DO UPDATE SET is_open = excluded.is_open, open_at = excluded.open_at, close_at = excluded.close_at",
cancellationToken, ("id", Ids.NewId(clock)), ("m", Market), ("d", BarStore.DayText(day)), ("o", session is not null),
("oa", session?.OpenUtc), ("ca", session?.CloseUtc), ("now", now)).ConfigureAwait(false);
}
}, cancellationToken).ConfigureAwait(false);
for (var day = first; day <= last; day = day.AddDays(1))
{
_days[day] = byDay.GetValueOrDefault(day);
}
}
public async Task LoadAsync(CancellationToken cancellationToken)
{
var rows = await db.ReadAsync(s => s.QueryAsync("SELECT session_date, is_open, open_at, close_at FROM market_calendar WHERE market = @m",
r =>
{
var day = BarStore.ParseDay(r.Text("session_date"));
return (Day: day, Session: r.Bool("is_open") ? new MarketSession(day, r.Moment("open_at"), r.Moment("close_at")) : null);
}, ("m", Market)), cancellationToken).ConfigureAwait(false);
foreach (var (day, session) in rows)
{
_days[day] = session;
}
}
/// <summary>Sessions in [first, last], oldest first.</summary>
public IReadOnlyList<MarketSession> Sessions(DateOnly first, DateOnly last) =>
_days.Where(d => d.Key >= first && d.Key <= last && d.Value is not null).Select(d => d.Value!).OrderBy(x => x.Date).ToList();
}
/// <summary>State of an external source: when it was last asked, last answered, and why it is off (SPEC 4.4).</summary>
public sealed record SourceState(string Source, bool Enabled, DateTimeOffset? LastRequestAt, DateTimeOffset? LastSuccessAt, string? LastError, string? Note);
public sealed class SourceStateStore(Database db, IClock clock)
{
public Task RecordAsync(string source, bool enabled, bool success, string? error, string? note, CancellationToken cancellationToken)
{
var now = clock.UtcNow;
return db.WriteAsync(async s =>
{
var exists = await s.CountAsync("SELECT COUNT(*) FROM source_state WHERE source = @s", ("s", source)).ConfigureAwait(false) > 0;
if (!exists)
{
await s.ExecuteAsync("INSERT INTO source_state (id, source, enabled, created_at, updated_at) VALUES (@id, @s, @e, @now, @now)",
cancellationToken, ("id", Ids.NewId(clock)), ("s", source), ("e", enabled), ("now", now)).ConfigureAwait(false);
}
await s.ExecuteAsync(success
? "UPDATE source_state SET enabled = @e, last_request_at = @now, last_success_at = @now, last_error = NULL, note = @n, updated_at = @now WHERE source = @s"
: "UPDATE source_state SET enabled = @e, last_request_at = @now, last_error = @err, note = @n, updated_at = @now WHERE source = @s",
cancellationToken, success
? [("e", enabled), ("now", now), ("n", note), ("s", source)]
: [("e", enabled), ("now", now), ("err", error), ("n", note), ("s", source)]).ConfigureAwait(false);
}, cancellationToken);
}
public async Task<IReadOnlyList<SourceState>> ListAsync(CancellationToken cancellationToken) =>
await db.ReadAsync(s => s.QueryAsync("SELECT source, enabled, last_request_at, last_success_at, last_error, note FROM source_state ORDER BY source",
r => new SourceState(r.Text("source"), r.Bool("enabled"), r.MomentOrNull("last_request_at"), r.MomentOrNull("last_success_at"),
r.TextOrNull("last_error"), r.TextOrNull("note"))), cancellationToken).ConfigureAwait(false);
}
+221
View File
@@ -0,0 +1,221 @@
using System.Globalization;
using Anubis.Brokers;
using Anubis.Core;
using Anubis.Db;
namespace Anubis.Data;
/// <summary>
/// Downloads the free sources of SPEC 4.4 within their limits and records each attempt in <c>source_state</c>, so
/// limits hold across restarts and the interface can say what is off and why. A source that fails is retried at
/// its next turn; it never stops the others.
/// </summary>
public sealed class ReferenceSources(
ApiHttp http,
Database db,
IClock clock,
SettingsStore settings,
Func<string, CancellationToken, Task<string?>> secret,
OkxClient? okx = null)
{
/// <summary>Currencies kept from the ECB: the dollar for display and the tax export (SPEC 6.8).</summary>
public static readonly IReadOnlyList<string> Currencies = ["USD"];
public const string OkxDerivatives = "okx_derivatives";
/// <summary>
/// Shortest time between two requests to the same source. Forex Factory asks for no more than two requests every
/// five minutes; the others publish once a day or less.
/// </summary>
public static readonly IReadOnlyDictionary<string, TimeSpan> MinInterval = new Dictionary<string, TimeSpan>
{
[Ecb.Source] = TimeSpan.FromMinutes(30),
[ForexFactory.Source] = TimeSpan.FromMinutes(55),
[Fred.Source] = TimeSpan.FromHours(12),
[Finnhub.Source] = TimeSpan.FromHours(12),
[FearGreed.Source] = TimeSpan.FromMinutes(55),
[VixHistory.Source] = TimeSpan.FromHours(6),
[OkxDerivatives] = TimeSpan.FromMinutes(55),
[CentralBanks.Source] = TimeSpan.Zero,
};
private readonly SourceStateStore _states = new(db, clock);
private readonly FxStore _fx = new(db, clock);
private readonly CalendarEventStore _calendar = new(db, clock);
private readonly MoodStore _mood = new(db, clock);
/// <summary>Reference rates: the whole history the first time, the last 90 days after a pause, otherwise the day's file.</summary>
public Task<bool> RefreshFxAsync(CancellationToken cancellationToken) => RunAsync(Ecb.Source, async () =>
{
var latest = await _fx.LatestAsync("EUR", "USD", cancellationToken).ConfigureAwait(false);
var today = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime);
IReadOnlyList<FxRate> rates;
if (latest is null)
{
var zip = await http.GetBytesAsync(Ecb.History, null, null, cancellationToken).ConfigureAwait(false)
?? throw new MarketDataException("BCE: archivio storico dei cambi non trovato.", retryable: true);
rates = Ecb.ParseHistoryCsv(Ecb.ExtractHistoryCsv(zip), Currencies);
}
else
{
var url = latest.Date < today.AddDays(-4) ? Ecb.Last90Days : Ecb.Daily;
rates = Ecb.ParseXml(await http.GetStringAsync(url, null, null, cancellationToken).ConfigureAwait(false), Currencies);
}
await _fx.UpsertAsync(rates, cancellationToken).ConfigureAwait(false);
return $"{rates.Count} cambi";
}, cancellationToken);
/// <summary>The economic calendar of this week (Forex Factory), plus the central bank list of the settings.</summary>
public async Task<bool> RefreshCalendarAsync(CancellationToken cancellationToken)
{
var banks = await RunAsync(CentralBanks.Source, async () =>
{
var list = await settings.GetAsync(SettingsStore.CentralBankMeetings.Key, cancellationToken).ConfigureAwait(false);
var events = CentralBanks.Parse(list.Value);
await _calendar.UpsertAsync(events, cancellationToken).ConfigureAwait(false);
var last = events.Count == 0 ? (DateTimeOffset?)null : events.Max(e => e.OccursAt);
return last is { } l && l < clock.UtcNow.AddDays(60)
? $"L'elenco delle riunioni finisce il {l:dd/MM/yyyy}: aggiungi le date del nuovo anno nelle impostazioni."
: $"{events.Count} riunioni";
}, cancellationToken).ConfigureAwait(false);
var week = await RunAsync(ForexFactory.Source, async () =>
{
var events = ForexFactory.Parse(await http.GetStringAsync(ForexFactory.ThisWeek, null, null, cancellationToken).ConfigureAwait(false));
await _calendar.UpsertAsync(events, cancellationToken).ConfigureAwait(false);
return $"{events.Count} eventi";
}, cancellationToken).ConfigureAwait(false);
return banks && week;
}
/// <summary>Release dates of the main US data from FRED; off without a key, and the interface says so.</summary>
public async Task<bool> RefreshFredAsync(CancellationToken cancellationToken)
{
var key = await secret("FRED_API_KEY", cancellationToken).ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(key))
{
await _states.RecordAsync(Fred.Source, enabled: false, success: false, null,
"Spento: manca la chiave FRED_API_KEY. Il calendario usa solo Forex Factory.", cancellationToken).ConfigureAwait(false);
return false;
}
return await RunAsync(Fred.Source, async () =>
{
var from = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime);
var count = 0;
foreach (var (id, title) in Fred.Releases)
{
var events = Fred.Parse(await http.GetStringAsync(Fred.ReleaseDates(id, key, from), null, null, cancellationToken).ConfigureAwait(false), title, from);
count += await _calendar.UpsertAsync(events.Where(e => e.OccursAt < clock.UtcNow.AddDays(120)), cancellationToken).ConfigureAwait(false);
}
return $"{count} date";
}, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Earnings dates for the next two weeks. Without a key the earnings filter is off (SPEC 4.4): the state says so
/// and the strategies read <see cref="EarningsFilterActiveAsync"/>.
/// </summary>
public async Task<bool> RefreshEarningsAsync(IReadOnlySet<string>? symbols, CancellationToken cancellationToken)
{
var key = await secret("FINNHUB_API_KEY", cancellationToken).ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(key))
{
await _states.RecordAsync(Finnhub.Source, enabled: false, success: false, null,
"Spento: manca la chiave FINNHUB_API_KEY. Il filtro sulle trimestrali è disattivato.", cancellationToken).ConfigureAwait(false);
return false;
}
return await RunAsync(Finnhub.Source, async () =>
{
var today = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime);
var events = Finnhub.Parse(await http.GetStringAsync(Finnhub.Earnings(today, today.AddDays(14), key), null, null, cancellationToken).ConfigureAwait(false),
symbols);
await _calendar.UpsertAsync(events, cancellationToken).ConfigureAwait(false);
return $"{events.Count} trimestrali";
}, cancellationToken).ConfigureAwait(false);
}
/// <summary>True when earnings dates are known: the source is on and answered in the last two days.</summary>
public async Task<bool> EarningsFilterActiveAsync(CancellationToken cancellationToken)
{
var state = (await _states.ListAsync(cancellationToken).ConfigureAwait(false)).FirstOrDefault(s => s.Source == Finnhub.Source);
return state is { Enabled: true, LastSuccessAt: { } at } && clock.UtcNow - at < TimeSpan.FromDays(2);
}
/// <summary>Mood indicators: fear and greed, VIX, and Bitcoin funding and open interest from OKX public data.</summary>
public async Task<bool> RefreshMoodAsync(CancellationToken cancellationToken)
{
var fng = await RunAsync(FearGreed.Source, async () =>
{
var first = await _mood.LatestAsync(MoodIndicators.FearGreed, cancellationToken).ConfigureAwait(false) is null;
var readings = FearGreed.Parse(await http.GetStringAsync(FearGreed.Latest(first ? 400 : 2), null, null, cancellationToken).ConfigureAwait(false));
await _mood.UpsertAsync(readings, cancellationToken).ConfigureAwait(false);
return FearGreed.Attribution;
}, cancellationToken).ConfigureAwait(false);
var vix = await RunAsync(VixHistory.Source, async () =>
{
var from = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime).AddDays(-400);
var readings = VixHistory.Parse(await http.GetStringAsync(VixHistory.Csv, null, null, cancellationToken).ConfigureAwait(false), from);
await _mood.UpsertAsync(readings, cancellationToken).ConfigureAwait(false);
return $"{readings.Count} chiusure";
}, cancellationToken).ConfigureAwait(false);
if (okx is null)
{
return fng && vix;
}
var derivatives = await RunAsync(OkxDerivatives, async () =>
{
var funding = await okx.FetchFundingRateAsync("BTC-USDT-SWAP", cancellationToken).ConfigureAwait(false);
var interest = await okx.FetchOpenInterestAsync("BTC-USDT-SWAP", cancellationToken).ConfigureAwait(false);
await _mood.UpsertAsync(
[
new MoodReading(MoodIndicators.BtcFunding, funding.At, funding.Rate, MoodLabels.Funding(funding.Rate, funding.Interval), Venues.Okx),
new MoodReading(MoodIndicators.BtcOpenInterest, interest.At, interest.Usd, null, Venues.Okx),
], cancellationToken).ConfigureAwait(false);
return "funding " + funding.Rate.ToString("P4", CultureInfo.InvariantCulture);
}, cancellationToken).ConfigureAwait(false);
return fng && vix && derivatives;
}
public Task<IReadOnlyList<SourceState>> StatesAsync(CancellationToken cancellationToken) => _states.ListAsync(cancellationToken);
/// <summary>
/// Runs one source if its minimum interval has passed and records the outcome. Returns true when it succeeded or
/// was not due yet; false when it failed.
/// </summary>
private async Task<bool> RunAsync(string source, Func<Task<string>> work, CancellationToken cancellationToken)
{
var state = (await _states.ListAsync(cancellationToken).ConfigureAwait(false)).FirstOrDefault(s => s.Source == source);
if (state?.LastRequestAt is { } last && clock.UtcNow - last < MinInterval.GetValueOrDefault(source, TimeSpan.FromMinutes(5)))
{
return true;
}
try
{
var note = await work().ConfigureAwait(false);
await _states.RecordAsync(source, enabled: true, success: true, null, note, cancellationToken).ConfigureAwait(false);
return true;
}
catch (Exception ex) when (ex is MarketDataException or HttpRequestException or FormatException or System.Text.Json.JsonException
or System.Xml.XmlException or TaskCanceledException or InvalidDataException or KeyNotFoundException or InvalidOperationException or IndexOutOfRangeException)
{
if (cancellationToken.IsCancellationRequested)
{
throw;
}
await _states.RecordAsync(source, enabled: true, success: false, Redactor.Redact(ex.Message), state?.Note, cancellationToken).ConfigureAwait(false);
return false;
}
}
}
+8
View File
@@ -130,6 +130,14 @@ internal sealed class FastClock(DateTimeOffset start) : Anubis.Core.IClock
public TimeSpan Waited { get; private set; }
public void Advance(TimeSpan by)
{
lock (_gate)
{
_now += by;
}
}
public Task WaitUntilAsync(DateTimeOffset dueUtc, CancellationToken cancellationToken)
{
lock (_gate)
+431
View File
@@ -0,0 +1,431 @@
using System.IO.Compression;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using Anubis.Brokers;
using Anubis.Core;
using Anubis.Data;
namespace Anubis.Tests;
/// <summary>Readers of the free sources on recorded responses, their stores, and the research importers (issues #21, #22, #24).</summary>
[TestClass]
public sealed class ReferenceSourcesTests
{
private static readonly DateTimeOffset _start = new(2026, 11, 10, 8, 0, 0, TimeSpan.Zero);
[TestMethod]
[TestCategory(Category.Unit)]
public void EcbDailyNinetyDayAndHistoryFilesGiveTheSameRates()
{
string[] wanted = ["USD", "GBP", "BGN"];
var daily = Ecb.ParseXml(Fixture.Text("free/ecb-daily.xml"), wanted);
Assert.AreEqual(2, daily.Count, "BGN is no longer published");
Assert.AreEqual(new FxRate("EUR", "USD", new DateOnly(2026, 11, 9), 1.1193m, Ecb.Source), daily.Single(r => r.Quote == "USD"));
Assert.AreEqual(0.83430m, daily.Single(r => r.Quote == "GBP").Rate);
var ninety = Ecb.ParseXml(Fixture.Text("free/ecb-hist-90d.xml"), ["USD"]);
Assert.IsTrue(ninety.Count > 1);
Assert.AreEqual(1.1193m, ninety.Single(r => r.Date == new DateOnly(2026, 11, 9)).Rate, "the 90-day file uses double quotes");
CollectionAssert.AreEqual(ninety.OrderBy(r => r.Date).ToList(), ninety.ToList(), "oldest first");
var history = Ecb.ParseHistoryCsv(Fixture.Text("free/ecb-hist.csv"), wanted);
Assert.AreEqual(4, history.Count, "N/A is skipped and the trailing comma ignored");
Assert.AreEqual(new DateOnly(2026, 11, 6), history[0].Date);
Assert.AreEqual(1.1165m, history.Single(r => r.Quote == "USD" && r.Date == new DateOnly(2026, 11, 6)).Rate);
using var zip = new MemoryStream();
using (var archive = new ZipArchive(zip, ZipArchiveMode.Create, leaveOpen: true))
using (var writer = new StreamWriter(archive.CreateEntry("eurofxref-hist.csv").Open()))
{
writer.Write(Fixture.Text("free/ecb-hist.csv"));
}
Assert.AreEqual(Fixture.Text("free/ecb-hist.csv"), Ecb.ExtractHistoryCsv(zip.ToArray()));
}
[TestMethod]
[TestCategory(Category.Unit)]
public void ForexFactoryEventsKeepTheirTimeAndHolidaysOnlyTheDay()
{
var events = ForexFactory.Parse(Fixture.Text("free/ff-calendar.json"));
Assert.AreEqual(5, events.Count);
var cpi = events.Single(e => e.Title == "CPI m/m");
Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 13, 30, 0, TimeSpan.Zero), cpi.OccursAt, "the offset -05:00 becomes UTC");
Assert.AreEqual("USD", cpi.Currency);
Assert.AreEqual("High", cpi.Impact);
Assert.AreEqual("0.3%", cpi.Forecast);
Assert.AreEqual("0.6%", cpi.Previous);
Assert.IsTrue(cpi.TimeKnown);
var holiday = events.Single(e => e.Title == "Bank Holiday");
Assert.AreEqual(CalendarEventKinds.Holiday, holiday.Kind);
Assert.IsFalse(holiday.TimeKnown);
Assert.IsNull(holiday.Impact);
Assert.IsNull(events.Single(e => e.Title == "RBA Assist Gov Hunter Speaks").Forecast, "an empty forecast is no forecast");
Assert.AreEqual(events.Count, events.Select(e => e.SourceKey).Distinct().Count());
}
[TestMethod]
[TestCategory(Category.Unit)]
public void FredFinnhubAndCentralBankDatesBecomeTimedEvents()
{
var fred = Fred.Parse(Fixture.Text("free/fred-release-dates.json"), "PIL USA", new DateOnly(2010, 1, 1));
Assert.AreEqual(4, fred.Count, "dates before the start are skipped");
Assert.AreEqual(new DateTimeOffset(2010, 2, 11, 13, 30, 0, TimeSpan.Zero), fred[0].OccursAt, "08:30 New York in winter");
Assert.AreEqual("fred:82:2010-02-11", fred[0].SourceKey);
Assert.AreEqual(new DateTimeOffset(2012, 10, 24, 12, 30, 0, TimeSpan.Zero), fred[2].OccursAt, "08:30 New York in summer time");
StringAssert.Contains(Fred.ReleaseDates(10, "k", new DateOnly(2026, 11, 10)).Query, "include_release_dates_with_no_data=true");
var earnings = Finnhub.Parse(Fixture.Text("free/finnhub-earnings.json"));
Assert.AreEqual(2, earnings.Count);
var aapl = earnings.Single(e => e.OccursAt.Year == 2020);
Assert.AreEqual(new DateTimeOffset(2020, 1, 28, 21, 5, 0, TimeSpan.Zero), aapl.OccursAt, "amc: 16:05 New York");
Assert.AreEqual("amc", aapl.TimeHint);
Assert.AreEqual("AAPL", aapl.Symbol);
Assert.AreEqual(0, Finnhub.Parse(Fixture.Text("free/finnhub-earnings.json"), new HashSet<string> { "MSFT" }).Count);
var banks = CentralBanks.Parse(CentralBanks.DefaultList);
Assert.AreEqual(18, banks.Count);
var fed = banks.Single(e => e.SourceKey == "cb:USD:2026-12-09");
Assert.AreEqual(new DateTimeOffset(2026, 12, 9, 19, 0, 0, TimeSpan.Zero), fed.OccursAt, "14:00 New York");
var ecb = banks.Single(e => e.SourceKey == "cb:EUR:2027-07-22");
Assert.AreEqual(new DateTimeOffset(2027, 7, 22, 12, 15, 0, TimeSpan.Zero), ecb.OccursAt, "14:15 Frankfurt in summer time");
// Every decision day of the default list is in the official calendars read for the fixture.
var official = Fixture.Text("free/central-banks.md");
foreach (var e in banks)
{
StringAssert.Contains(official, $"| {e.OccursAt:yyyy-MM-dd} |");
}
Assert.AreEqual(1, CentralBanks.Parse("# commento\n\nfed 2027-01-27\n").Count);
var error = Assert.ThrowsExactly<FormatException>(() => CentralBanks.Parse("FED 2027-01-27\nBOJ 2027-01-28"));
StringAssert.Contains(error.Message, "Riga 2");
Assert.ThrowsExactly<FormatException>(() => CentralBanks.Parse("FED 27/01/2027"));
}
[TestMethod]
[TestCategory(Category.Unit)]
public void MoodReadingsCarryTheLabelsOfTheSpec()
{
var fng = FearGreed.Parse(Fixture.Text("free/fng.json"));
Assert.AreEqual(2, fng.Count);
Assert.AreEqual(32d, fng[^1].Value, "oldest first: the latest reading is last");
Assert.AreEqual("paura", fng[^1].Label);
Assert.AreEqual(DateTimeOffset.FromUnixTimeSeconds(1794268800), fng[^1].AsOf);
var vix = VixHistory.Parse(Fixture.Text("free/vix.csv"), new DateOnly(2026, 1, 1));
Assert.AreEqual(2, vix.Count);
Assert.AreEqual(new DateTimeOffset(2026, 11, 9, 0, 0, 0, TimeSpan.Zero), vix[1].AsOf);
Assert.AreEqual(("nervoso", "calmo"), (vix[0].Label, vix[1].Label));
Assert.AreEqual("paura estrema", MoodLabels.FearGreed(24));
Assert.AreEqual("neutro", MoodLabels.FearGreed(55));
Assert.AreEqual("avidità estrema", MoodLabels.FearGreed(76));
Assert.AreEqual("panico", MoodLabels.Vix(30.01));
Assert.AreEqual("troppi al rialzo", MoodLabels.Funding(0.0003, TimeSpan.FromHours(8)));
Assert.AreEqual("troppi al rialzo", MoodLabels.Funding(0.00015, TimeSpan.FromHours(4)), "4-hour funding counts double");
Assert.AreEqual("nella norma", MoodLabels.Funding(-0.0001, TimeSpan.FromHours(8)));
Assert.AreEqual("troppi al ribasso", MoodLabels.Funding(-0.00021, TimeSpan.FromHours(8)));
}
[TestMethod]
[TestCategory(Category.Unit)]
public void NasdaqListExcludesFundsAndEverythingThatIsNotACommonShare()
{
var list = NasdaqTraded.Parse(Fixture.Text("free/nasdaqtraded.txt"));
Assert.AreEqual(12, list.Count, "header and footer skipped");
var excluded = list.Where(s => s.IsExcluded).Select(s => s.Symbol).ToList();
CollectionAssert.AreEquivalent(new[] { "SPY", "BULZ", "AMUB", "PDI", "ABR$D", "AACIW", "ZXIET" }, excluded);
Assert.AreEqual("TXSE", list.Single(s => s.Symbol == "CECO").Exchange);
Assert.AreEqual("NASDAQ", list.Single(s => s.Symbol == "AAPL").Exchange);
Assert.AreEqual("NYSE", list.Single(s => s.Symbol == "KO").Exchange);
Assert.AreEqual("D", list.Single(s => s.Symbol == "AACG").FinancialStatus);
}
[TestMethod]
[TestCategory(Category.Unit)]
public void BinanceArchivesReadMillisecondsAndMicrosecondsAndCheckTheirFingerprint()
{
var old = BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv"));
var recent = BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2025-01.csv"));
Assert.AreEqual(5, old.Count);
Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), old[0].OpensAt);
Assert.AreEqual(4261.48m, old[0].Open);
Assert.AreEqual(3454770.05073206m, old[0].QuoteVolume);
Assert.AreEqual(3427L, old[0].Trades);
Assert.AreEqual(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), recent[0].OpensAt, "microseconds from 2025");
Assert.AreEqual(new DateTimeOffset(2025, 1, 5, 0, 0, 0, TimeSpan.Zero), recent[^1].OpensAt);
var zip = Zip("BTCUSDT-1d-2017-08.csv", Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv"));
var checksum = Convert.ToHexStringLower(SHA256.HashData(zip)) + " BTCUSDT-1d-2017-08.zip\n";
Assert.IsTrue(BinanceArchive.Verify(zip, checksum));
Assert.IsFalse(BinanceArchive.Verify(zip, Fixture.Text("free/BTCUSDT-1d-2017-08.zip.CHECKSUM")), "the recorded checksum belongs to the real zip");
CollectionAssert.AreEqual(old.ToList(), BinanceArchive.ReadZip(zip).ToList());
Assert.AreEqual(
new Uri("https://data.binance.vision/data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2017-08.zip.CHECKSUM"),
BinanceArchive.ChecksumOf(BinanceArchive.Monthly("BTCUSDT", "1d", 2017, 8)));
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task ReferenceStoresRoundTripOnTheEngineOfThisRun()
{
var clock = new ManualClock(_start);
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var fx = new FxStore(test.Db, clock);
await fx.UpsertAsync(Ecb.ParseHistoryCsv(Fixture.Text("free/ecb-hist.csv"), ["USD"]), CancellationToken.None).ConfigureAwait(false);
await fx.UpsertAsync([new FxRate("EUR", "USD", new DateOnly(2026, 11, 9), 1.1194m, Ecb.Source)], CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(1.1194m, (await fx.LatestAsync("EUR", "USD", CancellationToken.None).ConfigureAwait(false))!.Rate, "a new publication replaces the old");
Assert.AreEqual(1.1165m, (await fx.RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 8), CancellationToken.None).ConfigureAwait(false))!.Rate,
"Sunday uses Friday's rate");
Assert.IsNull(await fx.RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 5), CancellationToken.None).ConfigureAwait(false));
var calendar = new CalendarEventStore(test.Db, clock);
var events = ForexFactory.Parse(Fixture.Text("free/ff-calendar.json"));
Assert.AreEqual(5, await calendar.UpsertAsync(events, CancellationToken.None).ConfigureAwait(false));
var moved = events.Single(e => e.Title == "CPI m/m") with { Forecast = "0.4%" };
await calendar.UpsertAsync([moved], CancellationToken.None).ConfigureAwait(false);
var week = await calendar.BetweenAsync(new DateTimeOffset(2026, 11, 10, 0, 0, 0, TimeSpan.Zero), new DateTimeOffset(2026, 11, 17, 0, 0, 0, TimeSpan.Zero),
CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(3, week.Count);
Assert.AreEqual(moved, week[0]);
Assert.IsFalse(week.Single(e => e.Kind == CalendarEventKinds.Holiday).TimeKnown);
var mood = new MoodStore(test.Db, clock);
await mood.UpsertAsync(FearGreed.Parse(Fixture.Text("free/fng.json")), CancellationToken.None).ConfigureAwait(false);
await mood.UpsertAsync(FearGreed.Parse(Fixture.Text("free/fng.json")), CancellationToken.None).ConfigureAwait(false);
var latest = await mood.LatestAsync(MoodIndicators.FearGreed, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(32d, latest!.Value);
Assert.AreEqual("paura", latest.Label);
Assert.IsNull(await mood.LatestAsync(MoodIndicators.Vix, CancellationToken.None).ConfigureAwait(false));
var states = new SourceStateStore(test.Db, clock);
await states.RecordAsync(Finnhub.Source, enabled: false, success: false, null, "manca la chiave FINNHUB_API_KEY", CancellationToken.None).ConfigureAwait(false);
await states.RecordAsync(Ecb.Source, enabled: true, success: true, null, null, CancellationToken.None).ConfigureAwait(false);
clock.Advance(TimeSpan.FromHours(1));
await states.RecordAsync(Ecb.Source, enabled: true, success: false, "timeout", null, CancellationToken.None).ConfigureAwait(false);
var list = await states.ListAsync(CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(2, list.Count);
var ecb = list.Single(s => s.Source == Ecb.Source);
Assert.AreEqual(_start, ecb.LastSuccessAt, "a failure keeps the last success");
Assert.AreEqual(_start.AddHours(1), ecb.LastRequestAt);
Assert.AreEqual("timeout", ecb.LastError);
Assert.IsFalse(list.Single(s => s.Source == Finnhub.Source).Enabled);
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task StoredMarketCalendarKeepsOpenAndClosedDaysAcrossRestarts()
{
var clock = new ManualClock(_start);
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var calendar = new StoredMarketCalendar(test.Db, clock, "us_equity");
Assert.IsNull(calendar.Coverage);
var friday = new DateOnly(2026, 11, 13);
var monday = new DateOnly(2026, 11, 16);
MarketSession Session(DateOnly day) => new(day, Zones.At(day, new TimeOnly(9, 30), Zones.NewYork), Zones.At(day, new TimeOnly(16, 0), Zones.NewYork));
await calendar.SaveAsync([Session(friday), Session(monday)], friday, monday, CancellationToken.None).ConfigureAwait(false);
var reloaded = new StoredMarketCalendar(test.Db, clock, "us_equity");
await reloaded.LoadAsync(CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual((friday, monday), reloaded.Coverage);
Assert.AreEqual(Session(friday), reloaded.SessionOn(friday));
Assert.IsNull(reloaded.SessionOn(friday.AddDays(1)), "Saturday is closed");
Assert.IsTrue(reloaded.Covers(friday.AddDays(1)));
Assert.IsFalse(reloaded.Covers(monday.AddDays(1)), "a day not downloaded is unknown, not closed");
Assert.AreEqual(2, reloaded.Sessions(friday, monday).Count);
Assert.AreEqual(new DateTimeOffset(2026, 11, 13, 14, 30, 0, TimeSpan.Zero), reloaded.SessionOn(friday)!.OpenUtc);
var other = new StoredMarketCalendar(test.Db, clock, "other");
await other.LoadAsync(CancellationToken.None).ConfigureAwait(false);
Assert.IsNull(other.Coverage, "each market has its own calendar");
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task BinanceImporterChecksEachArchiveAndStopsAtTheFirstMissingMonth()
{
var folder = TempFolder.Create("binance");
try
{
var august = Zip("BTCUSDT-1d-2017-08.csv", Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv"));
var http = new FakeHttp()
.On("2017-08.zip.CHECKSUM", Convert.ToHexStringLower(SHA256.HashData(august)) + " BTCUSDT-1d-2017-08.zip")
.OnBytes("2017-08.zip", august)
.On(_ => true, _ => FakeHttp.Respond("<Error><Code>NoSuchKey</Code></Error>", HttpStatusCode.NotFound));
var clock = new FastClock(new DateTimeOffset(2017, 10, 15, 0, 0, 0, TimeSpan.Zero));
var store = new ResearchStore(folder);
var importer = new BinanceImporter(new ApiHttp(http.Client(), clock, "binance"), store, clock);
var result = await importer.ImportAsync("BTCUSDT", "1d", 2017, 7, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(5, result.Bars);
Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), result.First);
Assert.AreEqual(1, result.Notes.Count);
StringAssert.Contains(result.Notes[0], "2017-09");
Assert.AreEqual(4, http.Requests.Count, "July (not listed yet), August and its checksum, September (not published)");
var stored = await store.ReadAsync(Venues.Binance, "BTCUSDT", Timeframes.Day, DateTimeOffset.MinValue, DateTimeOffset.MaxValue).ToListAsync().ConfigureAwait(false);
CollectionAssert.AreEqual(BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv")).ToList(), stored);
// A zip that does not match its fingerprint is refused.
var tampered = new FakeHttp()
.On("CHECKSUM", new string('0', 64) + " BTCUSDT-1d-2017-08.zip")
.OnBytes("2017-08.zip", august);
var refused = new BinanceImporter(new ApiHttp(tampered.Client(), clock, "binance"), new ResearchStore(Path.Combine(folder, "other")), clock);
await Assert.ThrowsExactlyAsync<MarketDataException>(() => refused.ImportAsync("BTCUSDT", "1d", 2017, 8, CancellationToken.None)).ConfigureAwait(false);
}
finally
{
TempFolder.Delete(folder);
}
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task CsvImporterFollowsTheMappingAndReportsWhatItCouldNotRead()
{
var folder = TempFolder.Create("csvimport");
try
{
var csv = Path.Combine(folder, "ethusd_2024.csv");
await File.WriteAllTextAsync(csv,
"Data;Apertura;Massimo;Minimo;Chiusura;Volume\n" +
"02/01/2024 01:00;2.300,50;2.310,00;2.295,25;2.305,75;12,5\n" +
"02/01/2024 01:01;2.305,75;2.306,00;2.301,00;2.302,00;3\n" +
"02/01/2024 01:02;non so;2.306,00;2.301,00;2.302,00;3\n" +
"02/01/2024 01:04;2.302,00;2.303,00;2.300,00;2.301,00;4\n").ConfigureAwait(false);
await File.WriteAllTextAsync(Path.ChangeExtension(csv, ".mapping.json"), """
{
// Exported from a spreadsheet in Italian.
"symbol": "ETHUSD",
"delimiter": ";",
"time": "Data",
"time_format": "dd/MM/yyyy HH:mm",
"time_zone": "Europe/Rome",
"open": "Apertura", "high": "Massimo", "low": "Minimo", "close": "Chiusura", "volume": "Volume",
"decimal_separator": ",",
}
""").ConfigureAwait(false);
var mapping = CsvImporter.MappingFor(csv);
Assert.AreEqual("ETHUSD", mapping.Symbol);
Assert.AreEqual(";", mapping.Delimiter);
var store = new ResearchStore(Path.Combine(folder, "research"));
var result = await new CsvImporter(store).ImportAsync(csv, mapping, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(3, result.Bars);
Assert.AreEqual(new DateTimeOffset(2024, 1, 2, 0, 0, 0, TimeSpan.Zero), result.First, "01:00 in Rome is midnight UTC in winter");
StringAssert.Contains(result.Notes[0], "riga 4");
Assert.IsTrue(result.Notes.Any(n => n.Contains("1 righe non lette su 5", StringComparison.Ordinal)), string.Join(" | ", result.Notes));
Assert.IsTrue(result.Notes.Any(n => n.StartsWith(IssueKinds.Gap, StringComparison.Ordinal)), "the missing minutes are reported");
var stored = await store.ReadAsync("import", "ETHUSD", Timeframes.Minute, DateTimeOffset.MinValue, DateTimeOffset.MaxValue).ToListAsync().ConfigureAwait(false);
Assert.AreEqual(3, stored.Count);
Assert.AreEqual(2300.50m, stored[0].Open);
Assert.AreEqual(12.5m, stored[0].Volume);
// Without a mapping file: Binance-like columns by number and the symbol from the file name.
var plain = Path.Combine(folder, "solusdt-1m.csv");
await File.WriteAllTextAsync(plain, string.Join('\n', Fixture.Text("free/binance-BTCUSDT-1d-2025-01.csv").Split('\n').Take(2))).ConfigureAwait(false);
var defaults = CsvImporter.MappingFor(plain) with { HasHeader = false, Timeframe = Timeframes.Day };
var second = await new CsvImporter(store).ImportAsync(plain, defaults, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual("SOLUSDT", second.Symbol);
Assert.AreEqual(2, second.Bars);
Assert.AreEqual(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), second.First, "microseconds recognised by size");
}
finally
{
TempFolder.Delete(folder);
}
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task DownloaderRespectsLimitsRecordsFailuresAndSaysWhatIsOff()
{
var clock = new FastClock(_start);
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var settings = new SettingsStore(test.Db, clock, new EventBus());
var ecbZip = Zip("eurofxref-hist.csv", Fixture.Text("free/ecb-hist.csv"));
var fngFails = true;
var http = new FakeHttp()
.OnBytes("eurofxref-hist.zip", ecbZip)
.On("eurofxref-daily.xml", Fixture.Text("free/ecb-daily.xml"))
.On("ff_calendar_thisweek", Fixture.Text("free/ff-calendar.json"))
.On(u => u.Host == "api.alternative.me", _ => fngFails
? FakeHttp.Respond("{}", HttpStatusCode.BadRequest)
: FakeHttp.Respond(Fixture.Text("free/fng.json")))
.On("VIX_History.csv", Fixture.Text("free/vix.csv"));
var sources = new ReferenceSources(new ApiHttp(http.Client(), clock, "reference"), test.Db, clock, settings, (_, _) => Task.FromResult<string?>(null));
// Exchange rates: the whole history the first time, then the day's file.
Assert.IsTrue(await sources.RefreshFxAsync(CancellationToken.None).ConfigureAwait(false));
clock.Advance(TimeSpan.FromHours(1));
Assert.IsTrue(await sources.RefreshFxAsync(CancellationToken.None).ConfigureAwait(false));
CollectionAssert.AreEqual(new[] { "eurofxref-hist.zip", "eurofxref-daily.xml" }, http.Urls.Select(u => u.Segments[^1]).ToList());
var rate = await new FxStore(test.Db, clock).RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 9), CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(1.1193m, rate!.Rate);
// The calendar: Forex Factory at most once an hour, the central banks from the settings.
Assert.IsTrue(await sources.RefreshCalendarAsync(CancellationToken.None).ConfigureAwait(false));
clock.Advance(TimeSpan.FromMinutes(10));
Assert.IsTrue(await sources.RefreshCalendarAsync(CancellationToken.None).ConfigureAwait(false));
Assert.AreEqual(1, http.Urls.Count(u => u.Host == "nfs.faireconomy.media"), "ten minutes later Forex Factory is not asked again");
var events = await new CalendarEventStore(test.Db, clock).BetweenAsync(_start.AddDays(-7), _start.AddYears(2), CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(5 + 18, events.Count);
// Without keys FRED and Finnhub are off and the state says why; the earnings filter is off.
Assert.IsFalse(await sources.RefreshFredAsync(CancellationToken.None).ConfigureAwait(false));
Assert.IsFalse(await sources.RefreshEarningsAsync(null, CancellationToken.None).ConfigureAwait(false));
Assert.IsFalse(await sources.EarningsFilterActiveAsync(CancellationToken.None).ConfigureAwait(false));
var states = await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false);
StringAssert.Contains(states.Single(s => s.Source == Finnhub.Source).Note, "filtro sulle trimestrali è disattivato");
Assert.IsFalse(states.Single(s => s.Source == Fred.Source).Enabled);
Assert.IsFalse(http.Urls.Any(u => u.Host.Contains("finnhub", StringComparison.Ordinal) || u.Host.Contains("stlouisfed", StringComparison.Ordinal)));
// A failing source is recorded and does not stop the others; it is tried again at its next turn.
Assert.IsFalse(await sources.RefreshMoodAsync(CancellationToken.None).ConfigureAwait(false));
states = await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false);
Assert.IsNotNull(states.Single(s => s.Source == FearGreed.Source).LastError);
Assert.IsNull(states.Single(s => s.Source == VixHistory.Source).LastError);
Assert.AreEqual("calmo", (await new MoodStore(test.Db, clock).LatestAsync(MoodIndicators.Vix, CancellationToken.None).ConfigureAwait(false))!.Label);
fngFails = false;
clock.Advance(TimeSpan.FromHours(1));
Assert.IsTrue(await sources.RefreshMoodAsync(CancellationToken.None).ConfigureAwait(false));
var fng = (await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false)).Single(s => s.Source == FearGreed.Source);
Assert.IsNull(fng.LastError);
Assert.AreEqual(FearGreed.Attribution, fng.Note, "the source is named, as its terms ask");
StringAssert.Contains(http.Urls.Last(u => u.Host == "api.alternative.me").Query, "limit=400", "the first success asks for the history");
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task CentralBankListIsEditableAndRefusedWhenWrong()
{
var clock = new ManualClock(_start);
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var settings = new SettingsStore(test.Db, clock, new EventBus());
var key = SettingsStore.CentralBankMeetings.Key;
Assert.AreEqual(CentralBanks.DefaultList, (await settings.GetAsync(key).ConfigureAwait(false)).Value);
var error = await Assert.ThrowsExactlyAsync<ArgumentException>(() => settings.SetAsync(key, "FED 2027-13-01", "test")).ConfigureAwait(false);
StringAssert.Contains(error.Message, "Riga 1");
await settings.SetAsync(key, CentralBanks.DefaultList + "\nFED 2028-01-26", "test").ConfigureAwait(false);
Assert.AreEqual(19, CentralBanks.Parse((await settings.GetAsync(key).ConfigureAwait(false)).Value).Count);
}
internal static byte[] Zip(string entryName, string content)
{
using var zip = new MemoryStream();
using (var archive = new ZipArchive(zip, ZipArchiveMode.Create, leaveOpen: true))
using (var stream = archive.CreateEntry(entryName).Open())
{
stream.Write(Encoding.UTF8.GetBytes(content));
}
return zip.ToArray();
}
}