diff --git a/src/Anubis/Core/Settings.cs b/src/Anubis/Core/Settings.cs
index fe9eaf8..fc66c67 100644
--- a/src/Anubis/Core/Settings.cs
+++ b/src/Anubis/Core/Settings.cs
@@ -3,8 +3,11 @@ using Anubis.Db;
namespace Anubis.Core;
-/// A setting the interface may change: key, default and allowed values.
-public sealed record SettingDefinition(string Key, string Label, string Default, IReadOnlyList? Allowed = null);
+///
+/// A setting the interface may change: key, default and allowed values. , when present, returns
+/// the reason a free-text value is refused, in the words the interface shows, or null when it is valid.
+///
+public sealed record SettingDefinition(string Key, string Label, string Default, IReadOnlyList? Allowed = null, Func? Check = null);
/// A value fixed by the startup configuration: it wins and is shown read-only with where to change it (SPEC 13.5).
public sealed record StartupOverride(string Value, string Source);
@@ -34,7 +37,11 @@ public sealed class SettingsStore(Database db, IClock clock, EventBus bus, IRead
/// Bot state behind the top bar buttons: active, paused (no new entries) or stopped (kill switch, SPEC 9.3).
public static readonly SettingDefinition BotState = new("bot.state", "Stato del bot", "active", ["active", "paused", "stopped"]);
- public static readonly IReadOnlyList All = [Theme, Currency, BotState];
+ /// Fed and ECB decision days, one per line (SPEC 4.4: a yearly list the user edits).
+ public static readonly SettingDefinition CentralBankMeetings = new("calendar.central_banks", "Riunioni di Fed e BCE",
+ Data.CentralBanks.DefaultList, Check: Data.CentralBanks.Check);
+
+ public static readonly IReadOnlyList All = [Theme, Currency, BotState, CentralBankMeetings];
private readonly IReadOnlyDictionary _overrides = overrides ?? new Dictionary();
@@ -68,6 +75,11 @@ public sealed class SettingsStore(Database db, IClock clock, EventBus bus, IRead
throw new ArgumentException($"Valore non ammesso per {definition.Label}: {value}.", nameof(value));
}
+ if (definition.Check?.Invoke(value) is { } problem)
+ {
+ throw new ArgumentException(problem, nameof(value));
+ }
+
var now = clock.UtcNow;
var json = new DbJson(JsonSerializer.Serialize(value));
var changed = await db.WriteAsync(async s =>
diff --git a/src/Anubis/Data/FreeSources.cs b/src/Anubis/Data/FreeSources.cs
new file mode 100644
index 0000000..70d340e
--- /dev/null
+++ b/src/Anubis/Data/FreeSources.cs
@@ -0,0 +1,467 @@
+using System.Globalization;
+using System.IO.Compression;
+using System.Security.Cryptography;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using System.Xml.Linq;
+
+namespace Anubis.Data;
+
+///
+/// Readers of the free sources of SPEC 4.4. Each is a pure function from the published text to Anubis records, tested
+/// on recorded responses; downloads them within their limits.
+///
+public static class Ecb
+{
+ public const string Source = "ecb";
+
+ public static readonly Uri Daily = new("https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml");
+
+ public static readonly Uri Last90Days = new("https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml");
+
+ public static readonly Uri History = new("https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.zip");
+
+ private static readonly XNamespace _ns = "http://www.ecb.int/vocabulary/2002-08-01/eurofxref";
+
+ /// Rates of the daily or 90-day XML for the asked, EUR as base.
+ public static IReadOnlyList ParseXml(string xml, IReadOnlyCollection currencies)
+ {
+ var document = XDocument.Parse(xml);
+ return document.Descendants(_ns + "Cube")
+ .Where(day => day.Attribute("time") is not null)
+ .SelectMany(day => day.Elements(_ns + "Cube").Select(rate => (Day: day, Rate: rate)))
+ .Where(x => currencies.Contains((string?)x.Rate.Attribute("currency") ?? ""))
+ .Select(x => new FxRate("EUR", (string)x.Rate.Attribute("currency")!,
+ DateOnly.ParseExact((string)x.Day.Attribute("time")!, "yyyy-MM-dd", CultureInfo.InvariantCulture),
+ decimal.Parse((string)x.Rate.Attribute("rate")!, NumberStyles.Float, CultureInfo.InvariantCulture), Source))
+ .OrderBy(r => r.Date)
+ .ToList();
+ }
+
+ /// The full history CSV (inside eurofxref-hist.zip): newest first, a trailing comma on every line, N/A where missing.
+ public static IReadOnlyList ParseHistoryCsv(string csv, IReadOnlyCollection currencies)
+ {
+ ArgumentNullException.ThrowIfNull(csv);
+ var lines = csv.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+ var header = lines[0].Split(',');
+ var rates = new List();
+ foreach (var line in lines.Skip(1))
+ {
+ var fields = line.Split(',');
+ var date = DateOnly.ParseExact(fields[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
+ for (var i = 1; i < Math.Min(header.Length, fields.Length); i++)
+ {
+ if (currencies.Contains(header[i]) && decimal.TryParse(fields[i], NumberStyles.Float, CultureInfo.InvariantCulture, out var rate))
+ {
+ rates.Add(new FxRate("EUR", header[i], date, rate, Source));
+ }
+ }
+ }
+
+ return rates.OrderBy(r => r.Date).ToList();
+ }
+
+ public static string ExtractHistoryCsv(byte[] zip)
+ {
+ using var archive = new ZipArchive(new MemoryStream(zip), ZipArchiveMode.Read);
+ var entry = archive.Entries.Single(e => e.Name.EndsWith(".csv", StringComparison.OrdinalIgnoreCase));
+ using var reader = new StreamReader(entry.Open());
+ return reader.ReadToEnd();
+ }
+}
+
+/// The weekly economic calendar of Forex Factory (faireconomy.media). It has no actual values.
+public static class ForexFactory
+{
+ public const string Source = "forexfactory";
+
+ public static readonly Uri ThisWeek = new("https://nfs.faireconomy.media/ff_calendar_thisweek.json");
+
+ public static IReadOnlyList Parse(string json)
+ {
+ using var document = JsonDocument.Parse(json);
+ var events = new List();
+ foreach (var item in document.RootElement.EnumerateArray())
+ {
+ var title = Text(item, "title");
+ var country = Text(item, "country");
+ var at = DateTimeOffset.Parse(Text(item, "date"), CultureInfo.InvariantCulture).ToUniversalTime();
+ var impact = Text(item, "impact");
+ var holiday = impact == "Holiday";
+ events.Add(new CalendarEvent(holiday ? CalendarEventKinds.Holiday : CalendarEventKinds.Economic, title, at, TimeKnown: !holiday, Source,
+ $"ff:{country}:{at:yyyy-MM-ddTHH:mm}:{title}")
+ {
+ Currency = country == "All" ? null : country,
+ Impact = holiday ? null : impact,
+ Forecast = Empty(Text(item, "forecast")),
+ Previous = Empty(Text(item, "previous")),
+ });
+ }
+
+ return events;
+ }
+
+ private static string? Empty(string text) => text.Length == 0 ? null : text;
+
+ internal static string Text(JsonElement item, string name) =>
+ item.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString()! : "";
+}
+
+///
+/// Future release dates of the main US data from FRED (optional key). FRED gives only the day: the time is
+/// 08:30 New York and the impact high (SPEC 4.4).
+///
+public static class Fred
+{
+ public const string Source = "fred";
+
+ /// Releases followed, with the title the calendar shows (ids checked on fred.stlouisfed.org).
+ public static readonly IReadOnlyList<(int Id, string Title)> Releases =
+ [
+ (10, "CPI (inflazione USA)"),
+ (50, "Employment Situation (occupazione USA)"),
+ (53, "PIL USA"),
+ (54, "Personal Income and Outlays (PCE)"),
+ ];
+
+ public static Uri ReleaseDates(int releaseId, string apiKey, DateOnly from) => new(
+ "https://api.stlouisfed.org/fred/release/dates?release_id=" + releaseId.ToString(CultureInfo.InvariantCulture) +
+ "&api_key=" + Uri.EscapeDataString(apiKey) + "&file_type=json&include_release_dates_with_no_data=true&sort_order=asc" +
+ "&realtime_start=" + from.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
+
+ public static IReadOnlyList Parse(string json, string title, DateOnly from)
+ {
+ using var document = JsonDocument.Parse(json);
+ var events = new List();
+ foreach (var item in document.RootElement.GetProperty("release_dates").EnumerateArray())
+ {
+ var day = DateOnly.ParseExact(ForexFactory.Text(item, "date"), "yyyy-MM-dd", CultureInfo.InvariantCulture);
+ if (day < from)
+ {
+ continue;
+ }
+
+ var id = item.GetProperty("release_id").GetInt32();
+ events.Add(new CalendarEvent(CalendarEventKinds.Economic, title, Zones.At(day, new TimeOnly(8, 30), Zones.NewYork), TimeKnown: true, Source,
+ $"fred:{id}:{day:yyyy-MM-dd}")
+ {
+ Currency = "USD",
+ Impact = "High",
+ });
+ }
+
+ return events;
+ }
+}
+
+/// Earnings dates from Finnhub (optional key, free plan: 60 calls a minute).
+public static class Finnhub
+{
+ public const string Source = "finnhub";
+
+ public static Uri Earnings(DateOnly from, DateOnly to, string apiKey) => new(
+ $"https://finnhub.io/api/v1/calendar/earnings?from={from:yyyy-MM-dd}&to={to:yyyy-MM-dd}&token={Uri.EscapeDataString(apiKey)}");
+
+ ///
+ /// hour: bmo before the open (08:00 New York), amc after the close (16:05), dmh during market hours
+ /// (12:00); empty means unknown and the event keeps only its day.
+ ///
+ public static IReadOnlyList Parse(string json, IReadOnlySet? symbols = null)
+ {
+ using var document = JsonDocument.Parse(json);
+ var events = new List();
+ foreach (var item in document.RootElement.GetProperty("earningsCalendar").EnumerateArray())
+ {
+ var symbol = ForexFactory.Text(item, "symbol");
+ if (symbols is not null && !symbols.Contains(symbol))
+ {
+ continue;
+ }
+
+ var day = DateOnly.ParseExact(ForexFactory.Text(item, "date"), "yyyy-MM-dd", CultureInfo.InvariantCulture);
+ var hour = ForexFactory.Text(item, "hour");
+ TimeOnly? time = hour switch
+ {
+ "bmo" => new TimeOnly(8, 0),
+ "amc" => new TimeOnly(16, 5),
+ "dmh" => new TimeOnly(12, 0),
+ _ => null,
+ };
+ events.Add(new CalendarEvent(CalendarEventKinds.Earnings, "Trimestrale " + symbol, Zones.At(day, time ?? TimeOnly.MinValue, Zones.NewYork),
+ time is not null, Source, $"finnhub:{symbol}:{day:yyyy-MM-dd}")
+ {
+ Symbol = symbol,
+ Currency = "USD",
+ TimeHint = hour.Length == 0 ? null : hour,
+ Impact = "High",
+ });
+ }
+
+ return events;
+ }
+}
+
+///
+/// Meetings of the Fed and the ECB from the yearly list in the settings (SPEC 4.4): one per line,
+/// FED 2026-12-09 or BCE 2026-12-17. Fed decisions come at 14:00 New York, ECB ones at 14:15 Frankfurt.
+///
+public static class CentralBanks
+{
+ public const string Source = "central_banks";
+
+ /// Decision days from the official calendars, read on 10/11/2026 (API-NOTES).
+ public const string DefaultList =
+ "FED 2026-12-09\nFED 2027-01-27\nFED 2027-03-17\nFED 2027-04-28\nFED 2027-06-09\nFED 2027-07-28\nFED 2027-09-15\nFED 2027-10-27\nFED 2027-12-08\n" +
+ "BCE 2026-12-17\nBCE 2027-02-04\nBCE 2027-03-18\nBCE 2027-04-29\nBCE 2027-06-10\nBCE 2027-07-22\nBCE 2027-09-09\nBCE 2027-10-28\nBCE 2027-12-16";
+
+ /// Null when the list is valid, otherwise what is wrong with it.
+ public static string? Check(string list)
+ {
+ try
+ {
+ Parse(list);
+ return null;
+ }
+ catch (FormatException ex)
+ {
+ return ex.Message;
+ }
+ }
+
+ public static IReadOnlyList Parse(string list)
+ {
+ ArgumentNullException.ThrowIfNull(list);
+ var events = new List();
+ var number = 0;
+ foreach (var raw in list.Split('\n'))
+ {
+ number++;
+ var line = raw.Trim();
+ if (line.Length == 0 || line.StartsWith('#'))
+ {
+ continue;
+ }
+
+ var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
+ if (parts.Length != 2 || !DateOnly.TryParseExact(parts[1], "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var day))
+ {
+ throw new FormatException($"Riga {number} dell'elenco delle banche centrali non valida: «{line}». Scrivi per esempio «FED 2026-12-09».");
+ }
+
+ var bank = parts[0].ToUpperInvariant();
+ var (title, currency, at) = bank switch
+ {
+ "FED" => ("Decisione sui tassi della Fed", "USD", Zones.At(day, new TimeOnly(14, 0), Zones.NewYork)),
+ "BCE" or "ECB" => ("Decisione sui tassi della BCE", "EUR", Zones.At(day, new TimeOnly(14, 15), Zones.Frankfurt)),
+ _ => throw new FormatException($"Riga {number}: banca «{parts[0]}» sconosciuta, usa FED o BCE."),
+ };
+ events.Add(new CalendarEvent(CalendarEventKinds.CentralBank, title, at, TimeKnown: true, Source, $"cb:{currency}:{day:yyyy-MM-dd}")
+ {
+ Currency = currency,
+ Impact = "High",
+ });
+ }
+
+ return events;
+ }
+}
+
+/// Crypto Fear and Greed index of alternative.me (attribution required: the interface names the source).
+public static class FearGreed
+{
+ public const string Source = "alternative.me";
+
+ public const string Attribution = "Indice di paura e avidità: alternative.me";
+
+ public static Uri Latest(int days) => new($"https://api.alternative.me/fng/?limit={days.ToString(CultureInfo.InvariantCulture)}&format=json");
+
+ public static IReadOnlyList Parse(string json)
+ {
+ using var document = JsonDocument.Parse(json);
+ return document.RootElement.GetProperty("data").EnumerateArray()
+ .Select(item =>
+ {
+ var value = double.Parse(ForexFactory.Text(item, "value"), CultureInfo.InvariantCulture);
+ var at = DateTimeOffset.FromUnixTimeSeconds(long.Parse(ForexFactory.Text(item, "timestamp"), CultureInfo.InvariantCulture));
+ return new MoodReading(MoodIndicators.FearGreed, at, value, MoodLabels.FearGreed(value), Source);
+ })
+ .OrderBy(m => m.AsOf)
+ .ToList();
+ }
+}
+
+/// Daily VIX history from Cboe (no key). The file answers with a redirect to cdn-api.cboe.com.
+public static class VixHistory
+{
+ public const string Source = "cboe";
+
+ public static readonly Uri Csv = new("https://cdn.cboe.com/api/global/us_indices/daily_prices/VIX_History.csv");
+
+ /// Closes on or after , dated at midnight UTC of the session day.
+ public static IReadOnlyList Parse(string csv, DateOnly from)
+ {
+ ArgumentNullException.ThrowIfNull(csv);
+ var readings = new List();
+ foreach (var line in csv.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Skip(1))
+ {
+ var f = line.Split(',');
+ var day = DateOnly.ParseExact(f[0], "MM/dd/yyyy", CultureInfo.InvariantCulture);
+ if (day < from || f.Length < 5)
+ {
+ continue;
+ }
+
+ var close = double.Parse(f[4], CultureInfo.InvariantCulture);
+ readings.Add(new MoodReading(MoodIndicators.Vix, QualityChecks.Midnight(day), close, MoodLabels.Vix(close), Source));
+ }
+
+ return readings;
+ }
+}
+
+/// A row of nasdaqtraded.txt: every security traded in the USA with its ETF flag.
+public sealed record ListedSecurity(string Symbol, string Name, string Exchange, bool IsEtf, bool IsTest, string FinancialStatus)
+{
+ ///
+ /// ETF, ETN, fund, or not a common share (preferred, warrant, right, unit, note): excluded from the stock universe
+ /// (SPEC 4.6, 8.2). The ETF flag misses many ETNs and every closed-end fund, so the name is read too.
+ ///
+ public bool IsExcluded => IsEtf || IsTest || NotCommon().IsMatch(Name);
+
+ private static Regex NotCommon() => NasdaqTraded.NotCommonPattern();
+}
+
+/// The public list of US securities (SPEC 4.1): https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqtraded.txt.
+public static partial class NasdaqTraded
+{
+ public const string Source = "nasdaqtrader";
+
+ public static readonly Uri File = new("https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqtraded.txt");
+
+ /// Listing exchange codes: Q Nasdaq, N NYSE, A NYSE American, P NYSE Arca, Z Cboe BZX, V IEX, F Texas Stock Exchange, M NYSE Texas.
+ public static readonly IReadOnlyDictionary Exchanges = new Dictionary
+ {
+ ["Q"] = "NASDAQ",
+ ["N"] = "NYSE",
+ ["A"] = "AMEX",
+ ["P"] = "ARCA",
+ ["Z"] = "BATS",
+ ["V"] = "IEX",
+ ["F"] = "TXSE",
+ ["M"] = "NYSETX",
+ };
+
+ [GeneratedRegex(@"\b(ETF|ETN|ETNs|Exchange[- ]Traded|Fund|Funds|Trust Units?|Warrants?|Rights?|Units?|Preferred|Notes?|Debentures?|Depositary Shares? representing a fraction|Closed[- ]End|Income Shares|Beneficial Interest|Index|Leveraged|Inverse|SPAC)\b", RegexOptions.IgnoreCase)]
+ internal static partial Regex NotCommonPattern();
+
+ /// Rows of the pipe-separated file (CRLF), header and footer skipped.
+ public static IReadOnlyList Parse(string text)
+ {
+ ArgumentNullException.ThrowIfNull(text);
+ var lines = text.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+ var header = lines[0].Split('|');
+ int Column(string name) => Array.IndexOf(header, name) is var i and >= 0 ? i : throw new FormatException("nasdaqtraded.txt: manca la colonna " + name);
+ int symbol = Column("Symbol"), name = Column("Security Name"), exchange = Column("Listing Exchange"), etf = Column("ETF"),
+ test = Column("Test Issue"), status = Column("Financial Status");
+ var list = new List();
+ foreach (var line in lines.Skip(1))
+ {
+ if (line.StartsWith("File Creation Time", StringComparison.Ordinal))
+ {
+ continue;
+ }
+
+ var f = line.Split('|');
+ if (f.Length < header.Length)
+ {
+ continue;
+ }
+
+ list.Add(new ListedSecurity(f[symbol], f[name], Exchanges.GetValueOrDefault(f[exchange], f[exchange]), f[etf] == "Y", f[test] == "Y", f[status]));
+ }
+
+ return list;
+ }
+}
+
+///
+/// The public archives of data.binance.vision (SPEC 4.2, 6.6): monthly zips of klines with a SHA-256 checksum file.
+/// Research only: Binance does not serve Italian residents since July 2026.
+///
+public static class BinanceArchive
+{
+ public const string Host = "https://data.binance.vision";
+
+ public static Uri Monthly(string symbol, string interval, int year, int month) =>
+ new($"{Host}/data/spot/monthly/klines/{symbol}/{interval}/{symbol}-{interval}-{year:D4}-{month:D2}.zip");
+
+ public static Uri Daily(string symbol, string interval, DateOnly day) =>
+ new($"{Host}/data/spot/daily/klines/{symbol}/{interval}/{symbol}-{interval}-{day:yyyy-MM-dd}.zip");
+
+ public static Uri ChecksumOf(Uri zip) => new(zip + ".CHECKSUM");
+
+ /// Checks the zip against its CHECKSUM file: "<sha256 hex> <file name>".
+ public static bool Verify(byte[] zip, string checksumFile)
+ {
+ ArgumentNullException.ThrowIfNull(checksumFile);
+ var expected = checksumFile.Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries)[0];
+ return string.Equals(Convert.ToHexStringLower(SHA256.HashData(zip)), expected, StringComparison.OrdinalIgnoreCase);
+ }
+
+ public static IReadOnlyList ReadZip(byte[] zip)
+ {
+ using var archive = new ZipArchive(new MemoryStream(zip), ZipArchiveMode.Read);
+ var entry = archive.Entries.Single(e => e.Name.EndsWith(".csv", StringComparison.OrdinalIgnoreCase));
+ using var reader = new StreamReader(entry.Open());
+ return ParseCsv(reader.ReadToEnd());
+ }
+
+ ///
+ /// Klines without header: open_time, open, high, low, close, volume, close_time, quote_volume, trades, … The
+ /// open time is in milliseconds up to 2024 and in microseconds from 2025; the last column is ignored.
+ ///
+ public static IReadOnlyList ParseCsv(string csv)
+ {
+ ArgumentNullException.ThrowIfNull(csv);
+ var bars = new List();
+ foreach (var line in csv.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
+ {
+ var f = line.Split(',');
+ if (f.Length < 9 || !long.TryParse(f[0], NumberStyles.None, CultureInfo.InvariantCulture, out var time))
+ {
+ continue; // A header line some archives carry.
+ }
+
+ bars.Add(new ResearchBar(FromEpoch(time), Number(f[1]), Number(f[2]), Number(f[3]), Number(f[4]), Number(f[5]))
+ {
+ QuoteVolume = Number(f[7]),
+ Trades = long.Parse(f[8], NumberStyles.None, CultureInfo.InvariantCulture),
+ });
+ }
+
+ return bars;
+ }
+
+ /// Microseconds when the number is too large to be milliseconds (after year 5138).
+ public static DateTimeOffset FromEpoch(long value) =>
+ value >= 100_000_000_000_000 ? DateTimeOffset.UnixEpoch.AddTicks(value * 10) : DateTimeOffset.FromUnixTimeMilliseconds(value);
+
+ private static decimal Number(string text) => decimal.Parse(text, NumberStyles.Float, CultureInfo.InvariantCulture);
+}
+
+/// Time zones of the exchanges and central banks, from the system's database (both Windows and Linux know the IANA ids).
+public static class Zones
+{
+ public static readonly TimeZoneInfo NewYork = TimeZoneInfo.FindSystemTimeZoneById("America/New_York");
+
+ public static readonly TimeZoneInfo Frankfurt = TimeZoneInfo.FindSystemTimeZoneById("Europe/Berlin");
+
+ public static readonly TimeZoneInfo Rome = TimeZoneInfo.FindSystemTimeZoneById("Europe/Rome");
+
+ public static DateTimeOffset At(DateOnly day, TimeOnly time, TimeZoneInfo zone)
+ {
+ ArgumentNullException.ThrowIfNull(zone);
+ var local = day.ToDateTime(time);
+ return new DateTimeOffset(local, zone.GetUtcOffset(local)).ToUniversalTime();
+ }
+}
diff --git a/src/Anubis/Data/Importers.cs b/src/Anubis/Data/Importers.cs
new file mode 100644
index 0000000..3f5ea09
--- /dev/null
+++ b/src/Anubis/Data/Importers.cs
@@ -0,0 +1,246 @@
+using System.Globalization;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Anubis.Brokers;
+using Anubis.Core;
+
+namespace Anubis.Data;
+
+/// What an import did, for the log and the interface.
+public sealed record ImportResult(string Venue, string Symbol, string Timeframe, int Bars, DateTimeOffset? First, DateTimeOffset? Last, IReadOnlyList Notes);
+
+///
+/// Downloads the monthly kline archives of data.binance.vision into the research history (issue #21). Every zip
+/// is checked against its SHA-256 file before use; a month not published yet (404) ends the run without error.
+/// Years already complete on disk are skipped.
+///
+public sealed class BinanceImporter(ApiHttp http, ResearchStore store, IClock clock)
+{
+ /// The archive serves files from S3: a few requests a second is far below any limit.
+ private readonly RateLimiter _limiter = new(clock, 5, TimeSpan.FromSeconds(1));
+
+ public async Task ImportAsync(string symbol, string interval, int fromYear, int fromMonth, CancellationToken cancellationToken)
+ {
+ var timeframe = interval == "1m" ? Timeframes.Minute : interval == "1d" ? Timeframes.Day : interval;
+ var notes = new List();
+ var total = 0;
+ DateTimeOffset? first = null, last = null;
+ var today = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime);
+ var have = store.Years(Venues.Binance, symbol, timeframe).ToHashSet();
+ for (var month = new DateOnly(fromYear, fromMonth, 1); month <= today; month = month.AddMonths(1))
+ {
+ // Months go in order, so a year is complete once the next one has a file. A year interrupted halfway is
+ // read again: writing merges with what is on disk.
+ if (have.Contains(month.Year) && have.Contains(month.Year + 1))
+ {
+ continue;
+ }
+
+ var url = BinanceArchive.Monthly(symbol, interval, month.Year, month.Month);
+ var zip = await http.GetBytesAsync(url, _limiter, null, cancellationToken).ConfigureAwait(false);
+ if (zip is null)
+ {
+ if (total > 0 || month >= new DateOnly(today.Year, today.Month, 1).AddMonths(-1))
+ {
+ notes.Add($"{month:yyyy-MM}: archivio non ancora pubblicato");
+ break;
+ }
+
+ continue; // Before the pair was listed.
+ }
+
+ var checksum = await http.GetStringAsync(BinanceArchive.ChecksumOf(url), _limiter, null, cancellationToken).ConfigureAwait(false);
+ if (!BinanceArchive.Verify(zip, checksum))
+ {
+ throw new MarketDataException($"Binance: l'archivio {month:yyyy-MM} di {symbol} non corrisponde alla sua impronta SHA-256.", retryable: true);
+ }
+
+ var bars = BinanceArchive.ReadZip(zip);
+ if (bars.Count == 0)
+ {
+ continue;
+ }
+
+ await store.WriteAsync(Venues.Binance, symbol, timeframe, bars, cancellationToken).ConfigureAwait(false);
+ total += bars.Count;
+ first ??= bars[0].OpensAt;
+ last = bars[^1].OpensAt;
+ }
+
+ return new ImportResult(Venues.Binance, symbol, timeframe, total, first, last, notes);
+ }
+}
+
+///
+/// Column mapping of a user CSV in /data/import (SPEC 6.6): a JSON file next to the CSV with the same name and
+/// .mapping.json, or mapping.json for the whole folder. Columns are named by header or by 0-based number.
+///
+public sealed record CsvMapping
+{
+ /// Symbol of the series, e.g. BTCEUR; when empty, the part of the file name before the first '-' or '_'.
+ public string? Symbol { get; init; }
+
+ public string Venue { get; init; } = "import";
+
+ public string Timeframe { get; init; } = Timeframes.Minute;
+
+ public string Delimiter { get; init; } = ",";
+
+ public bool HasHeader { get; init; } = true;
+
+ public string Time { get; init; } = "0";
+
+ /// unix_ms, unix_s, unix_us, auto (milliseconds or microseconds by size), or a .NET date format such as yyyy-MM-dd HH:mm:ss.
+ public string TimeFormat { get; init; } = "auto";
+
+ /// Time zone of text times (IANA id); unix times are always UTC.
+ public string TimeZone { get; init; } = "UTC";
+
+ public string Open { get; init; } = "1";
+
+ public string High { get; init; } = "2";
+
+ public string Low { get; init; } = "3";
+
+ public string Close { get; init; } = "4";
+
+ public string Volume { get; init; } = "5";
+
+ public string? QuoteVolume { get; init; }
+
+ /// Decimal separator of the numbers: "." or ","; with "," a "." is read as the thousands separator.
+ public string DecimalSeparator { get; init; } = ".";
+
+ /// True for a market that never closes (crypto): every missing minute or day is reported as a gap.
+ public bool Continuous { get; init; } = true;
+
+ public static readonly JsonSerializerOptions Json = new()
+ {
+ PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
+ ReadCommentHandling = JsonCommentHandling.Skip,
+ AllowTrailingCommas = true,
+ DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+ };
+}
+
+///
+/// The generic importer of user CSV files with one-minute or daily bars into the research history (issue #21).
+/// Lines that cannot be read are counted and reported, never silently fixed.
+///
+public sealed class CsvImporter(ResearchStore store)
+{
+ public const int MaxReportedErrors = 5;
+
+ public static CsvMapping MappingFor(string csvPath)
+ {
+ ArgumentNullException.ThrowIfNull(csvPath);
+ var own = Path.ChangeExtension(csvPath, ".mapping.json");
+ var shared = Path.Combine(Path.GetDirectoryName(csvPath) ?? ".", "mapping.json");
+ var path = File.Exists(own) ? own : File.Exists(shared) ? shared : null;
+ return path is null ? new CsvMapping() : JsonSerializer.Deserialize(File.ReadAllText(path), CsvMapping.Json) ?? new CsvMapping();
+ }
+
+ public async Task ImportAsync(string csvPath, CsvMapping mapping, CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(mapping);
+ var symbol = string.IsNullOrWhiteSpace(mapping.Symbol)
+ ? Path.GetFileNameWithoutExtension(csvPath).Split('-', '_')[0].ToUpperInvariant()
+ : mapping.Symbol;
+ var zone = TimeZoneInfo.FindSystemTimeZoneById(mapping.TimeZone);
+ var numbers = new NumberFormatInfo { NumberDecimalSeparator = mapping.DecimalSeparator, NumberGroupSeparator = mapping.DecimalSeparator == "," ? "." : "," };
+ var style = mapping.DecimalSeparator == "," ? NumberStyles.Float | NumberStyles.AllowThousands : NumberStyles.Float;
+ var bars = new List();
+ var notes = new List();
+ var errors = 0;
+ string[]? header = null;
+ var lineNumber = 0;
+ foreach (var line in await File.ReadAllLinesAsync(csvPath, cancellationToken).ConfigureAwait(false))
+ {
+ lineNumber++;
+ if (line.Length == 0)
+ {
+ continue;
+ }
+
+ var fields = line.Split(mapping.Delimiter);
+ if (header is null && mapping.HasHeader)
+ {
+ header = fields.Select(f => f.Trim().Trim('"')).ToArray();
+ continue;
+ }
+
+ try
+ {
+ string Field(string column) => fields[Index(column, header)].Trim().Trim('"');
+ decimal Number(string column) => decimal.Parse(Field(column), style, numbers);
+ bars.Add(new ResearchBar(ParseTime(Field(mapping.Time), mapping.TimeFormat, zone), Number(mapping.Open), Number(mapping.High),
+ Number(mapping.Low), Number(mapping.Close), Number(mapping.Volume))
+ {
+ QuoteVolume = mapping.QuoteVolume is { Length: > 0 } q ? Number(q) : null,
+ });
+ }
+ catch (Exception ex) when (ex is FormatException or IndexOutOfRangeException or OverflowException or ArgumentException)
+ {
+ if (++errors <= MaxReportedErrors)
+ {
+ notes.Add($"riga {lineNumber}: {ex.Message}");
+ }
+ }
+ }
+
+ if (errors > 0)
+ {
+ notes.Add($"{errors} righe non lette su {lineNumber}");
+ }
+
+ var ordered = bars.OrderBy(b => b.OpensAt).ToList();
+ IReadOnlyList issues = [];
+ if (ordered.Count > 0 && mapping.Timeframe == Timeframes.Minute)
+ {
+ issues = QualityChecks.CheckMinutes(ordered.Select(b => b.ToBar()).ToList(),
+ mapping.Continuous ? QualityChecks.EveryMinute(ordered[0].OpensAt, ordered[^1].OpensAt.AddMinutes(1)) : null);
+ }
+ else if (ordered.Count > 0)
+ {
+ var days = ordered.Select(b => new DailyBar(DateOnly.FromDateTime(b.OpensAt.UtcDateTime), (double)b.Open, (double)b.High, (double)b.Low,
+ (double)b.Close, (double)b.Volume)).ToList();
+ issues = QualityChecks.CheckDaily(days, mapping.Continuous ? QualityChecks.EveryDay(days[0].Day, days[^1].Day) : null);
+ }
+
+ notes.AddRange(issues.Take(MaxReportedErrors).Select(i => $"{i.Kind} {i.StartsAt:yyyy-MM-dd HH:mm}: {i.Detail}"));
+
+ await store.WriteAsync(mapping.Venue, symbol, mapping.Timeframe, bars, cancellationToken).ConfigureAwait(false);
+ return new ImportResult(mapping.Venue, symbol, mapping.Timeframe, bars.Count, ordered.FirstOrDefault()?.OpensAt, ordered.LastOrDefault()?.OpensAt, notes);
+ }
+
+ private static int Index(string column, string[]? header)
+ {
+ if (int.TryParse(column, NumberStyles.None, CultureInfo.InvariantCulture, out var index))
+ {
+ return index;
+ }
+
+ var found = header is null ? -1 : Array.FindIndex(header, h => string.Equals(h, column, StringComparison.OrdinalIgnoreCase));
+ return found >= 0 ? found : throw new ArgumentException($"colonna «{column}» assente");
+ }
+
+ private static DateTimeOffset ParseTime(string text, string format, TimeZoneInfo zone)
+ {
+ switch (format)
+ {
+ case "unix_s":
+ return DateTimeOffset.FromUnixTimeSeconds(long.Parse(text, CultureInfo.InvariantCulture));
+ case "unix_ms":
+ return DateTimeOffset.FromUnixTimeMilliseconds(long.Parse(text, CultureInfo.InvariantCulture));
+ case "unix_us":
+ return DateTimeOffset.UnixEpoch.AddTicks(long.Parse(text, CultureInfo.InvariantCulture) * 10);
+ case "auto" when long.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out var number):
+ return number < 100_000_000_000 ? DateTimeOffset.FromUnixTimeSeconds(number) : BinanceArchive.FromEpoch(number);
+ case "auto":
+ return DateTimeOffset.Parse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
+ default:
+ var local = DateTime.ParseExact(text, format, CultureInfo.InvariantCulture, DateTimeStyles.None);
+ return new DateTimeOffset(local, zone.GetUtcOffset(local)).ToUniversalTime();
+ }
+ }
+}
diff --git a/src/Anubis/Data/ReferenceData.cs b/src/Anubis/Data/ReferenceData.cs
new file mode 100644
index 0000000..795ed57
--- /dev/null
+++ b/src/Anubis/Data/ReferenceData.cs
@@ -0,0 +1,278 @@
+using System.Collections.Concurrent;
+using Anubis.Core;
+using Anubis.Db;
+
+namespace Anubis.Data;
+
+/// A reference exchange rate: units of for one , as published for .
+public sealed record FxRate(string Base, string Quote, DateOnly Date, decimal Rate, string Source);
+
+///
+/// Reference exchange rates (SPEC 6.8): one per day as the ECB publishes them. A day without a rate uses the last
+/// one before it, as the tax export requires (SPEC 4.6).
+///
+public sealed class FxStore(Database db, IClock clock)
+{
+ public async Task UpsertAsync(IEnumerable rates, CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(rates);
+ var list = rates.ToList();
+ var now = clock.UtcNow;
+ await db.WriteAsync(async s =>
+ {
+ foreach (var rate in list)
+ {
+ await s.ExecuteAsync(
+ "INSERT INTO fx_rates (id, base, quote, rate_date, rate, source, created_at) VALUES (@id, @b, @q, @d, @r, @s, @now) " +
+ "ON CONFLICT (base, quote, rate_date) DO UPDATE SET rate = excluded.rate, source = excluded.source",
+ cancellationToken, ("id", Ids.NewId(clock)), ("b", rate.Base), ("q", rate.Quote), ("d", BarStore.DayText(rate.Date)),
+ ("r", rate.Rate), ("s", rate.Source), ("now", now)).ConfigureAwait(false);
+ }
+ }, cancellationToken).ConfigureAwait(false);
+ return list.Count;
+ }
+
+ /// The rate published for , or the last one before it; null if none is known.
+ public Task RateOnAsync(string baseCurrency, string quote, DateOnly day, CancellationToken cancellationToken) =>
+ db.ReadAsync(s => s.FirstOrDefaultAsync(
+ "SELECT base, quote, rate_date, rate, source FROM fx_rates WHERE base = @b AND quote = @q AND rate_date <= @d ORDER BY rate_date DESC",
+ Read, ("b", baseCurrency), ("q", quote), ("d", BarStore.DayText(day))), cancellationToken);
+
+ public Task LatestAsync(string baseCurrency, string quote, CancellationToken cancellationToken) =>
+ RateOnAsync(baseCurrency, quote, DateOnly.MaxValue, cancellationToken);
+
+ private static FxRate Read(DbRow r) =>
+ new(r.Text("base"), r.Text("quote"), BarStore.ParseDay(r.Text("rate_date")), r.Money("rate"), r.Text("source"));
+}
+
+public static class CalendarEventKinds
+{
+ public const string Economic = "economic";
+ public const string CentralBank = "central_bank";
+ public const string Earnings = "earnings";
+ public const string Holiday = "holiday";
+}
+
+///
+/// An event of the calendar (SPEC 10.4). is false when the source gives only the day
+/// (earnings without an hour, holidays): is then midnight of that day in the exchange's zone.
+///
+public sealed record CalendarEvent(string Kind, string Title, DateTimeOffset OccursAt, bool TimeKnown, string Source, string SourceKey)
+{
+ public string? Currency { get; init; }
+
+ public string? Symbol { get; init; }
+
+ /// For earnings: bmo before the open, amc after the close, dmh during market hours.
+ public string? TimeHint { get; init; }
+
+ /// Low, Medium or High.
+ public string? Impact { get; init; }
+
+ public string? Forecast { get; init; }
+
+ public string? Previous { get; init; }
+}
+
+public sealed class CalendarEventStore(Database db, IClock clock)
+{
+ private const string Columns = "kind, title, currency, symbol, occurs_at, time_known, time_hint, impact, forecast, previous, source, source_key";
+
+ /// Adds new events and updates known ones (a moved date, a new forecast). Returns how many arrived.
+ public async Task UpsertAsync(IEnumerable events, CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(events);
+ var list = events.DistinctBy(e => e.SourceKey).ToList();
+ var now = clock.UtcNow;
+ await db.WriteAsync(async s =>
+ {
+ foreach (var e in list)
+ {
+ await s.ExecuteAsync(
+ $"INSERT INTO calendar_events (id, {Columns}, created_at, updated_at) VALUES (@id, @kind, @title, @ccy, @sym, @at, @known, @hint, @impact, " +
+ "@forecast, @previous, @source, @key, @now, @now) ON CONFLICT (source_key) DO UPDATE SET kind = excluded.kind, title = excluded.title, " +
+ "currency = excluded.currency, symbol = excluded.symbol, occurs_at = excluded.occurs_at, time_known = excluded.time_known, " +
+ "time_hint = excluded.time_hint, impact = excluded.impact, forecast = excluded.forecast, previous = excluded.previous, updated_at = excluded.updated_at",
+ cancellationToken, ("id", Ids.NewId(clock)), ("kind", e.Kind), ("title", e.Title), ("ccy", e.Currency), ("sym", e.Symbol), ("at", e.OccursAt),
+ ("known", e.TimeKnown), ("hint", e.TimeHint), ("impact", e.Impact), ("forecast", e.Forecast), ("previous", e.Previous), ("source", e.Source),
+ ("key", e.SourceKey), ("now", now)).ConfigureAwait(false);
+ }
+ }, cancellationToken).ConfigureAwait(false);
+ return list.Count;
+ }
+
+ public async Task> BetweenAsync(DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken) =>
+ await db.ReadAsync(s => s.QueryAsync($"SELECT {Columns} FROM calendar_events WHERE occurs_at >= @f AND occurs_at < @t ORDER BY occurs_at, title",
+ r => new CalendarEvent(r.Text("kind"), r.Text("title"), r.Moment("occurs_at"), r.Bool("time_known"), r.Text("source"), r.Text("source_key"))
+ {
+ Currency = r.TextOrNull("currency"),
+ Symbol = r.TextOrNull("symbol"),
+ TimeHint = r.TextOrNull("time_hint"),
+ Impact = r.TextOrNull("impact"),
+ Forecast = r.TextOrNull("forecast"),
+ Previous = r.TextOrNull("previous"),
+ }, ("f", from), ("t", to)), cancellationToken).ConfigureAwait(false);
+}
+
+public static class MoodIndicators
+{
+ public const string FearGreed = "fear_greed";
+ public const string Vix = "vix";
+ public const string BtcFunding = "btc_funding";
+ public const string BtcOpenInterest = "btc_open_interest";
+}
+
+public sealed record MoodReading(string Indicator, DateTimeOffset AsOf, double Value, string? Label, string Source);
+
+/// The labels of SPEC 10.4, in the words the interface shows.
+public static class MoodLabels
+{
+ public static string FearGreed(double value) => value switch
+ {
+ <= 24 => "paura estrema",
+ <= 44 => "paura",
+ <= 55 => "neutro",
+ <= 75 => "avidità",
+ _ => "avidità estrema",
+ };
+
+ public static string Vix(double value) => value switch
+ {
+ < 20 => "calmo",
+ <= 30 => "nervoso",
+ _ => "panico",
+ };
+
+ /// is the funding of one period; its length (usually 8 hours).
+ public static string Funding(double rate, TimeSpan interval)
+ {
+ var perEightHours = interval > TimeSpan.Zero ? rate * (TimeSpan.FromHours(8) / interval) : rate;
+ return perEightHours switch
+ {
+ > 0.0002 => "troppi al rialzo",
+ < -0.0002 => "troppi al ribasso",
+ _ => "nella norma",
+ };
+ }
+}
+
+public sealed class MoodStore(Database db, IClock clock)
+{
+ public async Task UpsertAsync(IEnumerable readings, CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(readings);
+ var list = readings.ToList();
+ var now = clock.UtcNow;
+ await db.WriteAsync(async s =>
+ {
+ foreach (var m in list)
+ {
+ await s.ExecuteAsync(
+ "INSERT INTO market_mood (id, indicator, as_of, value, label, source, created_at) VALUES (@id, @i, @a, @v, @l, @s, @now) " +
+ "ON CONFLICT (indicator, as_of) DO UPDATE SET value = excluded.value, label = excluded.label, source = excluded.source",
+ cancellationToken, ("id", Ids.NewId(clock)), ("i", m.Indicator), ("a", m.AsOf), ("v", m.Value), ("l", m.Label), ("s", m.Source), ("now", now))
+ .ConfigureAwait(false);
+ }
+ }, cancellationToken).ConfigureAwait(false);
+ return list.Count;
+ }
+
+ public Task LatestAsync(string indicator, CancellationToken cancellationToken) =>
+ db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT indicator, as_of, value, label, source FROM market_mood WHERE indicator = @i ORDER BY as_of DESC",
+ r => new MoodReading(r.Text("indicator"), r.Moment("as_of"), r.Measure("value"), r.TextOrNull("label"), r.Text("source")), ("i", indicator)),
+ cancellationToken);
+}
+
+///
+/// The exchange calendar as the broker gives it, saved day by day so the scheduler can read it without network
+/// (SPEC 5.5: market times come from the broker's calendar, never from Italian time).
+///
+public sealed class StoredMarketCalendar(Database db, IClock clock, string market) : IMarketCalendar
+{
+ private readonly ConcurrentDictionary _days = new();
+
+ public string Market { get; } = market;
+
+ /// First and last day the stored calendar covers; null when it is empty.
+ public (DateOnly First, DateOnly Last)? Coverage => _days.IsEmpty ? null : (_days.Keys.Min(), _days.Keys.Max());
+
+ /// The session of ; null when the market is closed or the day is not covered yet.
+ public MarketSession? SessionOn(DateOnly day) => _days.TryGetValue(day, out var session) ? session : null;
+
+ public bool Covers(DateOnly day) => _days.ContainsKey(day);
+
+ /// Saves the sessions of [first, last]; days without a session are saved as closed.
+ public async Task SaveAsync(IReadOnlyCollection sessions, DateOnly first, DateOnly last, CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(sessions);
+ var byDay = sessions.Where(x => x.Date >= first && x.Date <= last).ToDictionary(x => x.Date);
+ var now = clock.UtcNow;
+ await db.WriteAsync(async s =>
+ {
+ for (var day = first; day <= last; day = day.AddDays(1))
+ {
+ var session = byDay.GetValueOrDefault(day);
+ await s.ExecuteAsync(
+ "INSERT INTO market_calendar (id, market, session_date, is_open, open_at, close_at, created_at) VALUES (@id, @m, @d, @o, @oa, @ca, @now) " +
+ "ON CONFLICT (market, session_date) DO UPDATE SET is_open = excluded.is_open, open_at = excluded.open_at, close_at = excluded.close_at",
+ cancellationToken, ("id", Ids.NewId(clock)), ("m", Market), ("d", BarStore.DayText(day)), ("o", session is not null),
+ ("oa", session?.OpenUtc), ("ca", session?.CloseUtc), ("now", now)).ConfigureAwait(false);
+ }
+ }, cancellationToken).ConfigureAwait(false);
+
+ for (var day = first; day <= last; day = day.AddDays(1))
+ {
+ _days[day] = byDay.GetValueOrDefault(day);
+ }
+ }
+
+ public async Task LoadAsync(CancellationToken cancellationToken)
+ {
+ var rows = await db.ReadAsync(s => s.QueryAsync("SELECT session_date, is_open, open_at, close_at FROM market_calendar WHERE market = @m",
+ r =>
+ {
+ var day = BarStore.ParseDay(r.Text("session_date"));
+ return (Day: day, Session: r.Bool("is_open") ? new MarketSession(day, r.Moment("open_at"), r.Moment("close_at")) : null);
+ }, ("m", Market)), cancellationToken).ConfigureAwait(false);
+ foreach (var (day, session) in rows)
+ {
+ _days[day] = session;
+ }
+ }
+
+ /// Sessions in [first, last], oldest first.
+ public IReadOnlyList Sessions(DateOnly first, DateOnly last) =>
+ _days.Where(d => d.Key >= first && d.Key <= last && d.Value is not null).Select(d => d.Value!).OrderBy(x => x.Date).ToList();
+}
+
+/// State of an external source: when it was last asked, last answered, and why it is off (SPEC 4.4).
+public sealed record SourceState(string Source, bool Enabled, DateTimeOffset? LastRequestAt, DateTimeOffset? LastSuccessAt, string? LastError, string? Note);
+
+public sealed class SourceStateStore(Database db, IClock clock)
+{
+ public Task RecordAsync(string source, bool enabled, bool success, string? error, string? note, CancellationToken cancellationToken)
+ {
+ var now = clock.UtcNow;
+ return db.WriteAsync(async s =>
+ {
+ var exists = await s.CountAsync("SELECT COUNT(*) FROM source_state WHERE source = @s", ("s", source)).ConfigureAwait(false) > 0;
+ if (!exists)
+ {
+ await s.ExecuteAsync("INSERT INTO source_state (id, source, enabled, created_at, updated_at) VALUES (@id, @s, @e, @now, @now)",
+ cancellationToken, ("id", Ids.NewId(clock)), ("s", source), ("e", enabled), ("now", now)).ConfigureAwait(false);
+ }
+
+ await s.ExecuteAsync(success
+ ? "UPDATE source_state SET enabled = @e, last_request_at = @now, last_success_at = @now, last_error = NULL, note = @n, updated_at = @now WHERE source = @s"
+ : "UPDATE source_state SET enabled = @e, last_request_at = @now, last_error = @err, note = @n, updated_at = @now WHERE source = @s",
+ cancellationToken, success
+ ? [("e", enabled), ("now", now), ("n", note), ("s", source)]
+ : [("e", enabled), ("now", now), ("err", error), ("n", note), ("s", source)]).ConfigureAwait(false);
+ }, cancellationToken);
+ }
+
+ public async Task> ListAsync(CancellationToken cancellationToken) =>
+ await db.ReadAsync(s => s.QueryAsync("SELECT source, enabled, last_request_at, last_success_at, last_error, note FROM source_state ORDER BY source",
+ r => new SourceState(r.Text("source"), r.Bool("enabled"), r.MomentOrNull("last_request_at"), r.MomentOrNull("last_success_at"),
+ r.TextOrNull("last_error"), r.TextOrNull("note"))), cancellationToken).ConfigureAwait(false);
+}
diff --git a/src/Anubis/Data/ReferenceSources.cs b/src/Anubis/Data/ReferenceSources.cs
new file mode 100644
index 0000000..3f8c60d
--- /dev/null
+++ b/src/Anubis/Data/ReferenceSources.cs
@@ -0,0 +1,221 @@
+using System.Globalization;
+using Anubis.Brokers;
+using Anubis.Core;
+using Anubis.Db;
+
+namespace Anubis.Data;
+
+///
+/// Downloads the free sources of SPEC 4.4 within their limits and records each attempt in source_state, so
+/// limits hold across restarts and the interface can say what is off and why. A source that fails is retried at
+/// its next turn; it never stops the others.
+///
+public sealed class ReferenceSources(
+ ApiHttp http,
+ Database db,
+ IClock clock,
+ SettingsStore settings,
+ Func> secret,
+ OkxClient? okx = null)
+{
+ /// Currencies kept from the ECB: the dollar for display and the tax export (SPEC 6.8).
+ public static readonly IReadOnlyList Currencies = ["USD"];
+
+ public const string OkxDerivatives = "okx_derivatives";
+
+ ///
+ /// Shortest time between two requests to the same source. Forex Factory asks for no more than two requests every
+ /// five minutes; the others publish once a day or less.
+ ///
+ public static readonly IReadOnlyDictionary MinInterval = new Dictionary
+ {
+ [Ecb.Source] = TimeSpan.FromMinutes(30),
+ [ForexFactory.Source] = TimeSpan.FromMinutes(55),
+ [Fred.Source] = TimeSpan.FromHours(12),
+ [Finnhub.Source] = TimeSpan.FromHours(12),
+ [FearGreed.Source] = TimeSpan.FromMinutes(55),
+ [VixHistory.Source] = TimeSpan.FromHours(6),
+ [OkxDerivatives] = TimeSpan.FromMinutes(55),
+ [CentralBanks.Source] = TimeSpan.Zero,
+ };
+
+ private readonly SourceStateStore _states = new(db, clock);
+
+ private readonly FxStore _fx = new(db, clock);
+
+ private readonly CalendarEventStore _calendar = new(db, clock);
+
+ private readonly MoodStore _mood = new(db, clock);
+
+ /// Reference rates: the whole history the first time, the last 90 days after a pause, otherwise the day's file.
+ public Task RefreshFxAsync(CancellationToken cancellationToken) => RunAsync(Ecb.Source, async () =>
+ {
+ var latest = await _fx.LatestAsync("EUR", "USD", cancellationToken).ConfigureAwait(false);
+ var today = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime);
+ IReadOnlyList rates;
+ if (latest is null)
+ {
+ var zip = await http.GetBytesAsync(Ecb.History, null, null, cancellationToken).ConfigureAwait(false)
+ ?? throw new MarketDataException("BCE: archivio storico dei cambi non trovato.", retryable: true);
+ rates = Ecb.ParseHistoryCsv(Ecb.ExtractHistoryCsv(zip), Currencies);
+ }
+ else
+ {
+ var url = latest.Date < today.AddDays(-4) ? Ecb.Last90Days : Ecb.Daily;
+ rates = Ecb.ParseXml(await http.GetStringAsync(url, null, null, cancellationToken).ConfigureAwait(false), Currencies);
+ }
+
+ await _fx.UpsertAsync(rates, cancellationToken).ConfigureAwait(false);
+ return $"{rates.Count} cambi";
+ }, cancellationToken);
+
+ /// The economic calendar of this week (Forex Factory), plus the central bank list of the settings.
+ public async Task RefreshCalendarAsync(CancellationToken cancellationToken)
+ {
+ var banks = await RunAsync(CentralBanks.Source, async () =>
+ {
+ var list = await settings.GetAsync(SettingsStore.CentralBankMeetings.Key, cancellationToken).ConfigureAwait(false);
+ var events = CentralBanks.Parse(list.Value);
+ await _calendar.UpsertAsync(events, cancellationToken).ConfigureAwait(false);
+ var last = events.Count == 0 ? (DateTimeOffset?)null : events.Max(e => e.OccursAt);
+ return last is { } l && l < clock.UtcNow.AddDays(60)
+ ? $"L'elenco delle riunioni finisce il {l:dd/MM/yyyy}: aggiungi le date del nuovo anno nelle impostazioni."
+ : $"{events.Count} riunioni";
+ }, cancellationToken).ConfigureAwait(false);
+
+ var week = await RunAsync(ForexFactory.Source, async () =>
+ {
+ var events = ForexFactory.Parse(await http.GetStringAsync(ForexFactory.ThisWeek, null, null, cancellationToken).ConfigureAwait(false));
+ await _calendar.UpsertAsync(events, cancellationToken).ConfigureAwait(false);
+ return $"{events.Count} eventi";
+ }, cancellationToken).ConfigureAwait(false);
+ return banks && week;
+ }
+
+ /// Release dates of the main US data from FRED; off without a key, and the interface says so.
+ public async Task RefreshFredAsync(CancellationToken cancellationToken)
+ {
+ var key = await secret("FRED_API_KEY", cancellationToken).ConfigureAwait(false);
+ if (string.IsNullOrWhiteSpace(key))
+ {
+ await _states.RecordAsync(Fred.Source, enabled: false, success: false, null,
+ "Spento: manca la chiave FRED_API_KEY. Il calendario usa solo Forex Factory.", cancellationToken).ConfigureAwait(false);
+ return false;
+ }
+
+ return await RunAsync(Fred.Source, async () =>
+ {
+ var from = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime);
+ var count = 0;
+ foreach (var (id, title) in Fred.Releases)
+ {
+ var events = Fred.Parse(await http.GetStringAsync(Fred.ReleaseDates(id, key, from), null, null, cancellationToken).ConfigureAwait(false), title, from);
+ count += await _calendar.UpsertAsync(events.Where(e => e.OccursAt < clock.UtcNow.AddDays(120)), cancellationToken).ConfigureAwait(false);
+ }
+
+ return $"{count} date";
+ }, cancellationToken).ConfigureAwait(false);
+ }
+
+ ///
+ /// Earnings dates for the next two weeks. Without a key the earnings filter is off (SPEC 4.4): the state says so
+ /// and the strategies read .
+ ///
+ public async Task RefreshEarningsAsync(IReadOnlySet? symbols, CancellationToken cancellationToken)
+ {
+ var key = await secret("FINNHUB_API_KEY", cancellationToken).ConfigureAwait(false);
+ if (string.IsNullOrWhiteSpace(key))
+ {
+ await _states.RecordAsync(Finnhub.Source, enabled: false, success: false, null,
+ "Spento: manca la chiave FINNHUB_API_KEY. Il filtro sulle trimestrali è disattivato.", cancellationToken).ConfigureAwait(false);
+ return false;
+ }
+
+ return await RunAsync(Finnhub.Source, async () =>
+ {
+ var today = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime);
+ var events = Finnhub.Parse(await http.GetStringAsync(Finnhub.Earnings(today, today.AddDays(14), key), null, null, cancellationToken).ConfigureAwait(false),
+ symbols);
+ await _calendar.UpsertAsync(events, cancellationToken).ConfigureAwait(false);
+ return $"{events.Count} trimestrali";
+ }, cancellationToken).ConfigureAwait(false);
+ }
+
+ /// True when earnings dates are known: the source is on and answered in the last two days.
+ public async Task EarningsFilterActiveAsync(CancellationToken cancellationToken)
+ {
+ var state = (await _states.ListAsync(cancellationToken).ConfigureAwait(false)).FirstOrDefault(s => s.Source == Finnhub.Source);
+ return state is { Enabled: true, LastSuccessAt: { } at } && clock.UtcNow - at < TimeSpan.FromDays(2);
+ }
+
+ /// Mood indicators: fear and greed, VIX, and Bitcoin funding and open interest from OKX public data.
+ public async Task RefreshMoodAsync(CancellationToken cancellationToken)
+ {
+ var fng = await RunAsync(FearGreed.Source, async () =>
+ {
+ var first = await _mood.LatestAsync(MoodIndicators.FearGreed, cancellationToken).ConfigureAwait(false) is null;
+ var readings = FearGreed.Parse(await http.GetStringAsync(FearGreed.Latest(first ? 400 : 2), null, null, cancellationToken).ConfigureAwait(false));
+ await _mood.UpsertAsync(readings, cancellationToken).ConfigureAwait(false);
+ return FearGreed.Attribution;
+ }, cancellationToken).ConfigureAwait(false);
+
+ var vix = await RunAsync(VixHistory.Source, async () =>
+ {
+ var from = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime).AddDays(-400);
+ var readings = VixHistory.Parse(await http.GetStringAsync(VixHistory.Csv, null, null, cancellationToken).ConfigureAwait(false), from);
+ await _mood.UpsertAsync(readings, cancellationToken).ConfigureAwait(false);
+ return $"{readings.Count} chiusure";
+ }, cancellationToken).ConfigureAwait(false);
+
+ if (okx is null)
+ {
+ return fng && vix;
+ }
+
+ var derivatives = await RunAsync(OkxDerivatives, async () =>
+ {
+ var funding = await okx.FetchFundingRateAsync("BTC-USDT-SWAP", cancellationToken).ConfigureAwait(false);
+ var interest = await okx.FetchOpenInterestAsync("BTC-USDT-SWAP", cancellationToken).ConfigureAwait(false);
+ await _mood.UpsertAsync(
+ [
+ new MoodReading(MoodIndicators.BtcFunding, funding.At, funding.Rate, MoodLabels.Funding(funding.Rate, funding.Interval), Venues.Okx),
+ new MoodReading(MoodIndicators.BtcOpenInterest, interest.At, interest.Usd, null, Venues.Okx),
+ ], cancellationToken).ConfigureAwait(false);
+ return "funding " + funding.Rate.ToString("P4", CultureInfo.InvariantCulture);
+ }, cancellationToken).ConfigureAwait(false);
+ return fng && vix && derivatives;
+ }
+
+ public Task> StatesAsync(CancellationToken cancellationToken) => _states.ListAsync(cancellationToken);
+
+ ///
+ /// Runs one source if its minimum interval has passed and records the outcome. Returns true when it succeeded or
+ /// was not due yet; false when it failed.
+ ///
+ private async Task RunAsync(string source, Func> work, CancellationToken cancellationToken)
+ {
+ var state = (await _states.ListAsync(cancellationToken).ConfigureAwait(false)).FirstOrDefault(s => s.Source == source);
+ if (state?.LastRequestAt is { } last && clock.UtcNow - last < MinInterval.GetValueOrDefault(source, TimeSpan.FromMinutes(5)))
+ {
+ return true;
+ }
+
+ try
+ {
+ var note = await work().ConfigureAwait(false);
+ await _states.RecordAsync(source, enabled: true, success: true, null, note, cancellationToken).ConfigureAwait(false);
+ return true;
+ }
+ catch (Exception ex) when (ex is MarketDataException or HttpRequestException or FormatException or System.Text.Json.JsonException
+ or System.Xml.XmlException or TaskCanceledException or InvalidDataException or KeyNotFoundException or InvalidOperationException or IndexOutOfRangeException)
+ {
+ if (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+
+ await _states.RecordAsync(source, enabled: true, success: false, Redactor.Redact(ex.Message), state?.Note, cancellationToken).ConfigureAwait(false);
+ return false;
+ }
+ }
+}
diff --git a/tests/Anubis.Tests/FakeNetwork.cs b/tests/Anubis.Tests/FakeNetwork.cs
index bcf05ca..310fb56 100644
--- a/tests/Anubis.Tests/FakeNetwork.cs
+++ b/tests/Anubis.Tests/FakeNetwork.cs
@@ -130,6 +130,14 @@ internal sealed class FastClock(DateTimeOffset start) : Anubis.Core.IClock
public TimeSpan Waited { get; private set; }
+ public void Advance(TimeSpan by)
+ {
+ lock (_gate)
+ {
+ _now += by;
+ }
+ }
+
public Task WaitUntilAsync(DateTimeOffset dueUtc, CancellationToken cancellationToken)
{
lock (_gate)
diff --git a/tests/Anubis.Tests/ReferenceSourcesTests.cs b/tests/Anubis.Tests/ReferenceSourcesTests.cs
new file mode 100644
index 0000000..b9ad7e4
--- /dev/null
+++ b/tests/Anubis.Tests/ReferenceSourcesTests.cs
@@ -0,0 +1,431 @@
+using System.IO.Compression;
+using System.Net;
+using System.Security.Cryptography;
+using System.Text;
+using Anubis.Brokers;
+using Anubis.Core;
+using Anubis.Data;
+
+namespace Anubis.Tests;
+
+/// Readers of the free sources on recorded responses, their stores, and the research importers (issues #21, #22, #24).
+[TestClass]
+public sealed class ReferenceSourcesTests
+{
+ private static readonly DateTimeOffset _start = new(2026, 11, 10, 8, 0, 0, TimeSpan.Zero);
+
+ [TestMethod]
+ [TestCategory(Category.Unit)]
+ public void EcbDailyNinetyDayAndHistoryFilesGiveTheSameRates()
+ {
+ string[] wanted = ["USD", "GBP", "BGN"];
+ var daily = Ecb.ParseXml(Fixture.Text("free/ecb-daily.xml"), wanted);
+ Assert.AreEqual(2, daily.Count, "BGN is no longer published");
+ Assert.AreEqual(new FxRate("EUR", "USD", new DateOnly(2026, 11, 9), 1.1193m, Ecb.Source), daily.Single(r => r.Quote == "USD"));
+ Assert.AreEqual(0.83430m, daily.Single(r => r.Quote == "GBP").Rate);
+
+ var ninety = Ecb.ParseXml(Fixture.Text("free/ecb-hist-90d.xml"), ["USD"]);
+ Assert.IsTrue(ninety.Count > 1);
+ Assert.AreEqual(1.1193m, ninety.Single(r => r.Date == new DateOnly(2026, 11, 9)).Rate, "the 90-day file uses double quotes");
+ CollectionAssert.AreEqual(ninety.OrderBy(r => r.Date).ToList(), ninety.ToList(), "oldest first");
+
+ var history = Ecb.ParseHistoryCsv(Fixture.Text("free/ecb-hist.csv"), wanted);
+ Assert.AreEqual(4, history.Count, "N/A is skipped and the trailing comma ignored");
+ Assert.AreEqual(new DateOnly(2026, 11, 6), history[0].Date);
+ Assert.AreEqual(1.1165m, history.Single(r => r.Quote == "USD" && r.Date == new DateOnly(2026, 11, 6)).Rate);
+
+ using var zip = new MemoryStream();
+ using (var archive = new ZipArchive(zip, ZipArchiveMode.Create, leaveOpen: true))
+ using (var writer = new StreamWriter(archive.CreateEntry("eurofxref-hist.csv").Open()))
+ {
+ writer.Write(Fixture.Text("free/ecb-hist.csv"));
+ }
+
+ Assert.AreEqual(Fixture.Text("free/ecb-hist.csv"), Ecb.ExtractHistoryCsv(zip.ToArray()));
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Unit)]
+ public void ForexFactoryEventsKeepTheirTimeAndHolidaysOnlyTheDay()
+ {
+ var events = ForexFactory.Parse(Fixture.Text("free/ff-calendar.json"));
+ Assert.AreEqual(5, events.Count);
+ var cpi = events.Single(e => e.Title == "CPI m/m");
+ Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 13, 30, 0, TimeSpan.Zero), cpi.OccursAt, "the offset -05:00 becomes UTC");
+ Assert.AreEqual("USD", cpi.Currency);
+ Assert.AreEqual("High", cpi.Impact);
+ Assert.AreEqual("0.3%", cpi.Forecast);
+ Assert.AreEqual("0.6%", cpi.Previous);
+ Assert.IsTrue(cpi.TimeKnown);
+
+ var holiday = events.Single(e => e.Title == "Bank Holiday");
+ Assert.AreEqual(CalendarEventKinds.Holiday, holiday.Kind);
+ Assert.IsFalse(holiday.TimeKnown);
+ Assert.IsNull(holiday.Impact);
+ Assert.IsNull(events.Single(e => e.Title == "RBA Assist Gov Hunter Speaks").Forecast, "an empty forecast is no forecast");
+ Assert.AreEqual(events.Count, events.Select(e => e.SourceKey).Distinct().Count());
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Unit)]
+ public void FredFinnhubAndCentralBankDatesBecomeTimedEvents()
+ {
+ var fred = Fred.Parse(Fixture.Text("free/fred-release-dates.json"), "PIL USA", new DateOnly(2010, 1, 1));
+ Assert.AreEqual(4, fred.Count, "dates before the start are skipped");
+ Assert.AreEqual(new DateTimeOffset(2010, 2, 11, 13, 30, 0, TimeSpan.Zero), fred[0].OccursAt, "08:30 New York in winter");
+ Assert.AreEqual("fred:82:2010-02-11", fred[0].SourceKey);
+ Assert.AreEqual(new DateTimeOffset(2012, 10, 24, 12, 30, 0, TimeSpan.Zero), fred[2].OccursAt, "08:30 New York in summer time");
+ StringAssert.Contains(Fred.ReleaseDates(10, "k", new DateOnly(2026, 11, 10)).Query, "include_release_dates_with_no_data=true");
+
+ var earnings = Finnhub.Parse(Fixture.Text("free/finnhub-earnings.json"));
+ Assert.AreEqual(2, earnings.Count);
+ var aapl = earnings.Single(e => e.OccursAt.Year == 2020);
+ Assert.AreEqual(new DateTimeOffset(2020, 1, 28, 21, 5, 0, TimeSpan.Zero), aapl.OccursAt, "amc: 16:05 New York");
+ Assert.AreEqual("amc", aapl.TimeHint);
+ Assert.AreEqual("AAPL", aapl.Symbol);
+ Assert.AreEqual(0, Finnhub.Parse(Fixture.Text("free/finnhub-earnings.json"), new HashSet { "MSFT" }).Count);
+
+ var banks = CentralBanks.Parse(CentralBanks.DefaultList);
+ Assert.AreEqual(18, banks.Count);
+ var fed = banks.Single(e => e.SourceKey == "cb:USD:2026-12-09");
+ Assert.AreEqual(new DateTimeOffset(2026, 12, 9, 19, 0, 0, TimeSpan.Zero), fed.OccursAt, "14:00 New York");
+ var ecb = banks.Single(e => e.SourceKey == "cb:EUR:2027-07-22");
+ Assert.AreEqual(new DateTimeOffset(2027, 7, 22, 12, 15, 0, TimeSpan.Zero), ecb.OccursAt, "14:15 Frankfurt in summer time");
+
+ // Every decision day of the default list is in the official calendars read for the fixture.
+ var official = Fixture.Text("free/central-banks.md");
+ foreach (var e in banks)
+ {
+ StringAssert.Contains(official, $"| {e.OccursAt:yyyy-MM-dd} |");
+ }
+
+ Assert.AreEqual(1, CentralBanks.Parse("# commento\n\nfed 2027-01-27\n").Count);
+ var error = Assert.ThrowsExactly(() => CentralBanks.Parse("FED 2027-01-27\nBOJ 2027-01-28"));
+ StringAssert.Contains(error.Message, "Riga 2");
+ Assert.ThrowsExactly(() => CentralBanks.Parse("FED 27/01/2027"));
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Unit)]
+ public void MoodReadingsCarryTheLabelsOfTheSpec()
+ {
+ var fng = FearGreed.Parse(Fixture.Text("free/fng.json"));
+ Assert.AreEqual(2, fng.Count);
+ Assert.AreEqual(32d, fng[^1].Value, "oldest first: the latest reading is last");
+ Assert.AreEqual("paura", fng[^1].Label);
+ Assert.AreEqual(DateTimeOffset.FromUnixTimeSeconds(1794268800), fng[^1].AsOf);
+
+ var vix = VixHistory.Parse(Fixture.Text("free/vix.csv"), new DateOnly(2026, 1, 1));
+ Assert.AreEqual(2, vix.Count);
+ Assert.AreEqual(new DateTimeOffset(2026, 11, 9, 0, 0, 0, TimeSpan.Zero), vix[1].AsOf);
+ Assert.AreEqual(("nervoso", "calmo"), (vix[0].Label, vix[1].Label));
+
+ Assert.AreEqual("paura estrema", MoodLabels.FearGreed(24));
+ Assert.AreEqual("neutro", MoodLabels.FearGreed(55));
+ Assert.AreEqual("avidità estrema", MoodLabels.FearGreed(76));
+ Assert.AreEqual("panico", MoodLabels.Vix(30.01));
+ Assert.AreEqual("troppi al rialzo", MoodLabels.Funding(0.0003, TimeSpan.FromHours(8)));
+ Assert.AreEqual("troppi al rialzo", MoodLabels.Funding(0.00015, TimeSpan.FromHours(4)), "4-hour funding counts double");
+ Assert.AreEqual("nella norma", MoodLabels.Funding(-0.0001, TimeSpan.FromHours(8)));
+ Assert.AreEqual("troppi al ribasso", MoodLabels.Funding(-0.00021, TimeSpan.FromHours(8)));
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Unit)]
+ public void NasdaqListExcludesFundsAndEverythingThatIsNotACommonShare()
+ {
+ var list = NasdaqTraded.Parse(Fixture.Text("free/nasdaqtraded.txt"));
+ Assert.AreEqual(12, list.Count, "header and footer skipped");
+ var excluded = list.Where(s => s.IsExcluded).Select(s => s.Symbol).ToList();
+ CollectionAssert.AreEquivalent(new[] { "SPY", "BULZ", "AMUB", "PDI", "ABR$D", "AACIW", "ZXIET" }, excluded);
+ Assert.AreEqual("TXSE", list.Single(s => s.Symbol == "CECO").Exchange);
+ Assert.AreEqual("NASDAQ", list.Single(s => s.Symbol == "AAPL").Exchange);
+ Assert.AreEqual("NYSE", list.Single(s => s.Symbol == "KO").Exchange);
+ Assert.AreEqual("D", list.Single(s => s.Symbol == "AACG").FinancialStatus);
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Unit)]
+ public void BinanceArchivesReadMillisecondsAndMicrosecondsAndCheckTheirFingerprint()
+ {
+ var old = BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv"));
+ var recent = BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2025-01.csv"));
+ Assert.AreEqual(5, old.Count);
+ Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), old[0].OpensAt);
+ Assert.AreEqual(4261.48m, old[0].Open);
+ Assert.AreEqual(3454770.05073206m, old[0].QuoteVolume);
+ Assert.AreEqual(3427L, old[0].Trades);
+ Assert.AreEqual(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), recent[0].OpensAt, "microseconds from 2025");
+ Assert.AreEqual(new DateTimeOffset(2025, 1, 5, 0, 0, 0, TimeSpan.Zero), recent[^1].OpensAt);
+
+ var zip = Zip("BTCUSDT-1d-2017-08.csv", Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv"));
+ var checksum = Convert.ToHexStringLower(SHA256.HashData(zip)) + " BTCUSDT-1d-2017-08.zip\n";
+ Assert.IsTrue(BinanceArchive.Verify(zip, checksum));
+ Assert.IsFalse(BinanceArchive.Verify(zip, Fixture.Text("free/BTCUSDT-1d-2017-08.zip.CHECKSUM")), "the recorded checksum belongs to the real zip");
+ CollectionAssert.AreEqual(old.ToList(), BinanceArchive.ReadZip(zip).ToList());
+ Assert.AreEqual(
+ new Uri("https://data.binance.vision/data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2017-08.zip.CHECKSUM"),
+ BinanceArchive.ChecksumOf(BinanceArchive.Monthly("BTCUSDT", "1d", 2017, 8)));
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Integration)]
+ public async Task ReferenceStoresRoundTripOnTheEngineOfThisRun()
+ {
+ var clock = new ManualClock(_start);
+ await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
+
+ var fx = new FxStore(test.Db, clock);
+ await fx.UpsertAsync(Ecb.ParseHistoryCsv(Fixture.Text("free/ecb-hist.csv"), ["USD"]), CancellationToken.None).ConfigureAwait(false);
+ await fx.UpsertAsync([new FxRate("EUR", "USD", new DateOnly(2026, 11, 9), 1.1194m, Ecb.Source)], CancellationToken.None).ConfigureAwait(false);
+ Assert.AreEqual(1.1194m, (await fx.LatestAsync("EUR", "USD", CancellationToken.None).ConfigureAwait(false))!.Rate, "a new publication replaces the old");
+ Assert.AreEqual(1.1165m, (await fx.RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 8), CancellationToken.None).ConfigureAwait(false))!.Rate,
+ "Sunday uses Friday's rate");
+ Assert.IsNull(await fx.RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 5), CancellationToken.None).ConfigureAwait(false));
+
+ var calendar = new CalendarEventStore(test.Db, clock);
+ var events = ForexFactory.Parse(Fixture.Text("free/ff-calendar.json"));
+ Assert.AreEqual(5, await calendar.UpsertAsync(events, CancellationToken.None).ConfigureAwait(false));
+ var moved = events.Single(e => e.Title == "CPI m/m") with { Forecast = "0.4%" };
+ await calendar.UpsertAsync([moved], CancellationToken.None).ConfigureAwait(false);
+ var week = await calendar.BetweenAsync(new DateTimeOffset(2026, 11, 10, 0, 0, 0, TimeSpan.Zero), new DateTimeOffset(2026, 11, 17, 0, 0, 0, TimeSpan.Zero),
+ CancellationToken.None).ConfigureAwait(false);
+ Assert.AreEqual(3, week.Count);
+ Assert.AreEqual(moved, week[0]);
+ Assert.IsFalse(week.Single(e => e.Kind == CalendarEventKinds.Holiday).TimeKnown);
+
+ var mood = new MoodStore(test.Db, clock);
+ await mood.UpsertAsync(FearGreed.Parse(Fixture.Text("free/fng.json")), CancellationToken.None).ConfigureAwait(false);
+ await mood.UpsertAsync(FearGreed.Parse(Fixture.Text("free/fng.json")), CancellationToken.None).ConfigureAwait(false);
+ var latest = await mood.LatestAsync(MoodIndicators.FearGreed, CancellationToken.None).ConfigureAwait(false);
+ Assert.AreEqual(32d, latest!.Value);
+ Assert.AreEqual("paura", latest.Label);
+ Assert.IsNull(await mood.LatestAsync(MoodIndicators.Vix, CancellationToken.None).ConfigureAwait(false));
+
+ var states = new SourceStateStore(test.Db, clock);
+ await states.RecordAsync(Finnhub.Source, enabled: false, success: false, null, "manca la chiave FINNHUB_API_KEY", CancellationToken.None).ConfigureAwait(false);
+ await states.RecordAsync(Ecb.Source, enabled: true, success: true, null, null, CancellationToken.None).ConfigureAwait(false);
+ clock.Advance(TimeSpan.FromHours(1));
+ await states.RecordAsync(Ecb.Source, enabled: true, success: false, "timeout", null, CancellationToken.None).ConfigureAwait(false);
+ var list = await states.ListAsync(CancellationToken.None).ConfigureAwait(false);
+ Assert.AreEqual(2, list.Count);
+ var ecb = list.Single(s => s.Source == Ecb.Source);
+ Assert.AreEqual(_start, ecb.LastSuccessAt, "a failure keeps the last success");
+ Assert.AreEqual(_start.AddHours(1), ecb.LastRequestAt);
+ Assert.AreEqual("timeout", ecb.LastError);
+ Assert.IsFalse(list.Single(s => s.Source == Finnhub.Source).Enabled);
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Integration)]
+ public async Task StoredMarketCalendarKeepsOpenAndClosedDaysAcrossRestarts()
+ {
+ var clock = new ManualClock(_start);
+ await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
+ var calendar = new StoredMarketCalendar(test.Db, clock, "us_equity");
+ Assert.IsNull(calendar.Coverage);
+
+ var friday = new DateOnly(2026, 11, 13);
+ var monday = new DateOnly(2026, 11, 16);
+ MarketSession Session(DateOnly day) => new(day, Zones.At(day, new TimeOnly(9, 30), Zones.NewYork), Zones.At(day, new TimeOnly(16, 0), Zones.NewYork));
+ await calendar.SaveAsync([Session(friday), Session(monday)], friday, monday, CancellationToken.None).ConfigureAwait(false);
+
+ var reloaded = new StoredMarketCalendar(test.Db, clock, "us_equity");
+ await reloaded.LoadAsync(CancellationToken.None).ConfigureAwait(false);
+ Assert.AreEqual((friday, monday), reloaded.Coverage);
+ Assert.AreEqual(Session(friday), reloaded.SessionOn(friday));
+ Assert.IsNull(reloaded.SessionOn(friday.AddDays(1)), "Saturday is closed");
+ Assert.IsTrue(reloaded.Covers(friday.AddDays(1)));
+ Assert.IsFalse(reloaded.Covers(monday.AddDays(1)), "a day not downloaded is unknown, not closed");
+ Assert.AreEqual(2, reloaded.Sessions(friday, monday).Count);
+ Assert.AreEqual(new DateTimeOffset(2026, 11, 13, 14, 30, 0, TimeSpan.Zero), reloaded.SessionOn(friday)!.OpenUtc);
+
+ var other = new StoredMarketCalendar(test.Db, clock, "other");
+ await other.LoadAsync(CancellationToken.None).ConfigureAwait(false);
+ Assert.IsNull(other.Coverage, "each market has its own calendar");
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Integration)]
+ public async Task BinanceImporterChecksEachArchiveAndStopsAtTheFirstMissingMonth()
+ {
+ var folder = TempFolder.Create("binance");
+ try
+ {
+ var august = Zip("BTCUSDT-1d-2017-08.csv", Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv"));
+ var http = new FakeHttp()
+ .On("2017-08.zip.CHECKSUM", Convert.ToHexStringLower(SHA256.HashData(august)) + " BTCUSDT-1d-2017-08.zip")
+ .OnBytes("2017-08.zip", august)
+ .On(_ => true, _ => FakeHttp.Respond("NoSuchKey", HttpStatusCode.NotFound));
+ var clock = new FastClock(new DateTimeOffset(2017, 10, 15, 0, 0, 0, TimeSpan.Zero));
+ var store = new ResearchStore(folder);
+ var importer = new BinanceImporter(new ApiHttp(http.Client(), clock, "binance"), store, clock);
+
+ var result = await importer.ImportAsync("BTCUSDT", "1d", 2017, 7, CancellationToken.None).ConfigureAwait(false);
+ Assert.AreEqual(5, result.Bars);
+ Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), result.First);
+ Assert.AreEqual(1, result.Notes.Count);
+ StringAssert.Contains(result.Notes[0], "2017-09");
+ Assert.AreEqual(4, http.Requests.Count, "July (not listed yet), August and its checksum, September (not published)");
+ var stored = await store.ReadAsync(Venues.Binance, "BTCUSDT", Timeframes.Day, DateTimeOffset.MinValue, DateTimeOffset.MaxValue).ToListAsync().ConfigureAwait(false);
+ CollectionAssert.AreEqual(BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv")).ToList(), stored);
+
+ // A zip that does not match its fingerprint is refused.
+ var tampered = new FakeHttp()
+ .On("CHECKSUM", new string('0', 64) + " BTCUSDT-1d-2017-08.zip")
+ .OnBytes("2017-08.zip", august);
+ var refused = new BinanceImporter(new ApiHttp(tampered.Client(), clock, "binance"), new ResearchStore(Path.Combine(folder, "other")), clock);
+ await Assert.ThrowsExactlyAsync(() => refused.ImportAsync("BTCUSDT", "1d", 2017, 8, CancellationToken.None)).ConfigureAwait(false);
+ }
+ finally
+ {
+ TempFolder.Delete(folder);
+ }
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Integration)]
+ public async Task CsvImporterFollowsTheMappingAndReportsWhatItCouldNotRead()
+ {
+ var folder = TempFolder.Create("csvimport");
+ try
+ {
+ var csv = Path.Combine(folder, "ethusd_2024.csv");
+ await File.WriteAllTextAsync(csv,
+ "Data;Apertura;Massimo;Minimo;Chiusura;Volume\n" +
+ "02/01/2024 01:00;2.300,50;2.310,00;2.295,25;2.305,75;12,5\n" +
+ "02/01/2024 01:01;2.305,75;2.306,00;2.301,00;2.302,00;3\n" +
+ "02/01/2024 01:02;non so;2.306,00;2.301,00;2.302,00;3\n" +
+ "02/01/2024 01:04;2.302,00;2.303,00;2.300,00;2.301,00;4\n").ConfigureAwait(false);
+ await File.WriteAllTextAsync(Path.ChangeExtension(csv, ".mapping.json"), """
+ {
+ // Exported from a spreadsheet in Italian.
+ "symbol": "ETHUSD",
+ "delimiter": ";",
+ "time": "Data",
+ "time_format": "dd/MM/yyyy HH:mm",
+ "time_zone": "Europe/Rome",
+ "open": "Apertura", "high": "Massimo", "low": "Minimo", "close": "Chiusura", "volume": "Volume",
+ "decimal_separator": ",",
+ }
+ """).ConfigureAwait(false);
+
+ var mapping = CsvImporter.MappingFor(csv);
+ Assert.AreEqual("ETHUSD", mapping.Symbol);
+ Assert.AreEqual(";", mapping.Delimiter);
+ var store = new ResearchStore(Path.Combine(folder, "research"));
+ var result = await new CsvImporter(store).ImportAsync(csv, mapping, CancellationToken.None).ConfigureAwait(false);
+
+ Assert.AreEqual(3, result.Bars);
+ Assert.AreEqual(new DateTimeOffset(2024, 1, 2, 0, 0, 0, TimeSpan.Zero), result.First, "01:00 in Rome is midnight UTC in winter");
+ StringAssert.Contains(result.Notes[0], "riga 4");
+ Assert.IsTrue(result.Notes.Any(n => n.Contains("1 righe non lette su 5", StringComparison.Ordinal)), string.Join(" | ", result.Notes));
+ Assert.IsTrue(result.Notes.Any(n => n.StartsWith(IssueKinds.Gap, StringComparison.Ordinal)), "the missing minutes are reported");
+
+ var stored = await store.ReadAsync("import", "ETHUSD", Timeframes.Minute, DateTimeOffset.MinValue, DateTimeOffset.MaxValue).ToListAsync().ConfigureAwait(false);
+ Assert.AreEqual(3, stored.Count);
+ Assert.AreEqual(2300.50m, stored[0].Open);
+ Assert.AreEqual(12.5m, stored[0].Volume);
+
+ // Without a mapping file: Binance-like columns by number and the symbol from the file name.
+ var plain = Path.Combine(folder, "solusdt-1m.csv");
+ await File.WriteAllTextAsync(plain, string.Join('\n', Fixture.Text("free/binance-BTCUSDT-1d-2025-01.csv").Split('\n').Take(2))).ConfigureAwait(false);
+ var defaults = CsvImporter.MappingFor(plain) with { HasHeader = false, Timeframe = Timeframes.Day };
+ var second = await new CsvImporter(store).ImportAsync(plain, defaults, CancellationToken.None).ConfigureAwait(false);
+ Assert.AreEqual("SOLUSDT", second.Symbol);
+ Assert.AreEqual(2, second.Bars);
+ Assert.AreEqual(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), second.First, "microseconds recognised by size");
+ }
+ finally
+ {
+ TempFolder.Delete(folder);
+ }
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Integration)]
+ public async Task DownloaderRespectsLimitsRecordsFailuresAndSaysWhatIsOff()
+ {
+ var clock = new FastClock(_start);
+ await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
+ var settings = new SettingsStore(test.Db, clock, new EventBus());
+ var ecbZip = Zip("eurofxref-hist.csv", Fixture.Text("free/ecb-hist.csv"));
+ var fngFails = true;
+ var http = new FakeHttp()
+ .OnBytes("eurofxref-hist.zip", ecbZip)
+ .On("eurofxref-daily.xml", Fixture.Text("free/ecb-daily.xml"))
+ .On("ff_calendar_thisweek", Fixture.Text("free/ff-calendar.json"))
+ .On(u => u.Host == "api.alternative.me", _ => fngFails
+ ? FakeHttp.Respond("{}", HttpStatusCode.BadRequest)
+ : FakeHttp.Respond(Fixture.Text("free/fng.json")))
+ .On("VIX_History.csv", Fixture.Text("free/vix.csv"));
+ var sources = new ReferenceSources(new ApiHttp(http.Client(), clock, "reference"), test.Db, clock, settings, (_, _) => Task.FromResult(null));
+
+ // Exchange rates: the whole history the first time, then the day's file.
+ Assert.IsTrue(await sources.RefreshFxAsync(CancellationToken.None).ConfigureAwait(false));
+ clock.Advance(TimeSpan.FromHours(1));
+ Assert.IsTrue(await sources.RefreshFxAsync(CancellationToken.None).ConfigureAwait(false));
+ CollectionAssert.AreEqual(new[] { "eurofxref-hist.zip", "eurofxref-daily.xml" }, http.Urls.Select(u => u.Segments[^1]).ToList());
+ var rate = await new FxStore(test.Db, clock).RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 9), CancellationToken.None).ConfigureAwait(false);
+ Assert.AreEqual(1.1193m, rate!.Rate);
+
+ // The calendar: Forex Factory at most once an hour, the central banks from the settings.
+ Assert.IsTrue(await sources.RefreshCalendarAsync(CancellationToken.None).ConfigureAwait(false));
+ clock.Advance(TimeSpan.FromMinutes(10));
+ Assert.IsTrue(await sources.RefreshCalendarAsync(CancellationToken.None).ConfigureAwait(false));
+ Assert.AreEqual(1, http.Urls.Count(u => u.Host == "nfs.faireconomy.media"), "ten minutes later Forex Factory is not asked again");
+ var events = await new CalendarEventStore(test.Db, clock).BetweenAsync(_start.AddDays(-7), _start.AddYears(2), CancellationToken.None).ConfigureAwait(false);
+ Assert.AreEqual(5 + 18, events.Count);
+
+ // Without keys FRED and Finnhub are off and the state says why; the earnings filter is off.
+ Assert.IsFalse(await sources.RefreshFredAsync(CancellationToken.None).ConfigureAwait(false));
+ Assert.IsFalse(await sources.RefreshEarningsAsync(null, CancellationToken.None).ConfigureAwait(false));
+ Assert.IsFalse(await sources.EarningsFilterActiveAsync(CancellationToken.None).ConfigureAwait(false));
+ var states = await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false);
+ StringAssert.Contains(states.Single(s => s.Source == Finnhub.Source).Note, "filtro sulle trimestrali è disattivato");
+ Assert.IsFalse(states.Single(s => s.Source == Fred.Source).Enabled);
+ Assert.IsFalse(http.Urls.Any(u => u.Host.Contains("finnhub", StringComparison.Ordinal) || u.Host.Contains("stlouisfed", StringComparison.Ordinal)));
+
+ // A failing source is recorded and does not stop the others; it is tried again at its next turn.
+ Assert.IsFalse(await sources.RefreshMoodAsync(CancellationToken.None).ConfigureAwait(false));
+ states = await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false);
+ Assert.IsNotNull(states.Single(s => s.Source == FearGreed.Source).LastError);
+ Assert.IsNull(states.Single(s => s.Source == VixHistory.Source).LastError);
+ Assert.AreEqual("calmo", (await new MoodStore(test.Db, clock).LatestAsync(MoodIndicators.Vix, CancellationToken.None).ConfigureAwait(false))!.Label);
+ fngFails = false;
+ clock.Advance(TimeSpan.FromHours(1));
+ Assert.IsTrue(await sources.RefreshMoodAsync(CancellationToken.None).ConfigureAwait(false));
+ var fng = (await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false)).Single(s => s.Source == FearGreed.Source);
+ Assert.IsNull(fng.LastError);
+ Assert.AreEqual(FearGreed.Attribution, fng.Note, "the source is named, as its terms ask");
+ StringAssert.Contains(http.Urls.Last(u => u.Host == "api.alternative.me").Query, "limit=400", "the first success asks for the history");
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Integration)]
+ public async Task CentralBankListIsEditableAndRefusedWhenWrong()
+ {
+ var clock = new ManualClock(_start);
+ await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
+ var settings = new SettingsStore(test.Db, clock, new EventBus());
+ var key = SettingsStore.CentralBankMeetings.Key;
+ Assert.AreEqual(CentralBanks.DefaultList, (await settings.GetAsync(key).ConfigureAwait(false)).Value);
+
+ var error = await Assert.ThrowsExactlyAsync(() => settings.SetAsync(key, "FED 2027-13-01", "test")).ConfigureAwait(false);
+ StringAssert.Contains(error.Message, "Riga 1");
+ await settings.SetAsync(key, CentralBanks.DefaultList + "\nFED 2028-01-26", "test").ConfigureAwait(false);
+ Assert.AreEqual(19, CentralBanks.Parse((await settings.GetAsync(key).ConfigureAwait(false)).Value).Count);
+ }
+
+ internal static byte[] Zip(string entryName, string content)
+ {
+ using var zip = new MemoryStream();
+ using (var archive = new ZipArchive(zip, ZipArchiveMode.Create, leaveOpen: true))
+ using (var stream = archive.CreateEntry(entryName).Open())
+ {
+ stream.Write(Encoding.UTF8.GetBytes(content));
+ }
+
+ return zip.ToArray();
+ }
+}