diff --git a/src/Anubis/Core/Settings.cs b/src/Anubis/Core/Settings.cs index fe9eaf8..fc66c67 100644 --- a/src/Anubis/Core/Settings.cs +++ b/src/Anubis/Core/Settings.cs @@ -3,8 +3,11 @@ using Anubis.Db; namespace Anubis.Core; -/// A setting the interface may change: key, default and allowed values. -public sealed record SettingDefinition(string Key, string Label, string Default, IReadOnlyList? Allowed = null); +/// +/// A setting the interface may change: key, default and allowed values. , when present, returns +/// the reason a free-text value is refused, in the words the interface shows, or null when it is valid. +/// +public sealed record SettingDefinition(string Key, string Label, string Default, IReadOnlyList? Allowed = null, Func? Check = null); /// A value fixed by the startup configuration: it wins and is shown read-only with where to change it (SPEC 13.5). public sealed record StartupOverride(string Value, string Source); @@ -34,7 +37,11 @@ public sealed class SettingsStore(Database db, IClock clock, EventBus bus, IRead /// Bot state behind the top bar buttons: active, paused (no new entries) or stopped (kill switch, SPEC 9.3). public static readonly SettingDefinition BotState = new("bot.state", "Stato del bot", "active", ["active", "paused", "stopped"]); - public static readonly IReadOnlyList All = [Theme, Currency, BotState]; + /// Fed and ECB decision days, one per line (SPEC 4.4: a yearly list the user edits). + public static readonly SettingDefinition CentralBankMeetings = new("calendar.central_banks", "Riunioni di Fed e BCE", + Data.CentralBanks.DefaultList, Check: Data.CentralBanks.Check); + + public static readonly IReadOnlyList All = [Theme, Currency, BotState, CentralBankMeetings]; private readonly IReadOnlyDictionary _overrides = overrides ?? new Dictionary(); @@ -68,6 +75,11 @@ public sealed class SettingsStore(Database db, IClock clock, EventBus bus, IRead throw new ArgumentException($"Valore non ammesso per {definition.Label}: {value}.", nameof(value)); } + if (definition.Check?.Invoke(value) is { } problem) + { + throw new ArgumentException(problem, nameof(value)); + } + var now = clock.UtcNow; var json = new DbJson(JsonSerializer.Serialize(value)); var changed = await db.WriteAsync(async s => diff --git a/src/Anubis/Data/FreeSources.cs b/src/Anubis/Data/FreeSources.cs new file mode 100644 index 0000000..70d340e --- /dev/null +++ b/src/Anubis/Data/FreeSources.cs @@ -0,0 +1,467 @@ +using System.Globalization; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Xml.Linq; + +namespace Anubis.Data; + +/// +/// Readers of the free sources of SPEC 4.4. Each is a pure function from the published text to Anubis records, tested +/// on recorded responses; downloads them within their limits. +/// +public static class Ecb +{ + public const string Source = "ecb"; + + public static readonly Uri Daily = new("https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml"); + + public static readonly Uri Last90Days = new("https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml"); + + public static readonly Uri History = new("https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.zip"); + + private static readonly XNamespace _ns = "http://www.ecb.int/vocabulary/2002-08-01/eurofxref"; + + /// Rates of the daily or 90-day XML for the asked, EUR as base. + public static IReadOnlyList ParseXml(string xml, IReadOnlyCollection currencies) + { + var document = XDocument.Parse(xml); + return document.Descendants(_ns + "Cube") + .Where(day => day.Attribute("time") is not null) + .SelectMany(day => day.Elements(_ns + "Cube").Select(rate => (Day: day, Rate: rate))) + .Where(x => currencies.Contains((string?)x.Rate.Attribute("currency") ?? "")) + .Select(x => new FxRate("EUR", (string)x.Rate.Attribute("currency")!, + DateOnly.ParseExact((string)x.Day.Attribute("time")!, "yyyy-MM-dd", CultureInfo.InvariantCulture), + decimal.Parse((string)x.Rate.Attribute("rate")!, NumberStyles.Float, CultureInfo.InvariantCulture), Source)) + .OrderBy(r => r.Date) + .ToList(); + } + + /// The full history CSV (inside eurofxref-hist.zip): newest first, a trailing comma on every line, N/A where missing. + public static IReadOnlyList ParseHistoryCsv(string csv, IReadOnlyCollection currencies) + { + ArgumentNullException.ThrowIfNull(csv); + var lines = csv.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var header = lines[0].Split(','); + var rates = new List(); + foreach (var line in lines.Skip(1)) + { + var fields = line.Split(','); + var date = DateOnly.ParseExact(fields[0], "yyyy-MM-dd", CultureInfo.InvariantCulture); + for (var i = 1; i < Math.Min(header.Length, fields.Length); i++) + { + if (currencies.Contains(header[i]) && decimal.TryParse(fields[i], NumberStyles.Float, CultureInfo.InvariantCulture, out var rate)) + { + rates.Add(new FxRate("EUR", header[i], date, rate, Source)); + } + } + } + + return rates.OrderBy(r => r.Date).ToList(); + } + + public static string ExtractHistoryCsv(byte[] zip) + { + using var archive = new ZipArchive(new MemoryStream(zip), ZipArchiveMode.Read); + var entry = archive.Entries.Single(e => e.Name.EndsWith(".csv", StringComparison.OrdinalIgnoreCase)); + using var reader = new StreamReader(entry.Open()); + return reader.ReadToEnd(); + } +} + +/// The weekly economic calendar of Forex Factory (faireconomy.media). It has no actual values. +public static class ForexFactory +{ + public const string Source = "forexfactory"; + + public static readonly Uri ThisWeek = new("https://nfs.faireconomy.media/ff_calendar_thisweek.json"); + + public static IReadOnlyList Parse(string json) + { + using var document = JsonDocument.Parse(json); + var events = new List(); + foreach (var item in document.RootElement.EnumerateArray()) + { + var title = Text(item, "title"); + var country = Text(item, "country"); + var at = DateTimeOffset.Parse(Text(item, "date"), CultureInfo.InvariantCulture).ToUniversalTime(); + var impact = Text(item, "impact"); + var holiday = impact == "Holiday"; + events.Add(new CalendarEvent(holiday ? CalendarEventKinds.Holiday : CalendarEventKinds.Economic, title, at, TimeKnown: !holiday, Source, + $"ff:{country}:{at:yyyy-MM-ddTHH:mm}:{title}") + { + Currency = country == "All" ? null : country, + Impact = holiday ? null : impact, + Forecast = Empty(Text(item, "forecast")), + Previous = Empty(Text(item, "previous")), + }); + } + + return events; + } + + private static string? Empty(string text) => text.Length == 0 ? null : text; + + internal static string Text(JsonElement item, string name) => + item.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString()! : ""; +} + +/// +/// Future release dates of the main US data from FRED (optional key). FRED gives only the day: the time is +/// 08:30 New York and the impact high (SPEC 4.4). +/// +public static class Fred +{ + public const string Source = "fred"; + + /// Releases followed, with the title the calendar shows (ids checked on fred.stlouisfed.org). + public static readonly IReadOnlyList<(int Id, string Title)> Releases = + [ + (10, "CPI (inflazione USA)"), + (50, "Employment Situation (occupazione USA)"), + (53, "PIL USA"), + (54, "Personal Income and Outlays (PCE)"), + ]; + + public static Uri ReleaseDates(int releaseId, string apiKey, DateOnly from) => new( + "https://api.stlouisfed.org/fred/release/dates?release_id=" + releaseId.ToString(CultureInfo.InvariantCulture) + + "&api_key=" + Uri.EscapeDataString(apiKey) + "&file_type=json&include_release_dates_with_no_data=true&sort_order=asc" + + "&realtime_start=" + from.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); + + public static IReadOnlyList Parse(string json, string title, DateOnly from) + { + using var document = JsonDocument.Parse(json); + var events = new List(); + foreach (var item in document.RootElement.GetProperty("release_dates").EnumerateArray()) + { + var day = DateOnly.ParseExact(ForexFactory.Text(item, "date"), "yyyy-MM-dd", CultureInfo.InvariantCulture); + if (day < from) + { + continue; + } + + var id = item.GetProperty("release_id").GetInt32(); + events.Add(new CalendarEvent(CalendarEventKinds.Economic, title, Zones.At(day, new TimeOnly(8, 30), Zones.NewYork), TimeKnown: true, Source, + $"fred:{id}:{day:yyyy-MM-dd}") + { + Currency = "USD", + Impact = "High", + }); + } + + return events; + } +} + +/// Earnings dates from Finnhub (optional key, free plan: 60 calls a minute). +public static class Finnhub +{ + public const string Source = "finnhub"; + + public static Uri Earnings(DateOnly from, DateOnly to, string apiKey) => new( + $"https://finnhub.io/api/v1/calendar/earnings?from={from:yyyy-MM-dd}&to={to:yyyy-MM-dd}&token={Uri.EscapeDataString(apiKey)}"); + + /// + /// hour: bmo before the open (08:00 New York), amc after the close (16:05), dmh during market hours + /// (12:00); empty means unknown and the event keeps only its day. + /// + public static IReadOnlyList Parse(string json, IReadOnlySet? symbols = null) + { + using var document = JsonDocument.Parse(json); + var events = new List(); + foreach (var item in document.RootElement.GetProperty("earningsCalendar").EnumerateArray()) + { + var symbol = ForexFactory.Text(item, "symbol"); + if (symbols is not null && !symbols.Contains(symbol)) + { + continue; + } + + var day = DateOnly.ParseExact(ForexFactory.Text(item, "date"), "yyyy-MM-dd", CultureInfo.InvariantCulture); + var hour = ForexFactory.Text(item, "hour"); + TimeOnly? time = hour switch + { + "bmo" => new TimeOnly(8, 0), + "amc" => new TimeOnly(16, 5), + "dmh" => new TimeOnly(12, 0), + _ => null, + }; + events.Add(new CalendarEvent(CalendarEventKinds.Earnings, "Trimestrale " + symbol, Zones.At(day, time ?? TimeOnly.MinValue, Zones.NewYork), + time is not null, Source, $"finnhub:{symbol}:{day:yyyy-MM-dd}") + { + Symbol = symbol, + Currency = "USD", + TimeHint = hour.Length == 0 ? null : hour, + Impact = "High", + }); + } + + return events; + } +} + +/// +/// Meetings of the Fed and the ECB from the yearly list in the settings (SPEC 4.4): one per line, +/// FED 2026-12-09 or BCE 2026-12-17. Fed decisions come at 14:00 New York, ECB ones at 14:15 Frankfurt. +/// +public static class CentralBanks +{ + public const string Source = "central_banks"; + + /// Decision days from the official calendars, read on 10/11/2026 (API-NOTES). + public const string DefaultList = + "FED 2026-12-09\nFED 2027-01-27\nFED 2027-03-17\nFED 2027-04-28\nFED 2027-06-09\nFED 2027-07-28\nFED 2027-09-15\nFED 2027-10-27\nFED 2027-12-08\n" + + "BCE 2026-12-17\nBCE 2027-02-04\nBCE 2027-03-18\nBCE 2027-04-29\nBCE 2027-06-10\nBCE 2027-07-22\nBCE 2027-09-09\nBCE 2027-10-28\nBCE 2027-12-16"; + + /// Null when the list is valid, otherwise what is wrong with it. + public static string? Check(string list) + { + try + { + Parse(list); + return null; + } + catch (FormatException ex) + { + return ex.Message; + } + } + + public static IReadOnlyList Parse(string list) + { + ArgumentNullException.ThrowIfNull(list); + var events = new List(); + var number = 0; + foreach (var raw in list.Split('\n')) + { + number++; + var line = raw.Trim(); + if (line.Length == 0 || line.StartsWith('#')) + { + continue; + } + + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 2 || !DateOnly.TryParseExact(parts[1], "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var day)) + { + throw new FormatException($"Riga {number} dell'elenco delle banche centrali non valida: «{line}». Scrivi per esempio «FED 2026-12-09»."); + } + + var bank = parts[0].ToUpperInvariant(); + var (title, currency, at) = bank switch + { + "FED" => ("Decisione sui tassi della Fed", "USD", Zones.At(day, new TimeOnly(14, 0), Zones.NewYork)), + "BCE" or "ECB" => ("Decisione sui tassi della BCE", "EUR", Zones.At(day, new TimeOnly(14, 15), Zones.Frankfurt)), + _ => throw new FormatException($"Riga {number}: banca «{parts[0]}» sconosciuta, usa FED o BCE."), + }; + events.Add(new CalendarEvent(CalendarEventKinds.CentralBank, title, at, TimeKnown: true, Source, $"cb:{currency}:{day:yyyy-MM-dd}") + { + Currency = currency, + Impact = "High", + }); + } + + return events; + } +} + +/// Crypto Fear and Greed index of alternative.me (attribution required: the interface names the source). +public static class FearGreed +{ + public const string Source = "alternative.me"; + + public const string Attribution = "Indice di paura e avidità: alternative.me"; + + public static Uri Latest(int days) => new($"https://api.alternative.me/fng/?limit={days.ToString(CultureInfo.InvariantCulture)}&format=json"); + + public static IReadOnlyList Parse(string json) + { + using var document = JsonDocument.Parse(json); + return document.RootElement.GetProperty("data").EnumerateArray() + .Select(item => + { + var value = double.Parse(ForexFactory.Text(item, "value"), CultureInfo.InvariantCulture); + var at = DateTimeOffset.FromUnixTimeSeconds(long.Parse(ForexFactory.Text(item, "timestamp"), CultureInfo.InvariantCulture)); + return new MoodReading(MoodIndicators.FearGreed, at, value, MoodLabels.FearGreed(value), Source); + }) + .OrderBy(m => m.AsOf) + .ToList(); + } +} + +/// Daily VIX history from Cboe (no key). The file answers with a redirect to cdn-api.cboe.com. +public static class VixHistory +{ + public const string Source = "cboe"; + + public static readonly Uri Csv = new("https://cdn.cboe.com/api/global/us_indices/daily_prices/VIX_History.csv"); + + /// Closes on or after , dated at midnight UTC of the session day. + public static IReadOnlyList Parse(string csv, DateOnly from) + { + ArgumentNullException.ThrowIfNull(csv); + var readings = new List(); + foreach (var line in csv.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Skip(1)) + { + var f = line.Split(','); + var day = DateOnly.ParseExact(f[0], "MM/dd/yyyy", CultureInfo.InvariantCulture); + if (day < from || f.Length < 5) + { + continue; + } + + var close = double.Parse(f[4], CultureInfo.InvariantCulture); + readings.Add(new MoodReading(MoodIndicators.Vix, QualityChecks.Midnight(day), close, MoodLabels.Vix(close), Source)); + } + + return readings; + } +} + +/// A row of nasdaqtraded.txt: every security traded in the USA with its ETF flag. +public sealed record ListedSecurity(string Symbol, string Name, string Exchange, bool IsEtf, bool IsTest, string FinancialStatus) +{ + /// + /// ETF, ETN, fund, or not a common share (preferred, warrant, right, unit, note): excluded from the stock universe + /// (SPEC 4.6, 8.2). The ETF flag misses many ETNs and every closed-end fund, so the name is read too. + /// + public bool IsExcluded => IsEtf || IsTest || NotCommon().IsMatch(Name); + + private static Regex NotCommon() => NasdaqTraded.NotCommonPattern(); +} + +/// The public list of US securities (SPEC 4.1): https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqtraded.txt. +public static partial class NasdaqTraded +{ + public const string Source = "nasdaqtrader"; + + public static readonly Uri File = new("https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqtraded.txt"); + + /// Listing exchange codes: Q Nasdaq, N NYSE, A NYSE American, P NYSE Arca, Z Cboe BZX, V IEX, F Texas Stock Exchange, M NYSE Texas. + public static readonly IReadOnlyDictionary Exchanges = new Dictionary + { + ["Q"] = "NASDAQ", + ["N"] = "NYSE", + ["A"] = "AMEX", + ["P"] = "ARCA", + ["Z"] = "BATS", + ["V"] = "IEX", + ["F"] = "TXSE", + ["M"] = "NYSETX", + }; + + [GeneratedRegex(@"\b(ETF|ETN|ETNs|Exchange[- ]Traded|Fund|Funds|Trust Units?|Warrants?|Rights?|Units?|Preferred|Notes?|Debentures?|Depositary Shares? representing a fraction|Closed[- ]End|Income Shares|Beneficial Interest|Index|Leveraged|Inverse|SPAC)\b", RegexOptions.IgnoreCase)] + internal static partial Regex NotCommonPattern(); + + /// Rows of the pipe-separated file (CRLF), header and footer skipped. + public static IReadOnlyList Parse(string text) + { + ArgumentNullException.ThrowIfNull(text); + var lines = text.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var header = lines[0].Split('|'); + int Column(string name) => Array.IndexOf(header, name) is var i and >= 0 ? i : throw new FormatException("nasdaqtraded.txt: manca la colonna " + name); + int symbol = Column("Symbol"), name = Column("Security Name"), exchange = Column("Listing Exchange"), etf = Column("ETF"), + test = Column("Test Issue"), status = Column("Financial Status"); + var list = new List(); + foreach (var line in lines.Skip(1)) + { + if (line.StartsWith("File Creation Time", StringComparison.Ordinal)) + { + continue; + } + + var f = line.Split('|'); + if (f.Length < header.Length) + { + continue; + } + + list.Add(new ListedSecurity(f[symbol], f[name], Exchanges.GetValueOrDefault(f[exchange], f[exchange]), f[etf] == "Y", f[test] == "Y", f[status])); + } + + return list; + } +} + +/// +/// The public archives of data.binance.vision (SPEC 4.2, 6.6): monthly zips of klines with a SHA-256 checksum file. +/// Research only: Binance does not serve Italian residents since July 2026. +/// +public static class BinanceArchive +{ + public const string Host = "https://data.binance.vision"; + + public static Uri Monthly(string symbol, string interval, int year, int month) => + new($"{Host}/data/spot/monthly/klines/{symbol}/{interval}/{symbol}-{interval}-{year:D4}-{month:D2}.zip"); + + public static Uri Daily(string symbol, string interval, DateOnly day) => + new($"{Host}/data/spot/daily/klines/{symbol}/{interval}/{symbol}-{interval}-{day:yyyy-MM-dd}.zip"); + + public static Uri ChecksumOf(Uri zip) => new(zip + ".CHECKSUM"); + + /// Checks the zip against its CHECKSUM file: "<sha256 hex> <file name>". + public static bool Verify(byte[] zip, string checksumFile) + { + ArgumentNullException.ThrowIfNull(checksumFile); + var expected = checksumFile.Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries)[0]; + return string.Equals(Convert.ToHexStringLower(SHA256.HashData(zip)), expected, StringComparison.OrdinalIgnoreCase); + } + + public static IReadOnlyList ReadZip(byte[] zip) + { + using var archive = new ZipArchive(new MemoryStream(zip), ZipArchiveMode.Read); + var entry = archive.Entries.Single(e => e.Name.EndsWith(".csv", StringComparison.OrdinalIgnoreCase)); + using var reader = new StreamReader(entry.Open()); + return ParseCsv(reader.ReadToEnd()); + } + + /// + /// Klines without header: open_time, open, high, low, close, volume, close_time, quote_volume, trades, … The + /// open time is in milliseconds up to 2024 and in microseconds from 2025; the last column is ignored. + /// + public static IReadOnlyList ParseCsv(string csv) + { + ArgumentNullException.ThrowIfNull(csv); + var bars = new List(); + foreach (var line in csv.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var f = line.Split(','); + if (f.Length < 9 || !long.TryParse(f[0], NumberStyles.None, CultureInfo.InvariantCulture, out var time)) + { + continue; // A header line some archives carry. + } + + bars.Add(new ResearchBar(FromEpoch(time), Number(f[1]), Number(f[2]), Number(f[3]), Number(f[4]), Number(f[5])) + { + QuoteVolume = Number(f[7]), + Trades = long.Parse(f[8], NumberStyles.None, CultureInfo.InvariantCulture), + }); + } + + return bars; + } + + /// Microseconds when the number is too large to be milliseconds (after year 5138). + public static DateTimeOffset FromEpoch(long value) => + value >= 100_000_000_000_000 ? DateTimeOffset.UnixEpoch.AddTicks(value * 10) : DateTimeOffset.FromUnixTimeMilliseconds(value); + + private static decimal Number(string text) => decimal.Parse(text, NumberStyles.Float, CultureInfo.InvariantCulture); +} + +/// Time zones of the exchanges and central banks, from the system's database (both Windows and Linux know the IANA ids). +public static class Zones +{ + public static readonly TimeZoneInfo NewYork = TimeZoneInfo.FindSystemTimeZoneById("America/New_York"); + + public static readonly TimeZoneInfo Frankfurt = TimeZoneInfo.FindSystemTimeZoneById("Europe/Berlin"); + + public static readonly TimeZoneInfo Rome = TimeZoneInfo.FindSystemTimeZoneById("Europe/Rome"); + + public static DateTimeOffset At(DateOnly day, TimeOnly time, TimeZoneInfo zone) + { + ArgumentNullException.ThrowIfNull(zone); + var local = day.ToDateTime(time); + return new DateTimeOffset(local, zone.GetUtcOffset(local)).ToUniversalTime(); + } +} diff --git a/src/Anubis/Data/Importers.cs b/src/Anubis/Data/Importers.cs new file mode 100644 index 0000000..3f5ea09 --- /dev/null +++ b/src/Anubis/Data/Importers.cs @@ -0,0 +1,246 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; +using Anubis.Brokers; +using Anubis.Core; + +namespace Anubis.Data; + +/// What an import did, for the log and the interface. +public sealed record ImportResult(string Venue, string Symbol, string Timeframe, int Bars, DateTimeOffset? First, DateTimeOffset? Last, IReadOnlyList Notes); + +/// +/// Downloads the monthly kline archives of data.binance.vision into the research history (issue #21). Every zip +/// is checked against its SHA-256 file before use; a month not published yet (404) ends the run without error. +/// Years already complete on disk are skipped. +/// +public sealed class BinanceImporter(ApiHttp http, ResearchStore store, IClock clock) +{ + /// The archive serves files from S3: a few requests a second is far below any limit. + private readonly RateLimiter _limiter = new(clock, 5, TimeSpan.FromSeconds(1)); + + public async Task ImportAsync(string symbol, string interval, int fromYear, int fromMonth, CancellationToken cancellationToken) + { + var timeframe = interval == "1m" ? Timeframes.Minute : interval == "1d" ? Timeframes.Day : interval; + var notes = new List(); + var total = 0; + DateTimeOffset? first = null, last = null; + var today = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime); + var have = store.Years(Venues.Binance, symbol, timeframe).ToHashSet(); + for (var month = new DateOnly(fromYear, fromMonth, 1); month <= today; month = month.AddMonths(1)) + { + // Months go in order, so a year is complete once the next one has a file. A year interrupted halfway is + // read again: writing merges with what is on disk. + if (have.Contains(month.Year) && have.Contains(month.Year + 1)) + { + continue; + } + + var url = BinanceArchive.Monthly(symbol, interval, month.Year, month.Month); + var zip = await http.GetBytesAsync(url, _limiter, null, cancellationToken).ConfigureAwait(false); + if (zip is null) + { + if (total > 0 || month >= new DateOnly(today.Year, today.Month, 1).AddMonths(-1)) + { + notes.Add($"{month:yyyy-MM}: archivio non ancora pubblicato"); + break; + } + + continue; // Before the pair was listed. + } + + var checksum = await http.GetStringAsync(BinanceArchive.ChecksumOf(url), _limiter, null, cancellationToken).ConfigureAwait(false); + if (!BinanceArchive.Verify(zip, checksum)) + { + throw new MarketDataException($"Binance: l'archivio {month:yyyy-MM} di {symbol} non corrisponde alla sua impronta SHA-256.", retryable: true); + } + + var bars = BinanceArchive.ReadZip(zip); + if (bars.Count == 0) + { + continue; + } + + await store.WriteAsync(Venues.Binance, symbol, timeframe, bars, cancellationToken).ConfigureAwait(false); + total += bars.Count; + first ??= bars[0].OpensAt; + last = bars[^1].OpensAt; + } + + return new ImportResult(Venues.Binance, symbol, timeframe, total, first, last, notes); + } +} + +/// +/// Column mapping of a user CSV in /data/import (SPEC 6.6): a JSON file next to the CSV with the same name and +/// .mapping.json, or mapping.json for the whole folder. Columns are named by header or by 0-based number. +/// +public sealed record CsvMapping +{ + /// Symbol of the series, e.g. BTCEUR; when empty, the part of the file name before the first '-' or '_'. + public string? Symbol { get; init; } + + public string Venue { get; init; } = "import"; + + public string Timeframe { get; init; } = Timeframes.Minute; + + public string Delimiter { get; init; } = ","; + + public bool HasHeader { get; init; } = true; + + public string Time { get; init; } = "0"; + + /// unix_ms, unix_s, unix_us, auto (milliseconds or microseconds by size), or a .NET date format such as yyyy-MM-dd HH:mm:ss. + public string TimeFormat { get; init; } = "auto"; + + /// Time zone of text times (IANA id); unix times are always UTC. + public string TimeZone { get; init; } = "UTC"; + + public string Open { get; init; } = "1"; + + public string High { get; init; } = "2"; + + public string Low { get; init; } = "3"; + + public string Close { get; init; } = "4"; + + public string Volume { get; init; } = "5"; + + public string? QuoteVolume { get; init; } + + /// Decimal separator of the numbers: "." or ","; with "," a "." is read as the thousands separator. + public string DecimalSeparator { get; init; } = "."; + + /// True for a market that never closes (crypto): every missing minute or day is reported as a gap. + public bool Continuous { get; init; } = true; + + public static readonly JsonSerializerOptions Json = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; +} + +/// +/// The generic importer of user CSV files with one-minute or daily bars into the research history (issue #21). +/// Lines that cannot be read are counted and reported, never silently fixed. +/// +public sealed class CsvImporter(ResearchStore store) +{ + public const int MaxReportedErrors = 5; + + public static CsvMapping MappingFor(string csvPath) + { + ArgumentNullException.ThrowIfNull(csvPath); + var own = Path.ChangeExtension(csvPath, ".mapping.json"); + var shared = Path.Combine(Path.GetDirectoryName(csvPath) ?? ".", "mapping.json"); + var path = File.Exists(own) ? own : File.Exists(shared) ? shared : null; + return path is null ? new CsvMapping() : JsonSerializer.Deserialize(File.ReadAllText(path), CsvMapping.Json) ?? new CsvMapping(); + } + + public async Task ImportAsync(string csvPath, CsvMapping mapping, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(mapping); + var symbol = string.IsNullOrWhiteSpace(mapping.Symbol) + ? Path.GetFileNameWithoutExtension(csvPath).Split('-', '_')[0].ToUpperInvariant() + : mapping.Symbol; + var zone = TimeZoneInfo.FindSystemTimeZoneById(mapping.TimeZone); + var numbers = new NumberFormatInfo { NumberDecimalSeparator = mapping.DecimalSeparator, NumberGroupSeparator = mapping.DecimalSeparator == "," ? "." : "," }; + var style = mapping.DecimalSeparator == "," ? NumberStyles.Float | NumberStyles.AllowThousands : NumberStyles.Float; + var bars = new List(); + var notes = new List(); + var errors = 0; + string[]? header = null; + var lineNumber = 0; + foreach (var line in await File.ReadAllLinesAsync(csvPath, cancellationToken).ConfigureAwait(false)) + { + lineNumber++; + if (line.Length == 0) + { + continue; + } + + var fields = line.Split(mapping.Delimiter); + if (header is null && mapping.HasHeader) + { + header = fields.Select(f => f.Trim().Trim('"')).ToArray(); + continue; + } + + try + { + string Field(string column) => fields[Index(column, header)].Trim().Trim('"'); + decimal Number(string column) => decimal.Parse(Field(column), style, numbers); + bars.Add(new ResearchBar(ParseTime(Field(mapping.Time), mapping.TimeFormat, zone), Number(mapping.Open), Number(mapping.High), + Number(mapping.Low), Number(mapping.Close), Number(mapping.Volume)) + { + QuoteVolume = mapping.QuoteVolume is { Length: > 0 } q ? Number(q) : null, + }); + } + catch (Exception ex) when (ex is FormatException or IndexOutOfRangeException or OverflowException or ArgumentException) + { + if (++errors <= MaxReportedErrors) + { + notes.Add($"riga {lineNumber}: {ex.Message}"); + } + } + } + + if (errors > 0) + { + notes.Add($"{errors} righe non lette su {lineNumber}"); + } + + var ordered = bars.OrderBy(b => b.OpensAt).ToList(); + IReadOnlyList issues = []; + if (ordered.Count > 0 && mapping.Timeframe == Timeframes.Minute) + { + issues = QualityChecks.CheckMinutes(ordered.Select(b => b.ToBar()).ToList(), + mapping.Continuous ? QualityChecks.EveryMinute(ordered[0].OpensAt, ordered[^1].OpensAt.AddMinutes(1)) : null); + } + else if (ordered.Count > 0) + { + var days = ordered.Select(b => new DailyBar(DateOnly.FromDateTime(b.OpensAt.UtcDateTime), (double)b.Open, (double)b.High, (double)b.Low, + (double)b.Close, (double)b.Volume)).ToList(); + issues = QualityChecks.CheckDaily(days, mapping.Continuous ? QualityChecks.EveryDay(days[0].Day, days[^1].Day) : null); + } + + notes.AddRange(issues.Take(MaxReportedErrors).Select(i => $"{i.Kind} {i.StartsAt:yyyy-MM-dd HH:mm}: {i.Detail}")); + + await store.WriteAsync(mapping.Venue, symbol, mapping.Timeframe, bars, cancellationToken).ConfigureAwait(false); + return new ImportResult(mapping.Venue, symbol, mapping.Timeframe, bars.Count, ordered.FirstOrDefault()?.OpensAt, ordered.LastOrDefault()?.OpensAt, notes); + } + + private static int Index(string column, string[]? header) + { + if (int.TryParse(column, NumberStyles.None, CultureInfo.InvariantCulture, out var index)) + { + return index; + } + + var found = header is null ? -1 : Array.FindIndex(header, h => string.Equals(h, column, StringComparison.OrdinalIgnoreCase)); + return found >= 0 ? found : throw new ArgumentException($"colonna «{column}» assente"); + } + + private static DateTimeOffset ParseTime(string text, string format, TimeZoneInfo zone) + { + switch (format) + { + case "unix_s": + return DateTimeOffset.FromUnixTimeSeconds(long.Parse(text, CultureInfo.InvariantCulture)); + case "unix_ms": + return DateTimeOffset.FromUnixTimeMilliseconds(long.Parse(text, CultureInfo.InvariantCulture)); + case "unix_us": + return DateTimeOffset.UnixEpoch.AddTicks(long.Parse(text, CultureInfo.InvariantCulture) * 10); + case "auto" when long.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out var number): + return number < 100_000_000_000 ? DateTimeOffset.FromUnixTimeSeconds(number) : BinanceArchive.FromEpoch(number); + case "auto": + return DateTimeOffset.Parse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); + default: + var local = DateTime.ParseExact(text, format, CultureInfo.InvariantCulture, DateTimeStyles.None); + return new DateTimeOffset(local, zone.GetUtcOffset(local)).ToUniversalTime(); + } + } +} diff --git a/src/Anubis/Data/ReferenceData.cs b/src/Anubis/Data/ReferenceData.cs new file mode 100644 index 0000000..795ed57 --- /dev/null +++ b/src/Anubis/Data/ReferenceData.cs @@ -0,0 +1,278 @@ +using System.Collections.Concurrent; +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Data; + +/// A reference exchange rate: units of for one , as published for . +public sealed record FxRate(string Base, string Quote, DateOnly Date, decimal Rate, string Source); + +/// +/// Reference exchange rates (SPEC 6.8): one per day as the ECB publishes them. A day without a rate uses the last +/// one before it, as the tax export requires (SPEC 4.6). +/// +public sealed class FxStore(Database db, IClock clock) +{ + public async Task UpsertAsync(IEnumerable rates, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(rates); + var list = rates.ToList(); + var now = clock.UtcNow; + await db.WriteAsync(async s => + { + foreach (var rate in list) + { + await s.ExecuteAsync( + "INSERT INTO fx_rates (id, base, quote, rate_date, rate, source, created_at) VALUES (@id, @b, @q, @d, @r, @s, @now) " + + "ON CONFLICT (base, quote, rate_date) DO UPDATE SET rate = excluded.rate, source = excluded.source", + cancellationToken, ("id", Ids.NewId(clock)), ("b", rate.Base), ("q", rate.Quote), ("d", BarStore.DayText(rate.Date)), + ("r", rate.Rate), ("s", rate.Source), ("now", now)).ConfigureAwait(false); + } + }, cancellationToken).ConfigureAwait(false); + return list.Count; + } + + /// The rate published for , or the last one before it; null if none is known. + public Task RateOnAsync(string baseCurrency, string quote, DateOnly day, CancellationToken cancellationToken) => + db.ReadAsync(s => s.FirstOrDefaultAsync( + "SELECT base, quote, rate_date, rate, source FROM fx_rates WHERE base = @b AND quote = @q AND rate_date <= @d ORDER BY rate_date DESC", + Read, ("b", baseCurrency), ("q", quote), ("d", BarStore.DayText(day))), cancellationToken); + + public Task LatestAsync(string baseCurrency, string quote, CancellationToken cancellationToken) => + RateOnAsync(baseCurrency, quote, DateOnly.MaxValue, cancellationToken); + + private static FxRate Read(DbRow r) => + new(r.Text("base"), r.Text("quote"), BarStore.ParseDay(r.Text("rate_date")), r.Money("rate"), r.Text("source")); +} + +public static class CalendarEventKinds +{ + public const string Economic = "economic"; + public const string CentralBank = "central_bank"; + public const string Earnings = "earnings"; + public const string Holiday = "holiday"; +} + +/// +/// An event of the calendar (SPEC 10.4). is false when the source gives only the day +/// (earnings without an hour, holidays): is then midnight of that day in the exchange's zone. +/// +public sealed record CalendarEvent(string Kind, string Title, DateTimeOffset OccursAt, bool TimeKnown, string Source, string SourceKey) +{ + public string? Currency { get; init; } + + public string? Symbol { get; init; } + + /// For earnings: bmo before the open, amc after the close, dmh during market hours. + public string? TimeHint { get; init; } + + /// Low, Medium or High. + public string? Impact { get; init; } + + public string? Forecast { get; init; } + + public string? Previous { get; init; } +} + +public sealed class CalendarEventStore(Database db, IClock clock) +{ + private const string Columns = "kind, title, currency, symbol, occurs_at, time_known, time_hint, impact, forecast, previous, source, source_key"; + + /// Adds new events and updates known ones (a moved date, a new forecast). Returns how many arrived. + public async Task UpsertAsync(IEnumerable events, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(events); + var list = events.DistinctBy(e => e.SourceKey).ToList(); + var now = clock.UtcNow; + await db.WriteAsync(async s => + { + foreach (var e in list) + { + await s.ExecuteAsync( + $"INSERT INTO calendar_events (id, {Columns}, created_at, updated_at) VALUES (@id, @kind, @title, @ccy, @sym, @at, @known, @hint, @impact, " + + "@forecast, @previous, @source, @key, @now, @now) ON CONFLICT (source_key) DO UPDATE SET kind = excluded.kind, title = excluded.title, " + + "currency = excluded.currency, symbol = excluded.symbol, occurs_at = excluded.occurs_at, time_known = excluded.time_known, " + + "time_hint = excluded.time_hint, impact = excluded.impact, forecast = excluded.forecast, previous = excluded.previous, updated_at = excluded.updated_at", + cancellationToken, ("id", Ids.NewId(clock)), ("kind", e.Kind), ("title", e.Title), ("ccy", e.Currency), ("sym", e.Symbol), ("at", e.OccursAt), + ("known", e.TimeKnown), ("hint", e.TimeHint), ("impact", e.Impact), ("forecast", e.Forecast), ("previous", e.Previous), ("source", e.Source), + ("key", e.SourceKey), ("now", now)).ConfigureAwait(false); + } + }, cancellationToken).ConfigureAwait(false); + return list.Count; + } + + public async Task> BetweenAsync(DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync($"SELECT {Columns} FROM calendar_events WHERE occurs_at >= @f AND occurs_at < @t ORDER BY occurs_at, title", + r => new CalendarEvent(r.Text("kind"), r.Text("title"), r.Moment("occurs_at"), r.Bool("time_known"), r.Text("source"), r.Text("source_key")) + { + Currency = r.TextOrNull("currency"), + Symbol = r.TextOrNull("symbol"), + TimeHint = r.TextOrNull("time_hint"), + Impact = r.TextOrNull("impact"), + Forecast = r.TextOrNull("forecast"), + Previous = r.TextOrNull("previous"), + }, ("f", from), ("t", to)), cancellationToken).ConfigureAwait(false); +} + +public static class MoodIndicators +{ + public const string FearGreed = "fear_greed"; + public const string Vix = "vix"; + public const string BtcFunding = "btc_funding"; + public const string BtcOpenInterest = "btc_open_interest"; +} + +public sealed record MoodReading(string Indicator, DateTimeOffset AsOf, double Value, string? Label, string Source); + +/// The labels of SPEC 10.4, in the words the interface shows. +public static class MoodLabels +{ + public static string FearGreed(double value) => value switch + { + <= 24 => "paura estrema", + <= 44 => "paura", + <= 55 => "neutro", + <= 75 => "avidità", + _ => "avidità estrema", + }; + + public static string Vix(double value) => value switch + { + < 20 => "calmo", + <= 30 => "nervoso", + _ => "panico", + }; + + /// is the funding of one period; its length (usually 8 hours). + public static string Funding(double rate, TimeSpan interval) + { + var perEightHours = interval > TimeSpan.Zero ? rate * (TimeSpan.FromHours(8) / interval) : rate; + return perEightHours switch + { + > 0.0002 => "troppi al rialzo", + < -0.0002 => "troppi al ribasso", + _ => "nella norma", + }; + } +} + +public sealed class MoodStore(Database db, IClock clock) +{ + public async Task UpsertAsync(IEnumerable readings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(readings); + var list = readings.ToList(); + var now = clock.UtcNow; + await db.WriteAsync(async s => + { + foreach (var m in list) + { + await s.ExecuteAsync( + "INSERT INTO market_mood (id, indicator, as_of, value, label, source, created_at) VALUES (@id, @i, @a, @v, @l, @s, @now) " + + "ON CONFLICT (indicator, as_of) DO UPDATE SET value = excluded.value, label = excluded.label, source = excluded.source", + cancellationToken, ("id", Ids.NewId(clock)), ("i", m.Indicator), ("a", m.AsOf), ("v", m.Value), ("l", m.Label), ("s", m.Source), ("now", now)) + .ConfigureAwait(false); + } + }, cancellationToken).ConfigureAwait(false); + return list.Count; + } + + public Task LatestAsync(string indicator, CancellationToken cancellationToken) => + db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT indicator, as_of, value, label, source FROM market_mood WHERE indicator = @i ORDER BY as_of DESC", + r => new MoodReading(r.Text("indicator"), r.Moment("as_of"), r.Measure("value"), r.TextOrNull("label"), r.Text("source")), ("i", indicator)), + cancellationToken); +} + +/// +/// The exchange calendar as the broker gives it, saved day by day so the scheduler can read it without network +/// (SPEC 5.5: market times come from the broker's calendar, never from Italian time). +/// +public sealed class StoredMarketCalendar(Database db, IClock clock, string market) : IMarketCalendar +{ + private readonly ConcurrentDictionary _days = new(); + + public string Market { get; } = market; + + /// First and last day the stored calendar covers; null when it is empty. + public (DateOnly First, DateOnly Last)? Coverage => _days.IsEmpty ? null : (_days.Keys.Min(), _days.Keys.Max()); + + /// The session of ; null when the market is closed or the day is not covered yet. + public MarketSession? SessionOn(DateOnly day) => _days.TryGetValue(day, out var session) ? session : null; + + public bool Covers(DateOnly day) => _days.ContainsKey(day); + + /// Saves the sessions of [first, last]; days without a session are saved as closed. + public async Task SaveAsync(IReadOnlyCollection sessions, DateOnly first, DateOnly last, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(sessions); + var byDay = sessions.Where(x => x.Date >= first && x.Date <= last).ToDictionary(x => x.Date); + var now = clock.UtcNow; + await db.WriteAsync(async s => + { + for (var day = first; day <= last; day = day.AddDays(1)) + { + var session = byDay.GetValueOrDefault(day); + await s.ExecuteAsync( + "INSERT INTO market_calendar (id, market, session_date, is_open, open_at, close_at, created_at) VALUES (@id, @m, @d, @o, @oa, @ca, @now) " + + "ON CONFLICT (market, session_date) DO UPDATE SET is_open = excluded.is_open, open_at = excluded.open_at, close_at = excluded.close_at", + cancellationToken, ("id", Ids.NewId(clock)), ("m", Market), ("d", BarStore.DayText(day)), ("o", session is not null), + ("oa", session?.OpenUtc), ("ca", session?.CloseUtc), ("now", now)).ConfigureAwait(false); + } + }, cancellationToken).ConfigureAwait(false); + + for (var day = first; day <= last; day = day.AddDays(1)) + { + _days[day] = byDay.GetValueOrDefault(day); + } + } + + public async Task LoadAsync(CancellationToken cancellationToken) + { + var rows = await db.ReadAsync(s => s.QueryAsync("SELECT session_date, is_open, open_at, close_at FROM market_calendar WHERE market = @m", + r => + { + var day = BarStore.ParseDay(r.Text("session_date")); + return (Day: day, Session: r.Bool("is_open") ? new MarketSession(day, r.Moment("open_at"), r.Moment("close_at")) : null); + }, ("m", Market)), cancellationToken).ConfigureAwait(false); + foreach (var (day, session) in rows) + { + _days[day] = session; + } + } + + /// Sessions in [first, last], oldest first. + public IReadOnlyList Sessions(DateOnly first, DateOnly last) => + _days.Where(d => d.Key >= first && d.Key <= last && d.Value is not null).Select(d => d.Value!).OrderBy(x => x.Date).ToList(); +} + +/// State of an external source: when it was last asked, last answered, and why it is off (SPEC 4.4). +public sealed record SourceState(string Source, bool Enabled, DateTimeOffset? LastRequestAt, DateTimeOffset? LastSuccessAt, string? LastError, string? Note); + +public sealed class SourceStateStore(Database db, IClock clock) +{ + public Task RecordAsync(string source, bool enabled, bool success, string? error, string? note, CancellationToken cancellationToken) + { + var now = clock.UtcNow; + return db.WriteAsync(async s => + { + var exists = await s.CountAsync("SELECT COUNT(*) FROM source_state WHERE source = @s", ("s", source)).ConfigureAwait(false) > 0; + if (!exists) + { + await s.ExecuteAsync("INSERT INTO source_state (id, source, enabled, created_at, updated_at) VALUES (@id, @s, @e, @now, @now)", + cancellationToken, ("id", Ids.NewId(clock)), ("s", source), ("e", enabled), ("now", now)).ConfigureAwait(false); + } + + await s.ExecuteAsync(success + ? "UPDATE source_state SET enabled = @e, last_request_at = @now, last_success_at = @now, last_error = NULL, note = @n, updated_at = @now WHERE source = @s" + : "UPDATE source_state SET enabled = @e, last_request_at = @now, last_error = @err, note = @n, updated_at = @now WHERE source = @s", + cancellationToken, success + ? [("e", enabled), ("now", now), ("n", note), ("s", source)] + : [("e", enabled), ("now", now), ("err", error), ("n", note), ("s", source)]).ConfigureAwait(false); + }, cancellationToken); + } + + public async Task> ListAsync(CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync("SELECT source, enabled, last_request_at, last_success_at, last_error, note FROM source_state ORDER BY source", + r => new SourceState(r.Text("source"), r.Bool("enabled"), r.MomentOrNull("last_request_at"), r.MomentOrNull("last_success_at"), + r.TextOrNull("last_error"), r.TextOrNull("note"))), cancellationToken).ConfigureAwait(false); +} diff --git a/src/Anubis/Data/ReferenceSources.cs b/src/Anubis/Data/ReferenceSources.cs new file mode 100644 index 0000000..3f8c60d --- /dev/null +++ b/src/Anubis/Data/ReferenceSources.cs @@ -0,0 +1,221 @@ +using System.Globalization; +using Anubis.Brokers; +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Data; + +/// +/// Downloads the free sources of SPEC 4.4 within their limits and records each attempt in source_state, so +/// limits hold across restarts and the interface can say what is off and why. A source that fails is retried at +/// its next turn; it never stops the others. +/// +public sealed class ReferenceSources( + ApiHttp http, + Database db, + IClock clock, + SettingsStore settings, + Func> secret, + OkxClient? okx = null) +{ + /// Currencies kept from the ECB: the dollar for display and the tax export (SPEC 6.8). + public static readonly IReadOnlyList Currencies = ["USD"]; + + public const string OkxDerivatives = "okx_derivatives"; + + /// + /// Shortest time between two requests to the same source. Forex Factory asks for no more than two requests every + /// five minutes; the others publish once a day or less. + /// + public static readonly IReadOnlyDictionary MinInterval = new Dictionary + { + [Ecb.Source] = TimeSpan.FromMinutes(30), + [ForexFactory.Source] = TimeSpan.FromMinutes(55), + [Fred.Source] = TimeSpan.FromHours(12), + [Finnhub.Source] = TimeSpan.FromHours(12), + [FearGreed.Source] = TimeSpan.FromMinutes(55), + [VixHistory.Source] = TimeSpan.FromHours(6), + [OkxDerivatives] = TimeSpan.FromMinutes(55), + [CentralBanks.Source] = TimeSpan.Zero, + }; + + private readonly SourceStateStore _states = new(db, clock); + + private readonly FxStore _fx = new(db, clock); + + private readonly CalendarEventStore _calendar = new(db, clock); + + private readonly MoodStore _mood = new(db, clock); + + /// Reference rates: the whole history the first time, the last 90 days after a pause, otherwise the day's file. + public Task RefreshFxAsync(CancellationToken cancellationToken) => RunAsync(Ecb.Source, async () => + { + var latest = await _fx.LatestAsync("EUR", "USD", cancellationToken).ConfigureAwait(false); + var today = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime); + IReadOnlyList rates; + if (latest is null) + { + var zip = await http.GetBytesAsync(Ecb.History, null, null, cancellationToken).ConfigureAwait(false) + ?? throw new MarketDataException("BCE: archivio storico dei cambi non trovato.", retryable: true); + rates = Ecb.ParseHistoryCsv(Ecb.ExtractHistoryCsv(zip), Currencies); + } + else + { + var url = latest.Date < today.AddDays(-4) ? Ecb.Last90Days : Ecb.Daily; + rates = Ecb.ParseXml(await http.GetStringAsync(url, null, null, cancellationToken).ConfigureAwait(false), Currencies); + } + + await _fx.UpsertAsync(rates, cancellationToken).ConfigureAwait(false); + return $"{rates.Count} cambi"; + }, cancellationToken); + + /// The economic calendar of this week (Forex Factory), plus the central bank list of the settings. + public async Task RefreshCalendarAsync(CancellationToken cancellationToken) + { + var banks = await RunAsync(CentralBanks.Source, async () => + { + var list = await settings.GetAsync(SettingsStore.CentralBankMeetings.Key, cancellationToken).ConfigureAwait(false); + var events = CentralBanks.Parse(list.Value); + await _calendar.UpsertAsync(events, cancellationToken).ConfigureAwait(false); + var last = events.Count == 0 ? (DateTimeOffset?)null : events.Max(e => e.OccursAt); + return last is { } l && l < clock.UtcNow.AddDays(60) + ? $"L'elenco delle riunioni finisce il {l:dd/MM/yyyy}: aggiungi le date del nuovo anno nelle impostazioni." + : $"{events.Count} riunioni"; + }, cancellationToken).ConfigureAwait(false); + + var week = await RunAsync(ForexFactory.Source, async () => + { + var events = ForexFactory.Parse(await http.GetStringAsync(ForexFactory.ThisWeek, null, null, cancellationToken).ConfigureAwait(false)); + await _calendar.UpsertAsync(events, cancellationToken).ConfigureAwait(false); + return $"{events.Count} eventi"; + }, cancellationToken).ConfigureAwait(false); + return banks && week; + } + + /// Release dates of the main US data from FRED; off without a key, and the interface says so. + public async Task RefreshFredAsync(CancellationToken cancellationToken) + { + var key = await secret("FRED_API_KEY", cancellationToken).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(key)) + { + await _states.RecordAsync(Fred.Source, enabled: false, success: false, null, + "Spento: manca la chiave FRED_API_KEY. Il calendario usa solo Forex Factory.", cancellationToken).ConfigureAwait(false); + return false; + } + + return await RunAsync(Fred.Source, async () => + { + var from = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime); + var count = 0; + foreach (var (id, title) in Fred.Releases) + { + var events = Fred.Parse(await http.GetStringAsync(Fred.ReleaseDates(id, key, from), null, null, cancellationToken).ConfigureAwait(false), title, from); + count += await _calendar.UpsertAsync(events.Where(e => e.OccursAt < clock.UtcNow.AddDays(120)), cancellationToken).ConfigureAwait(false); + } + + return $"{count} date"; + }, cancellationToken).ConfigureAwait(false); + } + + /// + /// Earnings dates for the next two weeks. Without a key the earnings filter is off (SPEC 4.4): the state says so + /// and the strategies read . + /// + public async Task RefreshEarningsAsync(IReadOnlySet? symbols, CancellationToken cancellationToken) + { + var key = await secret("FINNHUB_API_KEY", cancellationToken).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(key)) + { + await _states.RecordAsync(Finnhub.Source, enabled: false, success: false, null, + "Spento: manca la chiave FINNHUB_API_KEY. Il filtro sulle trimestrali è disattivato.", cancellationToken).ConfigureAwait(false); + return false; + } + + return await RunAsync(Finnhub.Source, async () => + { + var today = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime); + var events = Finnhub.Parse(await http.GetStringAsync(Finnhub.Earnings(today, today.AddDays(14), key), null, null, cancellationToken).ConfigureAwait(false), + symbols); + await _calendar.UpsertAsync(events, cancellationToken).ConfigureAwait(false); + return $"{events.Count} trimestrali"; + }, cancellationToken).ConfigureAwait(false); + } + + /// True when earnings dates are known: the source is on and answered in the last two days. + public async Task EarningsFilterActiveAsync(CancellationToken cancellationToken) + { + var state = (await _states.ListAsync(cancellationToken).ConfigureAwait(false)).FirstOrDefault(s => s.Source == Finnhub.Source); + return state is { Enabled: true, LastSuccessAt: { } at } && clock.UtcNow - at < TimeSpan.FromDays(2); + } + + /// Mood indicators: fear and greed, VIX, and Bitcoin funding and open interest from OKX public data. + public async Task RefreshMoodAsync(CancellationToken cancellationToken) + { + var fng = await RunAsync(FearGreed.Source, async () => + { + var first = await _mood.LatestAsync(MoodIndicators.FearGreed, cancellationToken).ConfigureAwait(false) is null; + var readings = FearGreed.Parse(await http.GetStringAsync(FearGreed.Latest(first ? 400 : 2), null, null, cancellationToken).ConfigureAwait(false)); + await _mood.UpsertAsync(readings, cancellationToken).ConfigureAwait(false); + return FearGreed.Attribution; + }, cancellationToken).ConfigureAwait(false); + + var vix = await RunAsync(VixHistory.Source, async () => + { + var from = DateOnly.FromDateTime(clock.UtcNow.UtcDateTime).AddDays(-400); + var readings = VixHistory.Parse(await http.GetStringAsync(VixHistory.Csv, null, null, cancellationToken).ConfigureAwait(false), from); + await _mood.UpsertAsync(readings, cancellationToken).ConfigureAwait(false); + return $"{readings.Count} chiusure"; + }, cancellationToken).ConfigureAwait(false); + + if (okx is null) + { + return fng && vix; + } + + var derivatives = await RunAsync(OkxDerivatives, async () => + { + var funding = await okx.FetchFundingRateAsync("BTC-USDT-SWAP", cancellationToken).ConfigureAwait(false); + var interest = await okx.FetchOpenInterestAsync("BTC-USDT-SWAP", cancellationToken).ConfigureAwait(false); + await _mood.UpsertAsync( + [ + new MoodReading(MoodIndicators.BtcFunding, funding.At, funding.Rate, MoodLabels.Funding(funding.Rate, funding.Interval), Venues.Okx), + new MoodReading(MoodIndicators.BtcOpenInterest, interest.At, interest.Usd, null, Venues.Okx), + ], cancellationToken).ConfigureAwait(false); + return "funding " + funding.Rate.ToString("P4", CultureInfo.InvariantCulture); + }, cancellationToken).ConfigureAwait(false); + return fng && vix && derivatives; + } + + public Task> StatesAsync(CancellationToken cancellationToken) => _states.ListAsync(cancellationToken); + + /// + /// Runs one source if its minimum interval has passed and records the outcome. Returns true when it succeeded or + /// was not due yet; false when it failed. + /// + private async Task RunAsync(string source, Func> work, CancellationToken cancellationToken) + { + var state = (await _states.ListAsync(cancellationToken).ConfigureAwait(false)).FirstOrDefault(s => s.Source == source); + if (state?.LastRequestAt is { } last && clock.UtcNow - last < MinInterval.GetValueOrDefault(source, TimeSpan.FromMinutes(5))) + { + return true; + } + + try + { + var note = await work().ConfigureAwait(false); + await _states.RecordAsync(source, enabled: true, success: true, null, note, cancellationToken).ConfigureAwait(false); + return true; + } + catch (Exception ex) when (ex is MarketDataException or HttpRequestException or FormatException or System.Text.Json.JsonException + or System.Xml.XmlException or TaskCanceledException or InvalidDataException or KeyNotFoundException or InvalidOperationException or IndexOutOfRangeException) + { + if (cancellationToken.IsCancellationRequested) + { + throw; + } + + await _states.RecordAsync(source, enabled: true, success: false, Redactor.Redact(ex.Message), state?.Note, cancellationToken).ConfigureAwait(false); + return false; + } + } +} diff --git a/tests/Anubis.Tests/FakeNetwork.cs b/tests/Anubis.Tests/FakeNetwork.cs index bcf05ca..310fb56 100644 --- a/tests/Anubis.Tests/FakeNetwork.cs +++ b/tests/Anubis.Tests/FakeNetwork.cs @@ -130,6 +130,14 @@ internal sealed class FastClock(DateTimeOffset start) : Anubis.Core.IClock public TimeSpan Waited { get; private set; } + public void Advance(TimeSpan by) + { + lock (_gate) + { + _now += by; + } + } + public Task WaitUntilAsync(DateTimeOffset dueUtc, CancellationToken cancellationToken) { lock (_gate) diff --git a/tests/Anubis.Tests/ReferenceSourcesTests.cs b/tests/Anubis.Tests/ReferenceSourcesTests.cs new file mode 100644 index 0000000..b9ad7e4 --- /dev/null +++ b/tests/Anubis.Tests/ReferenceSourcesTests.cs @@ -0,0 +1,431 @@ +using System.IO.Compression; +using System.Net; +using System.Security.Cryptography; +using System.Text; +using Anubis.Brokers; +using Anubis.Core; +using Anubis.Data; + +namespace Anubis.Tests; + +/// Readers of the free sources on recorded responses, their stores, and the research importers (issues #21, #22, #24). +[TestClass] +public sealed class ReferenceSourcesTests +{ + private static readonly DateTimeOffset _start = new(2026, 11, 10, 8, 0, 0, TimeSpan.Zero); + + [TestMethod] + [TestCategory(Category.Unit)] + public void EcbDailyNinetyDayAndHistoryFilesGiveTheSameRates() + { + string[] wanted = ["USD", "GBP", "BGN"]; + var daily = Ecb.ParseXml(Fixture.Text("free/ecb-daily.xml"), wanted); + Assert.AreEqual(2, daily.Count, "BGN is no longer published"); + Assert.AreEqual(new FxRate("EUR", "USD", new DateOnly(2026, 11, 9), 1.1193m, Ecb.Source), daily.Single(r => r.Quote == "USD")); + Assert.AreEqual(0.83430m, daily.Single(r => r.Quote == "GBP").Rate); + + var ninety = Ecb.ParseXml(Fixture.Text("free/ecb-hist-90d.xml"), ["USD"]); + Assert.IsTrue(ninety.Count > 1); + Assert.AreEqual(1.1193m, ninety.Single(r => r.Date == new DateOnly(2026, 11, 9)).Rate, "the 90-day file uses double quotes"); + CollectionAssert.AreEqual(ninety.OrderBy(r => r.Date).ToList(), ninety.ToList(), "oldest first"); + + var history = Ecb.ParseHistoryCsv(Fixture.Text("free/ecb-hist.csv"), wanted); + Assert.AreEqual(4, history.Count, "N/A is skipped and the trailing comma ignored"); + Assert.AreEqual(new DateOnly(2026, 11, 6), history[0].Date); + Assert.AreEqual(1.1165m, history.Single(r => r.Quote == "USD" && r.Date == new DateOnly(2026, 11, 6)).Rate); + + using var zip = new MemoryStream(); + using (var archive = new ZipArchive(zip, ZipArchiveMode.Create, leaveOpen: true)) + using (var writer = new StreamWriter(archive.CreateEntry("eurofxref-hist.csv").Open())) + { + writer.Write(Fixture.Text("free/ecb-hist.csv")); + } + + Assert.AreEqual(Fixture.Text("free/ecb-hist.csv"), Ecb.ExtractHistoryCsv(zip.ToArray())); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void ForexFactoryEventsKeepTheirTimeAndHolidaysOnlyTheDay() + { + var events = ForexFactory.Parse(Fixture.Text("free/ff-calendar.json")); + Assert.AreEqual(5, events.Count); + var cpi = events.Single(e => e.Title == "CPI m/m"); + Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 13, 30, 0, TimeSpan.Zero), cpi.OccursAt, "the offset -05:00 becomes UTC"); + Assert.AreEqual("USD", cpi.Currency); + Assert.AreEqual("High", cpi.Impact); + Assert.AreEqual("0.3%", cpi.Forecast); + Assert.AreEqual("0.6%", cpi.Previous); + Assert.IsTrue(cpi.TimeKnown); + + var holiday = events.Single(e => e.Title == "Bank Holiday"); + Assert.AreEqual(CalendarEventKinds.Holiday, holiday.Kind); + Assert.IsFalse(holiday.TimeKnown); + Assert.IsNull(holiday.Impact); + Assert.IsNull(events.Single(e => e.Title == "RBA Assist Gov Hunter Speaks").Forecast, "an empty forecast is no forecast"); + Assert.AreEqual(events.Count, events.Select(e => e.SourceKey).Distinct().Count()); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void FredFinnhubAndCentralBankDatesBecomeTimedEvents() + { + var fred = Fred.Parse(Fixture.Text("free/fred-release-dates.json"), "PIL USA", new DateOnly(2010, 1, 1)); + Assert.AreEqual(4, fred.Count, "dates before the start are skipped"); + Assert.AreEqual(new DateTimeOffset(2010, 2, 11, 13, 30, 0, TimeSpan.Zero), fred[0].OccursAt, "08:30 New York in winter"); + Assert.AreEqual("fred:82:2010-02-11", fred[0].SourceKey); + Assert.AreEqual(new DateTimeOffset(2012, 10, 24, 12, 30, 0, TimeSpan.Zero), fred[2].OccursAt, "08:30 New York in summer time"); + StringAssert.Contains(Fred.ReleaseDates(10, "k", new DateOnly(2026, 11, 10)).Query, "include_release_dates_with_no_data=true"); + + var earnings = Finnhub.Parse(Fixture.Text("free/finnhub-earnings.json")); + Assert.AreEqual(2, earnings.Count); + var aapl = earnings.Single(e => e.OccursAt.Year == 2020); + Assert.AreEqual(new DateTimeOffset(2020, 1, 28, 21, 5, 0, TimeSpan.Zero), aapl.OccursAt, "amc: 16:05 New York"); + Assert.AreEqual("amc", aapl.TimeHint); + Assert.AreEqual("AAPL", aapl.Symbol); + Assert.AreEqual(0, Finnhub.Parse(Fixture.Text("free/finnhub-earnings.json"), new HashSet { "MSFT" }).Count); + + var banks = CentralBanks.Parse(CentralBanks.DefaultList); + Assert.AreEqual(18, banks.Count); + var fed = banks.Single(e => e.SourceKey == "cb:USD:2026-12-09"); + Assert.AreEqual(new DateTimeOffset(2026, 12, 9, 19, 0, 0, TimeSpan.Zero), fed.OccursAt, "14:00 New York"); + var ecb = banks.Single(e => e.SourceKey == "cb:EUR:2027-07-22"); + Assert.AreEqual(new DateTimeOffset(2027, 7, 22, 12, 15, 0, TimeSpan.Zero), ecb.OccursAt, "14:15 Frankfurt in summer time"); + + // Every decision day of the default list is in the official calendars read for the fixture. + var official = Fixture.Text("free/central-banks.md"); + foreach (var e in banks) + { + StringAssert.Contains(official, $"| {e.OccursAt:yyyy-MM-dd} |"); + } + + Assert.AreEqual(1, CentralBanks.Parse("# commento\n\nfed 2027-01-27\n").Count); + var error = Assert.ThrowsExactly(() => CentralBanks.Parse("FED 2027-01-27\nBOJ 2027-01-28")); + StringAssert.Contains(error.Message, "Riga 2"); + Assert.ThrowsExactly(() => CentralBanks.Parse("FED 27/01/2027")); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void MoodReadingsCarryTheLabelsOfTheSpec() + { + var fng = FearGreed.Parse(Fixture.Text("free/fng.json")); + Assert.AreEqual(2, fng.Count); + Assert.AreEqual(32d, fng[^1].Value, "oldest first: the latest reading is last"); + Assert.AreEqual("paura", fng[^1].Label); + Assert.AreEqual(DateTimeOffset.FromUnixTimeSeconds(1794268800), fng[^1].AsOf); + + var vix = VixHistory.Parse(Fixture.Text("free/vix.csv"), new DateOnly(2026, 1, 1)); + Assert.AreEqual(2, vix.Count); + Assert.AreEqual(new DateTimeOffset(2026, 11, 9, 0, 0, 0, TimeSpan.Zero), vix[1].AsOf); + Assert.AreEqual(("nervoso", "calmo"), (vix[0].Label, vix[1].Label)); + + Assert.AreEqual("paura estrema", MoodLabels.FearGreed(24)); + Assert.AreEqual("neutro", MoodLabels.FearGreed(55)); + Assert.AreEqual("avidità estrema", MoodLabels.FearGreed(76)); + Assert.AreEqual("panico", MoodLabels.Vix(30.01)); + Assert.AreEqual("troppi al rialzo", MoodLabels.Funding(0.0003, TimeSpan.FromHours(8))); + Assert.AreEqual("troppi al rialzo", MoodLabels.Funding(0.00015, TimeSpan.FromHours(4)), "4-hour funding counts double"); + Assert.AreEqual("nella norma", MoodLabels.Funding(-0.0001, TimeSpan.FromHours(8))); + Assert.AreEqual("troppi al ribasso", MoodLabels.Funding(-0.00021, TimeSpan.FromHours(8))); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void NasdaqListExcludesFundsAndEverythingThatIsNotACommonShare() + { + var list = NasdaqTraded.Parse(Fixture.Text("free/nasdaqtraded.txt")); + Assert.AreEqual(12, list.Count, "header and footer skipped"); + var excluded = list.Where(s => s.IsExcluded).Select(s => s.Symbol).ToList(); + CollectionAssert.AreEquivalent(new[] { "SPY", "BULZ", "AMUB", "PDI", "ABR$D", "AACIW", "ZXIET" }, excluded); + Assert.AreEqual("TXSE", list.Single(s => s.Symbol == "CECO").Exchange); + Assert.AreEqual("NASDAQ", list.Single(s => s.Symbol == "AAPL").Exchange); + Assert.AreEqual("NYSE", list.Single(s => s.Symbol == "KO").Exchange); + Assert.AreEqual("D", list.Single(s => s.Symbol == "AACG").FinancialStatus); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void BinanceArchivesReadMillisecondsAndMicrosecondsAndCheckTheirFingerprint() + { + var old = BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv")); + var recent = BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2025-01.csv")); + Assert.AreEqual(5, old.Count); + Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), old[0].OpensAt); + Assert.AreEqual(4261.48m, old[0].Open); + Assert.AreEqual(3454770.05073206m, old[0].QuoteVolume); + Assert.AreEqual(3427L, old[0].Trades); + Assert.AreEqual(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), recent[0].OpensAt, "microseconds from 2025"); + Assert.AreEqual(new DateTimeOffset(2025, 1, 5, 0, 0, 0, TimeSpan.Zero), recent[^1].OpensAt); + + var zip = Zip("BTCUSDT-1d-2017-08.csv", Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv")); + var checksum = Convert.ToHexStringLower(SHA256.HashData(zip)) + " BTCUSDT-1d-2017-08.zip\n"; + Assert.IsTrue(BinanceArchive.Verify(zip, checksum)); + Assert.IsFalse(BinanceArchive.Verify(zip, Fixture.Text("free/BTCUSDT-1d-2017-08.zip.CHECKSUM")), "the recorded checksum belongs to the real zip"); + CollectionAssert.AreEqual(old.ToList(), BinanceArchive.ReadZip(zip).ToList()); + Assert.AreEqual( + new Uri("https://data.binance.vision/data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2017-08.zip.CHECKSUM"), + BinanceArchive.ChecksumOf(BinanceArchive.Monthly("BTCUSDT", "1d", 2017, 8))); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task ReferenceStoresRoundTripOnTheEngineOfThisRun() + { + var clock = new ManualClock(_start); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + + var fx = new FxStore(test.Db, clock); + await fx.UpsertAsync(Ecb.ParseHistoryCsv(Fixture.Text("free/ecb-hist.csv"), ["USD"]), CancellationToken.None).ConfigureAwait(false); + await fx.UpsertAsync([new FxRate("EUR", "USD", new DateOnly(2026, 11, 9), 1.1194m, Ecb.Source)], CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(1.1194m, (await fx.LatestAsync("EUR", "USD", CancellationToken.None).ConfigureAwait(false))!.Rate, "a new publication replaces the old"); + Assert.AreEqual(1.1165m, (await fx.RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 8), CancellationToken.None).ConfigureAwait(false))!.Rate, + "Sunday uses Friday's rate"); + Assert.IsNull(await fx.RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 5), CancellationToken.None).ConfigureAwait(false)); + + var calendar = new CalendarEventStore(test.Db, clock); + var events = ForexFactory.Parse(Fixture.Text("free/ff-calendar.json")); + Assert.AreEqual(5, await calendar.UpsertAsync(events, CancellationToken.None).ConfigureAwait(false)); + var moved = events.Single(e => e.Title == "CPI m/m") with { Forecast = "0.4%" }; + await calendar.UpsertAsync([moved], CancellationToken.None).ConfigureAwait(false); + var week = await calendar.BetweenAsync(new DateTimeOffset(2026, 11, 10, 0, 0, 0, TimeSpan.Zero), new DateTimeOffset(2026, 11, 17, 0, 0, 0, TimeSpan.Zero), + CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(3, week.Count); + Assert.AreEqual(moved, week[0]); + Assert.IsFalse(week.Single(e => e.Kind == CalendarEventKinds.Holiday).TimeKnown); + + var mood = new MoodStore(test.Db, clock); + await mood.UpsertAsync(FearGreed.Parse(Fixture.Text("free/fng.json")), CancellationToken.None).ConfigureAwait(false); + await mood.UpsertAsync(FearGreed.Parse(Fixture.Text("free/fng.json")), CancellationToken.None).ConfigureAwait(false); + var latest = await mood.LatestAsync(MoodIndicators.FearGreed, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(32d, latest!.Value); + Assert.AreEqual("paura", latest.Label); + Assert.IsNull(await mood.LatestAsync(MoodIndicators.Vix, CancellationToken.None).ConfigureAwait(false)); + + var states = new SourceStateStore(test.Db, clock); + await states.RecordAsync(Finnhub.Source, enabled: false, success: false, null, "manca la chiave FINNHUB_API_KEY", CancellationToken.None).ConfigureAwait(false); + await states.RecordAsync(Ecb.Source, enabled: true, success: true, null, null, CancellationToken.None).ConfigureAwait(false); + clock.Advance(TimeSpan.FromHours(1)); + await states.RecordAsync(Ecb.Source, enabled: true, success: false, "timeout", null, CancellationToken.None).ConfigureAwait(false); + var list = await states.ListAsync(CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(2, list.Count); + var ecb = list.Single(s => s.Source == Ecb.Source); + Assert.AreEqual(_start, ecb.LastSuccessAt, "a failure keeps the last success"); + Assert.AreEqual(_start.AddHours(1), ecb.LastRequestAt); + Assert.AreEqual("timeout", ecb.LastError); + Assert.IsFalse(list.Single(s => s.Source == Finnhub.Source).Enabled); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task StoredMarketCalendarKeepsOpenAndClosedDaysAcrossRestarts() + { + var clock = new ManualClock(_start); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var calendar = new StoredMarketCalendar(test.Db, clock, "us_equity"); + Assert.IsNull(calendar.Coverage); + + var friday = new DateOnly(2026, 11, 13); + var monday = new DateOnly(2026, 11, 16); + MarketSession Session(DateOnly day) => new(day, Zones.At(day, new TimeOnly(9, 30), Zones.NewYork), Zones.At(day, new TimeOnly(16, 0), Zones.NewYork)); + await calendar.SaveAsync([Session(friday), Session(monday)], friday, monday, CancellationToken.None).ConfigureAwait(false); + + var reloaded = new StoredMarketCalendar(test.Db, clock, "us_equity"); + await reloaded.LoadAsync(CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual((friday, monday), reloaded.Coverage); + Assert.AreEqual(Session(friday), reloaded.SessionOn(friday)); + Assert.IsNull(reloaded.SessionOn(friday.AddDays(1)), "Saturday is closed"); + Assert.IsTrue(reloaded.Covers(friday.AddDays(1))); + Assert.IsFalse(reloaded.Covers(monday.AddDays(1)), "a day not downloaded is unknown, not closed"); + Assert.AreEqual(2, reloaded.Sessions(friday, monday).Count); + Assert.AreEqual(new DateTimeOffset(2026, 11, 13, 14, 30, 0, TimeSpan.Zero), reloaded.SessionOn(friday)!.OpenUtc); + + var other = new StoredMarketCalendar(test.Db, clock, "other"); + await other.LoadAsync(CancellationToken.None).ConfigureAwait(false); + Assert.IsNull(other.Coverage, "each market has its own calendar"); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task BinanceImporterChecksEachArchiveAndStopsAtTheFirstMissingMonth() + { + var folder = TempFolder.Create("binance"); + try + { + var august = Zip("BTCUSDT-1d-2017-08.csv", Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv")); + var http = new FakeHttp() + .On("2017-08.zip.CHECKSUM", Convert.ToHexStringLower(SHA256.HashData(august)) + " BTCUSDT-1d-2017-08.zip") + .OnBytes("2017-08.zip", august) + .On(_ => true, _ => FakeHttp.Respond("NoSuchKey", HttpStatusCode.NotFound)); + var clock = new FastClock(new DateTimeOffset(2017, 10, 15, 0, 0, 0, TimeSpan.Zero)); + var store = new ResearchStore(folder); + var importer = new BinanceImporter(new ApiHttp(http.Client(), clock, "binance"), store, clock); + + var result = await importer.ImportAsync("BTCUSDT", "1d", 2017, 7, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(5, result.Bars); + Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), result.First); + Assert.AreEqual(1, result.Notes.Count); + StringAssert.Contains(result.Notes[0], "2017-09"); + Assert.AreEqual(4, http.Requests.Count, "July (not listed yet), August and its checksum, September (not published)"); + var stored = await store.ReadAsync(Venues.Binance, "BTCUSDT", Timeframes.Day, DateTimeOffset.MinValue, DateTimeOffset.MaxValue).ToListAsync().ConfigureAwait(false); + CollectionAssert.AreEqual(BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv")).ToList(), stored); + + // A zip that does not match its fingerprint is refused. + var tampered = new FakeHttp() + .On("CHECKSUM", new string('0', 64) + " BTCUSDT-1d-2017-08.zip") + .OnBytes("2017-08.zip", august); + var refused = new BinanceImporter(new ApiHttp(tampered.Client(), clock, "binance"), new ResearchStore(Path.Combine(folder, "other")), clock); + await Assert.ThrowsExactlyAsync(() => refused.ImportAsync("BTCUSDT", "1d", 2017, 8, CancellationToken.None)).ConfigureAwait(false); + } + finally + { + TempFolder.Delete(folder); + } + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task CsvImporterFollowsTheMappingAndReportsWhatItCouldNotRead() + { + var folder = TempFolder.Create("csvimport"); + try + { + var csv = Path.Combine(folder, "ethusd_2024.csv"); + await File.WriteAllTextAsync(csv, + "Data;Apertura;Massimo;Minimo;Chiusura;Volume\n" + + "02/01/2024 01:00;2.300,50;2.310,00;2.295,25;2.305,75;12,5\n" + + "02/01/2024 01:01;2.305,75;2.306,00;2.301,00;2.302,00;3\n" + + "02/01/2024 01:02;non so;2.306,00;2.301,00;2.302,00;3\n" + + "02/01/2024 01:04;2.302,00;2.303,00;2.300,00;2.301,00;4\n").ConfigureAwait(false); + await File.WriteAllTextAsync(Path.ChangeExtension(csv, ".mapping.json"), """ + { + // Exported from a spreadsheet in Italian. + "symbol": "ETHUSD", + "delimiter": ";", + "time": "Data", + "time_format": "dd/MM/yyyy HH:mm", + "time_zone": "Europe/Rome", + "open": "Apertura", "high": "Massimo", "low": "Minimo", "close": "Chiusura", "volume": "Volume", + "decimal_separator": ",", + } + """).ConfigureAwait(false); + + var mapping = CsvImporter.MappingFor(csv); + Assert.AreEqual("ETHUSD", mapping.Symbol); + Assert.AreEqual(";", mapping.Delimiter); + var store = new ResearchStore(Path.Combine(folder, "research")); + var result = await new CsvImporter(store).ImportAsync(csv, mapping, CancellationToken.None).ConfigureAwait(false); + + Assert.AreEqual(3, result.Bars); + Assert.AreEqual(new DateTimeOffset(2024, 1, 2, 0, 0, 0, TimeSpan.Zero), result.First, "01:00 in Rome is midnight UTC in winter"); + StringAssert.Contains(result.Notes[0], "riga 4"); + Assert.IsTrue(result.Notes.Any(n => n.Contains("1 righe non lette su 5", StringComparison.Ordinal)), string.Join(" | ", result.Notes)); + Assert.IsTrue(result.Notes.Any(n => n.StartsWith(IssueKinds.Gap, StringComparison.Ordinal)), "the missing minutes are reported"); + + var stored = await store.ReadAsync("import", "ETHUSD", Timeframes.Minute, DateTimeOffset.MinValue, DateTimeOffset.MaxValue).ToListAsync().ConfigureAwait(false); + Assert.AreEqual(3, stored.Count); + Assert.AreEqual(2300.50m, stored[0].Open); + Assert.AreEqual(12.5m, stored[0].Volume); + + // Without a mapping file: Binance-like columns by number and the symbol from the file name. + var plain = Path.Combine(folder, "solusdt-1m.csv"); + await File.WriteAllTextAsync(plain, string.Join('\n', Fixture.Text("free/binance-BTCUSDT-1d-2025-01.csv").Split('\n').Take(2))).ConfigureAwait(false); + var defaults = CsvImporter.MappingFor(plain) with { HasHeader = false, Timeframe = Timeframes.Day }; + var second = await new CsvImporter(store).ImportAsync(plain, defaults, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual("SOLUSDT", second.Symbol); + Assert.AreEqual(2, second.Bars); + Assert.AreEqual(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), second.First, "microseconds recognised by size"); + } + finally + { + TempFolder.Delete(folder); + } + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task DownloaderRespectsLimitsRecordsFailuresAndSaysWhatIsOff() + { + var clock = new FastClock(_start); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var settings = new SettingsStore(test.Db, clock, new EventBus()); + var ecbZip = Zip("eurofxref-hist.csv", Fixture.Text("free/ecb-hist.csv")); + var fngFails = true; + var http = new FakeHttp() + .OnBytes("eurofxref-hist.zip", ecbZip) + .On("eurofxref-daily.xml", Fixture.Text("free/ecb-daily.xml")) + .On("ff_calendar_thisweek", Fixture.Text("free/ff-calendar.json")) + .On(u => u.Host == "api.alternative.me", _ => fngFails + ? FakeHttp.Respond("{}", HttpStatusCode.BadRequest) + : FakeHttp.Respond(Fixture.Text("free/fng.json"))) + .On("VIX_History.csv", Fixture.Text("free/vix.csv")); + var sources = new ReferenceSources(new ApiHttp(http.Client(), clock, "reference"), test.Db, clock, settings, (_, _) => Task.FromResult(null)); + + // Exchange rates: the whole history the first time, then the day's file. + Assert.IsTrue(await sources.RefreshFxAsync(CancellationToken.None).ConfigureAwait(false)); + clock.Advance(TimeSpan.FromHours(1)); + Assert.IsTrue(await sources.RefreshFxAsync(CancellationToken.None).ConfigureAwait(false)); + CollectionAssert.AreEqual(new[] { "eurofxref-hist.zip", "eurofxref-daily.xml" }, http.Urls.Select(u => u.Segments[^1]).ToList()); + var rate = await new FxStore(test.Db, clock).RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 9), CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(1.1193m, rate!.Rate); + + // The calendar: Forex Factory at most once an hour, the central banks from the settings. + Assert.IsTrue(await sources.RefreshCalendarAsync(CancellationToken.None).ConfigureAwait(false)); + clock.Advance(TimeSpan.FromMinutes(10)); + Assert.IsTrue(await sources.RefreshCalendarAsync(CancellationToken.None).ConfigureAwait(false)); + Assert.AreEqual(1, http.Urls.Count(u => u.Host == "nfs.faireconomy.media"), "ten minutes later Forex Factory is not asked again"); + var events = await new CalendarEventStore(test.Db, clock).BetweenAsync(_start.AddDays(-7), _start.AddYears(2), CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(5 + 18, events.Count); + + // Without keys FRED and Finnhub are off and the state says why; the earnings filter is off. + Assert.IsFalse(await sources.RefreshFredAsync(CancellationToken.None).ConfigureAwait(false)); + Assert.IsFalse(await sources.RefreshEarningsAsync(null, CancellationToken.None).ConfigureAwait(false)); + Assert.IsFalse(await sources.EarningsFilterActiveAsync(CancellationToken.None).ConfigureAwait(false)); + var states = await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false); + StringAssert.Contains(states.Single(s => s.Source == Finnhub.Source).Note, "filtro sulle trimestrali è disattivato"); + Assert.IsFalse(states.Single(s => s.Source == Fred.Source).Enabled); + Assert.IsFalse(http.Urls.Any(u => u.Host.Contains("finnhub", StringComparison.Ordinal) || u.Host.Contains("stlouisfed", StringComparison.Ordinal))); + + // A failing source is recorded and does not stop the others; it is tried again at its next turn. + Assert.IsFalse(await sources.RefreshMoodAsync(CancellationToken.None).ConfigureAwait(false)); + states = await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false); + Assert.IsNotNull(states.Single(s => s.Source == FearGreed.Source).LastError); + Assert.IsNull(states.Single(s => s.Source == VixHistory.Source).LastError); + Assert.AreEqual("calmo", (await new MoodStore(test.Db, clock).LatestAsync(MoodIndicators.Vix, CancellationToken.None).ConfigureAwait(false))!.Label); + fngFails = false; + clock.Advance(TimeSpan.FromHours(1)); + Assert.IsTrue(await sources.RefreshMoodAsync(CancellationToken.None).ConfigureAwait(false)); + var fng = (await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false)).Single(s => s.Source == FearGreed.Source); + Assert.IsNull(fng.LastError); + Assert.AreEqual(FearGreed.Attribution, fng.Note, "the source is named, as its terms ask"); + StringAssert.Contains(http.Urls.Last(u => u.Host == "api.alternative.me").Query, "limit=400", "the first success asks for the history"); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task CentralBankListIsEditableAndRefusedWhenWrong() + { + var clock = new ManualClock(_start); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var settings = new SettingsStore(test.Db, clock, new EventBus()); + var key = SettingsStore.CentralBankMeetings.Key; + Assert.AreEqual(CentralBanks.DefaultList, (await settings.GetAsync(key).ConfigureAwait(false)).Value); + + var error = await Assert.ThrowsExactlyAsync(() => settings.SetAsync(key, "FED 2027-13-01", "test")).ConfigureAwait(false); + StringAssert.Contains(error.Message, "Riga 1"); + await settings.SetAsync(key, CentralBanks.DefaultList + "\nFED 2028-01-26", "test").ConfigureAwait(false); + Assert.AreEqual(19, CentralBanks.Parse((await settings.GetAsync(key).ConfigureAwait(false)).Value).Count); + } + + internal static byte[] Zip(string entryName, string content) + { + using var zip = new MemoryStream(); + using (var archive = new ZipArchive(zip, ZipArchiveMode.Create, leaveOpen: true)) + using (var stream = archive.CreateEntry(entryName).Open()) + { + stream.Write(Encoding.UTF8.GetBytes(content)); + } + + return zip.ToArray(); + } +}