Files
Anubis/tests/Anubis.Tests/ReferenceSourcesTests.cs
T
Alby96andClaude Opus 5.5 b3fb69e696 Add free reference sources and research importers (refs #21 #22)
Readers for the ECB rates, the Forex Factory calendar, FRED release dates,
Finnhub earnings, the Fed/ECB meeting list, fear and greed, the Cboe VIX
history, nasdaqtraded.txt and the Binance kline archives, each tested on
recorded responses. ReferenceSources downloads them within their limits
and records every attempt in source_state, so limits hold across restarts
and the interface can say which source is off and why (FRED and Finnhub
without a key). The central-bank list is an editable, validated setting.

BinanceImporter checks every monthly zip against its SHA-256 file before
writing it to the research history; CsvImporter reads user files with a
configurable column mapping and reports unreadable lines and gaps instead
of fixing them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 15:20:35 +01:00

432 lines
26 KiB
C#

using System.IO.Compression;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using Anubis.Brokers;
using Anubis.Core;
using Anubis.Data;
namespace Anubis.Tests;
/// <summary>Readers of the free sources on recorded responses, their stores, and the research importers (issues #21, #22, #24).</summary>
[TestClass]
public sealed class ReferenceSourcesTests
{
private static readonly DateTimeOffset _start = new(2026, 11, 10, 8, 0, 0, TimeSpan.Zero);
[TestMethod]
[TestCategory(Category.Unit)]
public void EcbDailyNinetyDayAndHistoryFilesGiveTheSameRates()
{
string[] wanted = ["USD", "GBP", "BGN"];
var daily = Ecb.ParseXml(Fixture.Text("free/ecb-daily.xml"), wanted);
Assert.AreEqual(2, daily.Count, "BGN is no longer published");
Assert.AreEqual(new FxRate("EUR", "USD", new DateOnly(2026, 11, 9), 1.1193m, Ecb.Source), daily.Single(r => r.Quote == "USD"));
Assert.AreEqual(0.83430m, daily.Single(r => r.Quote == "GBP").Rate);
var ninety = Ecb.ParseXml(Fixture.Text("free/ecb-hist-90d.xml"), ["USD"]);
Assert.IsTrue(ninety.Count > 1);
Assert.AreEqual(1.1193m, ninety.Single(r => r.Date == new DateOnly(2026, 11, 9)).Rate, "the 90-day file uses double quotes");
CollectionAssert.AreEqual(ninety.OrderBy(r => r.Date).ToList(), ninety.ToList(), "oldest first");
var history = Ecb.ParseHistoryCsv(Fixture.Text("free/ecb-hist.csv"), wanted);
Assert.AreEqual(4, history.Count, "N/A is skipped and the trailing comma ignored");
Assert.AreEqual(new DateOnly(2026, 11, 6), history[0].Date);
Assert.AreEqual(1.1165m, history.Single(r => r.Quote == "USD" && r.Date == new DateOnly(2026, 11, 6)).Rate);
using var zip = new MemoryStream();
using (var archive = new ZipArchive(zip, ZipArchiveMode.Create, leaveOpen: true))
using (var writer = new StreamWriter(archive.CreateEntry("eurofxref-hist.csv").Open()))
{
writer.Write(Fixture.Text("free/ecb-hist.csv"));
}
Assert.AreEqual(Fixture.Text("free/ecb-hist.csv"), Ecb.ExtractHistoryCsv(zip.ToArray()));
}
[TestMethod]
[TestCategory(Category.Unit)]
public void ForexFactoryEventsKeepTheirTimeAndHolidaysOnlyTheDay()
{
var events = ForexFactory.Parse(Fixture.Text("free/ff-calendar.json"));
Assert.AreEqual(5, events.Count);
var cpi = events.Single(e => e.Title == "CPI m/m");
Assert.AreEqual(new DateTimeOffset(2026, 11, 10, 13, 30, 0, TimeSpan.Zero), cpi.OccursAt, "the offset -05:00 becomes UTC");
Assert.AreEqual("USD", cpi.Currency);
Assert.AreEqual("High", cpi.Impact);
Assert.AreEqual("0.3%", cpi.Forecast);
Assert.AreEqual("0.6%", cpi.Previous);
Assert.IsTrue(cpi.TimeKnown);
var holiday = events.Single(e => e.Title == "Bank Holiday");
Assert.AreEqual(CalendarEventKinds.Holiday, holiday.Kind);
Assert.IsFalse(holiday.TimeKnown);
Assert.IsNull(holiday.Impact);
Assert.IsNull(events.Single(e => e.Title == "RBA Assist Gov Hunter Speaks").Forecast, "an empty forecast is no forecast");
Assert.AreEqual(events.Count, events.Select(e => e.SourceKey).Distinct().Count());
}
[TestMethod]
[TestCategory(Category.Unit)]
public void FredFinnhubAndCentralBankDatesBecomeTimedEvents()
{
var fred = Fred.Parse(Fixture.Text("free/fred-release-dates.json"), "PIL USA", new DateOnly(2010, 1, 1));
Assert.AreEqual(4, fred.Count, "dates before the start are skipped");
Assert.AreEqual(new DateTimeOffset(2010, 2, 11, 13, 30, 0, TimeSpan.Zero), fred[0].OccursAt, "08:30 New York in winter");
Assert.AreEqual("fred:82:2010-02-11", fred[0].SourceKey);
Assert.AreEqual(new DateTimeOffset(2012, 10, 24, 12, 30, 0, TimeSpan.Zero), fred[2].OccursAt, "08:30 New York in summer time");
StringAssert.Contains(Fred.ReleaseDates(10, "k", new DateOnly(2026, 11, 10)).Query, "include_release_dates_with_no_data=true");
var earnings = Finnhub.Parse(Fixture.Text("free/finnhub-earnings.json"));
Assert.AreEqual(2, earnings.Count);
var aapl = earnings.Single(e => e.OccursAt.Year == 2020);
Assert.AreEqual(new DateTimeOffset(2020, 1, 28, 21, 5, 0, TimeSpan.Zero), aapl.OccursAt, "amc: 16:05 New York");
Assert.AreEqual("amc", aapl.TimeHint);
Assert.AreEqual("AAPL", aapl.Symbol);
Assert.AreEqual(0, Finnhub.Parse(Fixture.Text("free/finnhub-earnings.json"), new HashSet<string> { "MSFT" }).Count);
var banks = CentralBanks.Parse(CentralBanks.DefaultList);
Assert.AreEqual(18, banks.Count);
var fed = banks.Single(e => e.SourceKey == "cb:USD:2026-12-09");
Assert.AreEqual(new DateTimeOffset(2026, 12, 9, 19, 0, 0, TimeSpan.Zero), fed.OccursAt, "14:00 New York");
var ecb = banks.Single(e => e.SourceKey == "cb:EUR:2027-07-22");
Assert.AreEqual(new DateTimeOffset(2027, 7, 22, 12, 15, 0, TimeSpan.Zero), ecb.OccursAt, "14:15 Frankfurt in summer time");
// Every decision day of the default list is in the official calendars read for the fixture.
var official = Fixture.Text("free/central-banks.md");
foreach (var e in banks)
{
StringAssert.Contains(official, $"| {e.OccursAt:yyyy-MM-dd} |");
}
Assert.AreEqual(1, CentralBanks.Parse("# commento\n\nfed 2027-01-27\n").Count);
var error = Assert.ThrowsExactly<FormatException>(() => CentralBanks.Parse("FED 2027-01-27\nBOJ 2027-01-28"));
StringAssert.Contains(error.Message, "Riga 2");
Assert.ThrowsExactly<FormatException>(() => CentralBanks.Parse("FED 27/01/2027"));
}
[TestMethod]
[TestCategory(Category.Unit)]
public void MoodReadingsCarryTheLabelsOfTheSpec()
{
var fng = FearGreed.Parse(Fixture.Text("free/fng.json"));
Assert.AreEqual(2, fng.Count);
Assert.AreEqual(32d, fng[^1].Value, "oldest first: the latest reading is last");
Assert.AreEqual("paura", fng[^1].Label);
Assert.AreEqual(DateTimeOffset.FromUnixTimeSeconds(1794268800), fng[^1].AsOf);
var vix = VixHistory.Parse(Fixture.Text("free/vix.csv"), new DateOnly(2026, 1, 1));
Assert.AreEqual(2, vix.Count);
Assert.AreEqual(new DateTimeOffset(2026, 11, 9, 0, 0, 0, TimeSpan.Zero), vix[1].AsOf);
Assert.AreEqual(("nervoso", "calmo"), (vix[0].Label, vix[1].Label));
Assert.AreEqual("paura estrema", MoodLabels.FearGreed(24));
Assert.AreEqual("neutro", MoodLabels.FearGreed(55));
Assert.AreEqual("avidità estrema", MoodLabels.FearGreed(76));
Assert.AreEqual("panico", MoodLabels.Vix(30.01));
Assert.AreEqual("troppi al rialzo", MoodLabels.Funding(0.0003, TimeSpan.FromHours(8)));
Assert.AreEqual("troppi al rialzo", MoodLabels.Funding(0.00015, TimeSpan.FromHours(4)), "4-hour funding counts double");
Assert.AreEqual("nella norma", MoodLabels.Funding(-0.0001, TimeSpan.FromHours(8)));
Assert.AreEqual("troppi al ribasso", MoodLabels.Funding(-0.00021, TimeSpan.FromHours(8)));
}
[TestMethod]
[TestCategory(Category.Unit)]
public void NasdaqListExcludesFundsAndEverythingThatIsNotACommonShare()
{
var list = NasdaqTraded.Parse(Fixture.Text("free/nasdaqtraded.txt"));
Assert.AreEqual(12, list.Count, "header and footer skipped");
var excluded = list.Where(s => s.IsExcluded).Select(s => s.Symbol).ToList();
CollectionAssert.AreEquivalent(new[] { "SPY", "BULZ", "AMUB", "PDI", "ABR$D", "AACIW", "ZXIET" }, excluded);
Assert.AreEqual("TXSE", list.Single(s => s.Symbol == "CECO").Exchange);
Assert.AreEqual("NASDAQ", list.Single(s => s.Symbol == "AAPL").Exchange);
Assert.AreEqual("NYSE", list.Single(s => s.Symbol == "KO").Exchange);
Assert.AreEqual("D", list.Single(s => s.Symbol == "AACG").FinancialStatus);
}
[TestMethod]
[TestCategory(Category.Unit)]
public void BinanceArchivesReadMillisecondsAndMicrosecondsAndCheckTheirFingerprint()
{
var old = BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv"));
var recent = BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2025-01.csv"));
Assert.AreEqual(5, old.Count);
Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), old[0].OpensAt);
Assert.AreEqual(4261.48m, old[0].Open);
Assert.AreEqual(3454770.05073206m, old[0].QuoteVolume);
Assert.AreEqual(3427L, old[0].Trades);
Assert.AreEqual(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), recent[0].OpensAt, "microseconds from 2025");
Assert.AreEqual(new DateTimeOffset(2025, 1, 5, 0, 0, 0, TimeSpan.Zero), recent[^1].OpensAt);
var zip = Zip("BTCUSDT-1d-2017-08.csv", Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv"));
var checksum = Convert.ToHexStringLower(SHA256.HashData(zip)) + " BTCUSDT-1d-2017-08.zip\n";
Assert.IsTrue(BinanceArchive.Verify(zip, checksum));
Assert.IsFalse(BinanceArchive.Verify(zip, Fixture.Text("free/BTCUSDT-1d-2017-08.zip.CHECKSUM")), "the recorded checksum belongs to the real zip");
CollectionAssert.AreEqual(old.ToList(), BinanceArchive.ReadZip(zip).ToList());
Assert.AreEqual(
new Uri("https://data.binance.vision/data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2017-08.zip.CHECKSUM"),
BinanceArchive.ChecksumOf(BinanceArchive.Monthly("BTCUSDT", "1d", 2017, 8)));
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task ReferenceStoresRoundTripOnTheEngineOfThisRun()
{
var clock = new ManualClock(_start);
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var fx = new FxStore(test.Db, clock);
await fx.UpsertAsync(Ecb.ParseHistoryCsv(Fixture.Text("free/ecb-hist.csv"), ["USD"]), CancellationToken.None).ConfigureAwait(false);
await fx.UpsertAsync([new FxRate("EUR", "USD", new DateOnly(2026, 11, 9), 1.1194m, Ecb.Source)], CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(1.1194m, (await fx.LatestAsync("EUR", "USD", CancellationToken.None).ConfigureAwait(false))!.Rate, "a new publication replaces the old");
Assert.AreEqual(1.1165m, (await fx.RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 8), CancellationToken.None).ConfigureAwait(false))!.Rate,
"Sunday uses Friday's rate");
Assert.IsNull(await fx.RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 5), CancellationToken.None).ConfigureAwait(false));
var calendar = new CalendarEventStore(test.Db, clock);
var events = ForexFactory.Parse(Fixture.Text("free/ff-calendar.json"));
Assert.AreEqual(5, await calendar.UpsertAsync(events, CancellationToken.None).ConfigureAwait(false));
var moved = events.Single(e => e.Title == "CPI m/m") with { Forecast = "0.4%" };
await calendar.UpsertAsync([moved], CancellationToken.None).ConfigureAwait(false);
var week = await calendar.BetweenAsync(new DateTimeOffset(2026, 11, 10, 0, 0, 0, TimeSpan.Zero), new DateTimeOffset(2026, 11, 17, 0, 0, 0, TimeSpan.Zero),
CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(3, week.Count);
Assert.AreEqual(moved, week[0]);
Assert.IsFalse(week.Single(e => e.Kind == CalendarEventKinds.Holiday).TimeKnown);
var mood = new MoodStore(test.Db, clock);
await mood.UpsertAsync(FearGreed.Parse(Fixture.Text("free/fng.json")), CancellationToken.None).ConfigureAwait(false);
await mood.UpsertAsync(FearGreed.Parse(Fixture.Text("free/fng.json")), CancellationToken.None).ConfigureAwait(false);
var latest = await mood.LatestAsync(MoodIndicators.FearGreed, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(32d, latest!.Value);
Assert.AreEqual("paura", latest.Label);
Assert.IsNull(await mood.LatestAsync(MoodIndicators.Vix, CancellationToken.None).ConfigureAwait(false));
var states = new SourceStateStore(test.Db, clock);
await states.RecordAsync(Finnhub.Source, enabled: false, success: false, null, "manca la chiave FINNHUB_API_KEY", CancellationToken.None).ConfigureAwait(false);
await states.RecordAsync(Ecb.Source, enabled: true, success: true, null, null, CancellationToken.None).ConfigureAwait(false);
clock.Advance(TimeSpan.FromHours(1));
await states.RecordAsync(Ecb.Source, enabled: true, success: false, "timeout", null, CancellationToken.None).ConfigureAwait(false);
var list = await states.ListAsync(CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(2, list.Count);
var ecb = list.Single(s => s.Source == Ecb.Source);
Assert.AreEqual(_start, ecb.LastSuccessAt, "a failure keeps the last success");
Assert.AreEqual(_start.AddHours(1), ecb.LastRequestAt);
Assert.AreEqual("timeout", ecb.LastError);
Assert.IsFalse(list.Single(s => s.Source == Finnhub.Source).Enabled);
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task StoredMarketCalendarKeepsOpenAndClosedDaysAcrossRestarts()
{
var clock = new ManualClock(_start);
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var calendar = new StoredMarketCalendar(test.Db, clock, "us_equity");
Assert.IsNull(calendar.Coverage);
var friday = new DateOnly(2026, 11, 13);
var monday = new DateOnly(2026, 11, 16);
MarketSession Session(DateOnly day) => new(day, Zones.At(day, new TimeOnly(9, 30), Zones.NewYork), Zones.At(day, new TimeOnly(16, 0), Zones.NewYork));
await calendar.SaveAsync([Session(friday), Session(monday)], friday, monday, CancellationToken.None).ConfigureAwait(false);
var reloaded = new StoredMarketCalendar(test.Db, clock, "us_equity");
await reloaded.LoadAsync(CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual((friday, monday), reloaded.Coverage);
Assert.AreEqual(Session(friday), reloaded.SessionOn(friday));
Assert.IsNull(reloaded.SessionOn(friday.AddDays(1)), "Saturday is closed");
Assert.IsTrue(reloaded.Covers(friday.AddDays(1)));
Assert.IsFalse(reloaded.Covers(monday.AddDays(1)), "a day not downloaded is unknown, not closed");
Assert.AreEqual(2, reloaded.Sessions(friday, monday).Count);
Assert.AreEqual(new DateTimeOffset(2026, 11, 13, 14, 30, 0, TimeSpan.Zero), reloaded.SessionOn(friday)!.OpenUtc);
var other = new StoredMarketCalendar(test.Db, clock, "other");
await other.LoadAsync(CancellationToken.None).ConfigureAwait(false);
Assert.IsNull(other.Coverage, "each market has its own calendar");
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task BinanceImporterChecksEachArchiveAndStopsAtTheFirstMissingMonth()
{
var folder = TempFolder.Create("binance");
try
{
var august = Zip("BTCUSDT-1d-2017-08.csv", Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv"));
var http = new FakeHttp()
.On("2017-08.zip.CHECKSUM", Convert.ToHexStringLower(SHA256.HashData(august)) + " BTCUSDT-1d-2017-08.zip")
.OnBytes("2017-08.zip", august)
.On(_ => true, _ => FakeHttp.Respond("<Error><Code>NoSuchKey</Code></Error>", HttpStatusCode.NotFound));
var clock = new FastClock(new DateTimeOffset(2017, 10, 15, 0, 0, 0, TimeSpan.Zero));
var store = new ResearchStore(folder);
var importer = new BinanceImporter(new ApiHttp(http.Client(), clock, "binance"), store, clock);
var result = await importer.ImportAsync("BTCUSDT", "1d", 2017, 7, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(5, result.Bars);
Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), result.First);
Assert.AreEqual(1, result.Notes.Count);
StringAssert.Contains(result.Notes[0], "2017-09");
Assert.AreEqual(4, http.Requests.Count, "July (not listed yet), August and its checksum, September (not published)");
var stored = await store.ReadAsync(Venues.Binance, "BTCUSDT", Timeframes.Day, DateTimeOffset.MinValue, DateTimeOffset.MaxValue).ToListAsync().ConfigureAwait(false);
CollectionAssert.AreEqual(BinanceArchive.ParseCsv(Fixture.Text("free/binance-BTCUSDT-1d-2017-08.csv")).ToList(), stored);
// A zip that does not match its fingerprint is refused.
var tampered = new FakeHttp()
.On("CHECKSUM", new string('0', 64) + " BTCUSDT-1d-2017-08.zip")
.OnBytes("2017-08.zip", august);
var refused = new BinanceImporter(new ApiHttp(tampered.Client(), clock, "binance"), new ResearchStore(Path.Combine(folder, "other")), clock);
await Assert.ThrowsExactlyAsync<MarketDataException>(() => refused.ImportAsync("BTCUSDT", "1d", 2017, 8, CancellationToken.None)).ConfigureAwait(false);
}
finally
{
TempFolder.Delete(folder);
}
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task CsvImporterFollowsTheMappingAndReportsWhatItCouldNotRead()
{
var folder = TempFolder.Create("csvimport");
try
{
var csv = Path.Combine(folder, "ethusd_2024.csv");
await File.WriteAllTextAsync(csv,
"Data;Apertura;Massimo;Minimo;Chiusura;Volume\n" +
"02/01/2024 01:00;2.300,50;2.310,00;2.295,25;2.305,75;12,5\n" +
"02/01/2024 01:01;2.305,75;2.306,00;2.301,00;2.302,00;3\n" +
"02/01/2024 01:02;non so;2.306,00;2.301,00;2.302,00;3\n" +
"02/01/2024 01:04;2.302,00;2.303,00;2.300,00;2.301,00;4\n").ConfigureAwait(false);
await File.WriteAllTextAsync(Path.ChangeExtension(csv, ".mapping.json"), """
{
// Exported from a spreadsheet in Italian.
"symbol": "ETHUSD",
"delimiter": ";",
"time": "Data",
"time_format": "dd/MM/yyyy HH:mm",
"time_zone": "Europe/Rome",
"open": "Apertura", "high": "Massimo", "low": "Minimo", "close": "Chiusura", "volume": "Volume",
"decimal_separator": ",",
}
""").ConfigureAwait(false);
var mapping = CsvImporter.MappingFor(csv);
Assert.AreEqual("ETHUSD", mapping.Symbol);
Assert.AreEqual(";", mapping.Delimiter);
var store = new ResearchStore(Path.Combine(folder, "research"));
var result = await new CsvImporter(store).ImportAsync(csv, mapping, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(3, result.Bars);
Assert.AreEqual(new DateTimeOffset(2024, 1, 2, 0, 0, 0, TimeSpan.Zero), result.First, "01:00 in Rome is midnight UTC in winter");
StringAssert.Contains(result.Notes[0], "riga 4");
Assert.IsTrue(result.Notes.Any(n => n.Contains("1 righe non lette su 5", StringComparison.Ordinal)), string.Join(" | ", result.Notes));
Assert.IsTrue(result.Notes.Any(n => n.StartsWith(IssueKinds.Gap, StringComparison.Ordinal)), "the missing minutes are reported");
var stored = await store.ReadAsync("import", "ETHUSD", Timeframes.Minute, DateTimeOffset.MinValue, DateTimeOffset.MaxValue).ToListAsync().ConfigureAwait(false);
Assert.AreEqual(3, stored.Count);
Assert.AreEqual(2300.50m, stored[0].Open);
Assert.AreEqual(12.5m, stored[0].Volume);
// Without a mapping file: Binance-like columns by number and the symbol from the file name.
var plain = Path.Combine(folder, "solusdt-1m.csv");
await File.WriteAllTextAsync(plain, string.Join('\n', Fixture.Text("free/binance-BTCUSDT-1d-2025-01.csv").Split('\n').Take(2))).ConfigureAwait(false);
var defaults = CsvImporter.MappingFor(plain) with { HasHeader = false, Timeframe = Timeframes.Day };
var second = await new CsvImporter(store).ImportAsync(plain, defaults, CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual("SOLUSDT", second.Symbol);
Assert.AreEqual(2, second.Bars);
Assert.AreEqual(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), second.First, "microseconds recognised by size");
}
finally
{
TempFolder.Delete(folder);
}
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task DownloaderRespectsLimitsRecordsFailuresAndSaysWhatIsOff()
{
var clock = new FastClock(_start);
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var settings = new SettingsStore(test.Db, clock, new EventBus());
var ecbZip = Zip("eurofxref-hist.csv", Fixture.Text("free/ecb-hist.csv"));
var fngFails = true;
var http = new FakeHttp()
.OnBytes("eurofxref-hist.zip", ecbZip)
.On("eurofxref-daily.xml", Fixture.Text("free/ecb-daily.xml"))
.On("ff_calendar_thisweek", Fixture.Text("free/ff-calendar.json"))
.On(u => u.Host == "api.alternative.me", _ => fngFails
? FakeHttp.Respond("{}", HttpStatusCode.BadRequest)
: FakeHttp.Respond(Fixture.Text("free/fng.json")))
.On("VIX_History.csv", Fixture.Text("free/vix.csv"));
var sources = new ReferenceSources(new ApiHttp(http.Client(), clock, "reference"), test.Db, clock, settings, (_, _) => Task.FromResult<string?>(null));
// Exchange rates: the whole history the first time, then the day's file.
Assert.IsTrue(await sources.RefreshFxAsync(CancellationToken.None).ConfigureAwait(false));
clock.Advance(TimeSpan.FromHours(1));
Assert.IsTrue(await sources.RefreshFxAsync(CancellationToken.None).ConfigureAwait(false));
CollectionAssert.AreEqual(new[] { "eurofxref-hist.zip", "eurofxref-daily.xml" }, http.Urls.Select(u => u.Segments[^1]).ToList());
var rate = await new FxStore(test.Db, clock).RateOnAsync("EUR", "USD", new DateOnly(2026, 11, 9), CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(1.1193m, rate!.Rate);
// The calendar: Forex Factory at most once an hour, the central banks from the settings.
Assert.IsTrue(await sources.RefreshCalendarAsync(CancellationToken.None).ConfigureAwait(false));
clock.Advance(TimeSpan.FromMinutes(10));
Assert.IsTrue(await sources.RefreshCalendarAsync(CancellationToken.None).ConfigureAwait(false));
Assert.AreEqual(1, http.Urls.Count(u => u.Host == "nfs.faireconomy.media"), "ten minutes later Forex Factory is not asked again");
var events = await new CalendarEventStore(test.Db, clock).BetweenAsync(_start.AddDays(-7), _start.AddYears(2), CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual(5 + 18, events.Count);
// Without keys FRED and Finnhub are off and the state says why; the earnings filter is off.
Assert.IsFalse(await sources.RefreshFredAsync(CancellationToken.None).ConfigureAwait(false));
Assert.IsFalse(await sources.RefreshEarningsAsync(null, CancellationToken.None).ConfigureAwait(false));
Assert.IsFalse(await sources.EarningsFilterActiveAsync(CancellationToken.None).ConfigureAwait(false));
var states = await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false);
StringAssert.Contains(states.Single(s => s.Source == Finnhub.Source).Note, "filtro sulle trimestrali è disattivato");
Assert.IsFalse(states.Single(s => s.Source == Fred.Source).Enabled);
Assert.IsFalse(http.Urls.Any(u => u.Host.Contains("finnhub", StringComparison.Ordinal) || u.Host.Contains("stlouisfed", StringComparison.Ordinal)));
// A failing source is recorded and does not stop the others; it is tried again at its next turn.
Assert.IsFalse(await sources.RefreshMoodAsync(CancellationToken.None).ConfigureAwait(false));
states = await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false);
Assert.IsNotNull(states.Single(s => s.Source == FearGreed.Source).LastError);
Assert.IsNull(states.Single(s => s.Source == VixHistory.Source).LastError);
Assert.AreEqual("calmo", (await new MoodStore(test.Db, clock).LatestAsync(MoodIndicators.Vix, CancellationToken.None).ConfigureAwait(false))!.Label);
fngFails = false;
clock.Advance(TimeSpan.FromHours(1));
Assert.IsTrue(await sources.RefreshMoodAsync(CancellationToken.None).ConfigureAwait(false));
var fng = (await sources.StatesAsync(CancellationToken.None).ConfigureAwait(false)).Single(s => s.Source == FearGreed.Source);
Assert.IsNull(fng.LastError);
Assert.AreEqual(FearGreed.Attribution, fng.Note, "the source is named, as its terms ask");
StringAssert.Contains(http.Urls.Last(u => u.Host == "api.alternative.me").Query, "limit=400", "the first success asks for the history");
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task CentralBankListIsEditableAndRefusedWhenWrong()
{
var clock = new ManualClock(_start);
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var settings = new SettingsStore(test.Db, clock, new EventBus());
var key = SettingsStore.CentralBankMeetings.Key;
Assert.AreEqual(CentralBanks.DefaultList, (await settings.GetAsync(key).ConfigureAwait(false)).Value);
var error = await Assert.ThrowsExactlyAsync<ArgumentException>(() => settings.SetAsync(key, "FED 2027-13-01", "test")).ConfigureAwait(false);
StringAssert.Contains(error.Message, "Riga 1");
await settings.SetAsync(key, CentralBanks.DefaultList + "\nFED 2028-01-26", "test").ConfigureAwait(false);
Assert.AreEqual(19, CentralBanks.Parse((await settings.GetAsync(key).ConfigureAwait(false)).Value).Count);
}
internal static byte[] Zip(string entryName, string content)
{
using var zip = new MemoryStream();
using (var archive = new ZipArchive(zip, ZipArchiveMode.Create, leaveOpen: true))
using (var stream = archive.CreateEntry(entryName).Open())
{
stream.Write(Encoding.UTF8.GetBytes(content));
}
return zip.ToArray();
}
}