MarketDataService wires the collector, the reference sources, the universes and the corporate actions into the scheduler. Short jobs (rates, calendar, mood, spread samples, corporate actions) run on their schedule; long downloads (catalogues, the first history, thousands of stocks, the Binance archives) run on a background worker the jobs only wake, so they never delay the backup or each other. The live crypto feed follows the crypto universe, or BTC, ETH and SOL until one exists. Alpaca uses only the paper account keys; without them the stock part is off and the sources list says why. Market data is off in demo mode, with ANUBIS_MARKET_DATA=off, and in tests that bring no fake network. New: GET /api/data/quality (JSON or Markdown) and /api/data/sources, and "anubis data report|binance|csv" on the command line. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
170 lines
6.9 KiB
C#
170 lines
6.9 KiB
C#
using System.Net;
|
|
using System.Text.RegularExpressions;
|
|
using Anubis.Server;
|
|
using Microsoft.AspNetCore.Builder;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
|
|
namespace Anubis.Tests;
|
|
|
|
/// <summary>
|
|
/// The real server on a free local port, with a fake environment and its own data folder: never %APPDATA%, never
|
|
/// the developer's database. A cookie-keeping client that does not follow redirects, so tests see each step.
|
|
/// </summary>
|
|
internal sealed partial class ServerHarness : IAsyncDisposable
|
|
{
|
|
public const string AdminPassword = "fake-initial-pass-1";
|
|
public const string SecretKey = "fake-secret-key-0123456789-abcdefghij";
|
|
|
|
private readonly WebApplication _app;
|
|
|
|
private bool _ownsData;
|
|
|
|
private ServerHarness(WebApplication app, string dataDir, CookieContainer cookies, HttpClient http)
|
|
{
|
|
_app = app;
|
|
DataDir = dataDir;
|
|
Cookies = cookies;
|
|
Http = http;
|
|
}
|
|
|
|
public string DataDir { get; }
|
|
|
|
public CookieContainer Cookies { get; }
|
|
|
|
public HttpClient Http { get; }
|
|
|
|
public Uri BaseAddress => Http.BaseAddress!;
|
|
|
|
internal AnubisHost Host => _app.Services.GetRequiredService<AnubisHost>();
|
|
|
|
/// <summary>Docker configuration with SQLite in the data folder: what most tests need.</summary>
|
|
public static Dictionary<string, string?> DockerSqlite() => new(StringComparer.Ordinal)
|
|
{
|
|
["ANUBIS_DB_ENGINE"] = "sqlite",
|
|
["ANUBIS_SECRET_KEY"] = SecretKey,
|
|
["ANUBIS_ADMIN_PASSWORD"] = AdminPassword,
|
|
["TZ"] = "Europe/Rome",
|
|
};
|
|
|
|
/// <summary>
|
|
/// Docker configuration on the engine of this run (ANUBIS_TEST_DB): SQLite in the data folder, or a new PostgreSQL
|
|
/// database that the caller disposes.
|
|
/// </summary>
|
|
public static async Task<(Dictionary<string, string?> Env, TestDb? Postgres)> DockerOnEngineAsync()
|
|
{
|
|
var env = DockerSqlite();
|
|
if (TestDatabase.Engine != "postgres")
|
|
{
|
|
return (env, null);
|
|
}
|
|
|
|
var db = await TestDb.CreateAsync(Anubis.Platform.DbEngine.Postgres).ConfigureAwait(false);
|
|
UsePostgres(env, db);
|
|
return (env, db);
|
|
}
|
|
|
|
public static void UsePostgres(Dictionary<string, string?> env, TestDb db)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(env);
|
|
ArgumentNullException.ThrowIfNull(db);
|
|
env["ANUBIS_DB_ENGINE"] = "postgres";
|
|
env["ANUBIS_DB_HOST"] = db.Settings.Host;
|
|
env["ANUBIS_DB_PORT"] = db.Settings.Port.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
|
env["ANUBIS_DB_NAME"] = db.Settings.Database;
|
|
env["ANUBIS_DB_USER"] = db.Settings.User;
|
|
env["ANUBIS_DB_PASSWORD"] = db.Settings.Password;
|
|
}
|
|
|
|
public static async Task<ServerHarness> StartAsync(string platform, IReadOnlyDictionary<string, string?> environment, string? dataDir = null,
|
|
string[]? extraArgs = null, bool waitReady = true, HttpMessageHandler? marketHttp = null)
|
|
{
|
|
var owns = dataDir is null;
|
|
dataDir ??= TempFolder.Create("server");
|
|
string[] args = ["--platform", platform, "--data-dir", dataDir, "--urls", "http://127.0.0.1:0", .. extraArgs ?? []];
|
|
var app = ServerApp.Build(args, new ServerOverrides
|
|
{
|
|
Environment = environment,
|
|
PasswordIterations = 1000,
|
|
DatabaseCheckInterval = TimeSpan.FromMilliseconds(100),
|
|
MarketData = marketHttp is not null,
|
|
MarketHttp = marketHttp,
|
|
MarketSockets = marketHttp is null ? null : () => new FakeSocket(),
|
|
});
|
|
await app.StartAsync().ConfigureAwait(false);
|
|
var cookies = new CookieContainer();
|
|
#pragma warning disable CA2000 // The handler belongs to the client, disposed with the harness.
|
|
var http = new HttpClient(new HttpClientHandler { CookieContainer = cookies, AllowAutoRedirect = false })
|
|
#pragma warning restore CA2000
|
|
{
|
|
BaseAddress = new Uri(app.Urls.Single()),
|
|
Timeout = TimeSpan.FromSeconds(60),
|
|
};
|
|
var harness = new ServerHarness(app, dataDir, cookies, http) { _ownsData = owns };
|
|
if (waitReady)
|
|
{
|
|
await harness.Host.WaitForAsync(HostMode.Ready, TimeSpan.FromSeconds(30)).ConfigureAwait(false);
|
|
}
|
|
|
|
return harness;
|
|
}
|
|
|
|
public async Task WaitForAsync(HostMode mode) => await Host.WaitForAsync(mode, TimeSpan.FromSeconds(30)).ConfigureAwait(false);
|
|
|
|
public Task<HttpResponseMessage> GetAsync(string path) => Http.GetAsync(new Uri(path, UriKind.Relative));
|
|
|
|
public async Task<string> TextAsync(string path)
|
|
{
|
|
using var response = await GetAsync(path).ConfigureAwait(false);
|
|
return WebUtility.HtmlDecode(await response.Content.ReadAsStringAsync().ConfigureAwait(false));
|
|
}
|
|
|
|
/// <summary>The CSRF field of a page's forms.</summary>
|
|
public async Task<string> CsrfAsync(string page)
|
|
{
|
|
using var response = await GetAsync(page).ConfigureAwait(false);
|
|
var html = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
|
var match = CsrfField().Match(html);
|
|
Assert.IsTrue(match.Success, "No CSRF field on " + page);
|
|
return WebUtility.HtmlDecode(match.Groups[1].Value);
|
|
}
|
|
|
|
/// <summary>Posts a form the way the browser does, with the CSRF field of <paramref name="page"/>.</summary>
|
|
public async Task<HttpResponseMessage> PostFormAsync(string page, string action, params (string Name, string Value)[] fields)
|
|
{
|
|
var token = await CsrfAsync(page).ConfigureAwait(false);
|
|
using var content = new FormUrlEncodedContent([new("__csrf", token), .. fields.Select(f => new KeyValuePair<string, string>(f.Name, f.Value))]);
|
|
return await Http.PostAsync(new Uri(action, UriKind.Relative), content).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>Signs in with the initial password and replaces it, as the first access in Docker requires.</summary>
|
|
public async Task<string> SignInFirstTimeAsync()
|
|
{
|
|
using (var login = await PostFormAsync("/accesso", "/azioni/accesso", ("password", AdminPassword)).ConfigureAwait(false))
|
|
{
|
|
Assert.AreEqual(HttpStatusCode.Redirect, login.StatusCode);
|
|
}
|
|
|
|
const string chosen = "chosen-test-pass-2";
|
|
using var change = await PostFormAsync("/primo-accesso", "/azioni/primo-accesso", ("attuale", AdminPassword), ("nuova", chosen), ("conferma", chosen))
|
|
.ConfigureAwait(false);
|
|
Assert.AreEqual("/?ok=password", change.Headers.Location?.OriginalString);
|
|
return chosen;
|
|
}
|
|
|
|
public static string Location(HttpResponseMessage response) => response.Headers.Location?.OriginalString ?? "";
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
Http.Dispose();
|
|
await _app.StopAsync().ConfigureAwait(false);
|
|
await _app.DisposeAsync().ConfigureAwait(false);
|
|
if (_ownsData)
|
|
{
|
|
TempFolder.Delete(DataDir);
|
|
}
|
|
}
|
|
|
|
[GeneratedRegex("name=\"__csrf\" value=\"([^\"]+)\"")]
|
|
private static partial Regex CsrfField();
|
|
}
|