From d00dfbd32f1bf621fc33e3700e52ec88351cae7a Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Wed, 11 Nov 2026 19:36:32 +0100 Subject: [PATCH] Add the guard that watches the engine and protects positions (refs #48) The guard (SPEC 9.3, 9.4) is now the container's main process and the parent of the engine. It listens on a named pipe for the engine's heartbeat and startup steps, and steps in after two minutes of silence, ten minutes of a startup that does not move, or an exit without notice. It stops the engine, cancels our purchases and sales left open on demo accounts, puts back the stops the positions need from the state file the engine writes every ten seconds, writes a note, warns the user and restarts the engine without any injected fault. The third forced restart within an hour writes the STOP file. The container's stop is passed on to the engine, which closes in order. At restart the engine adopts the guard's stops before reconciling, cancels the orders it left open and plans again the exits already decided, so the rules of SPEC 5.6 apply to them. The acceptance test blocks the main loop right after a sale leaves: in process on a demo-like broker, and with real processes in the drill, which also runs inside the Docker image (tasks ci and the CI workflow). The guard uses the ASP.NET shared runtime so both programs share /app. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/ci.yml | 3 + deploy/docker/Dockerfile | 10 +- deploy/docker/entrypoint.sh | 5 +- docker-compose.yml | 2 + src/Anubis.Guard/Anubis.Guard.csproj | 2 + src/Anubis.Guard/Program.cs | 52 ++- src/Anubis.Server/AnubisHost.cs | 37 +- src/Anubis.Server/ServerApp.cs | 14 +- src/Anubis/Data/MarketDataService.cs | 4 +- src/Anubis/Notifications/TelegramClient.cs | 117 +++++ src/Anubis/Supervision/ChildEngine.cs | 158 +++++++ src/Anubis/Supervision/GuardBrokers.cs | 53 +++ src/Anubis/Supervision/GuardDrill.cs | 419 +++++++++++++++++ src/Anubis/Supervision/GuardLink.cs | 170 +++++++ src/Anubis/Supervision/GuardProtocol.cs | 120 +++++ src/Anubis/Supervision/GuardRepair.cs | 130 ++++++ src/Anubis/Supervision/GuardState.cs | 116 +++++ src/Anubis/Supervision/Guardian.cs | 499 +++++++++++++++++++++ src/Anubis/Trading/DiaryText.cs | 5 + src/Anubis/Trading/LiveEngine.cs | 52 +++ src/Anubis/Trading/OrderManager.Guard.cs | 84 ++++ src/Anubis/Trading/OrderManager.cs | 15 +- src/Anubis/Trading/ReasonCatalog.cs | 1 + src/Anubis/Trading/TradingService.cs | 135 +++++- tests/Anubis.Tests/GuardTests.cs | 414 +++++++++++++++++ tools/Anubis.Tasks/Program.cs | 9 +- 26 files changed, 2607 insertions(+), 19 deletions(-) create mode 100644 src/Anubis/Notifications/TelegramClient.cs create mode 100644 src/Anubis/Supervision/ChildEngine.cs create mode 100644 src/Anubis/Supervision/GuardBrokers.cs create mode 100644 src/Anubis/Supervision/GuardDrill.cs create mode 100644 src/Anubis/Supervision/GuardLink.cs create mode 100644 src/Anubis/Supervision/GuardProtocol.cs create mode 100644 src/Anubis/Supervision/GuardRepair.cs create mode 100644 src/Anubis/Supervision/GuardState.cs create mode 100644 src/Anubis/Supervision/Guardian.cs create mode 100644 src/Anubis/Trading/OrderManager.Guard.cs create mode 100644 tests/Anubis.Tests/GuardTests.cs diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 96bc28f..5e424b3 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -63,3 +63,6 @@ jobs: git checkout -q FETCH_HEAD - name: Build image (no publish) run: docker build -f deploy/docker/Dockerfile -t anubis:ci . + # The guard drill inside the image (issue #48): offline, its own SQLite database in a temporary folder. + - name: Guard drill in the image + run: docker run --rm anubis:ci drill diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index f1bc33a..4dcff4e 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -6,14 +6,19 @@ WORKDIR /src COPY global.json Directory.Build.props Directory.Packages.props ./ COPY src/Anubis/Anubis.csproj src/Anubis/packages.lock.json src/Anubis/ COPY src/Anubis.Server/Anubis.Server.csproj src/Anubis.Server/packages.lock.json src/Anubis.Server/ -RUN dotnet restore src/Anubis.Server/Anubis.Server.csproj --locked-mode +COPY src/Anubis.Guard/Anubis.Guard.csproj src/Anubis.Guard/packages.lock.json src/Anubis.Guard/ +RUN dotnet restore src/Anubis.Server/Anubis.Server.csproj --locked-mode && \ + dotnet restore src/Anubis.Guard/Anubis.Guard.csproj --locked-mode COPY src/Anubis/ src/Anubis/ COPY src/Anubis.Server/ src/Anubis.Server/ +COPY src/Anubis.Guard/ src/Anubis.Guard/ # The logo is embedded from docs/design/logo, its only source (ADR 0005). COPY docs/design/logo/ docs/design/logo/ # The backtest outcomes are read at the first start (SPEC 12.3); .dockerignore lets only their JSON files through. COPY docs/backtests/ docs/backtests/ RUN dotnet publish src/Anubis.Server/Anubis.Server.csproj -c Release --no-restore -o /app +# The guard (SPEC 9.3, ADR 0012) is the main process and starts the engine next to it, from the same folder. +RUN dotnet publish src/Anubis.Guard/Anubis.Guard.csproj -c Release --no-restore -o /app FROM mcr.microsoft.com/dotnet/aspnet:10.0 WORKDIR /app @@ -27,7 +32,8 @@ ENV ANUBIS_PLATFORM=docker \ PGID=100 EXPOSE 8787 VOLUME /data -# Starts as root only to give /data to PUID:PGID, then runs the app as that user (entrypoint.sh). +# Starts as root only to give /data to PUID:PGID, then runs the guard as that user (entrypoint.sh); the guard starts +# the engine. "docker run --rm drill" runs the guard drill instead (issue #48). # Health through bash's /dev/tcp: no curl or wget in the image. 503 (database down) counts as unhealthy. HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ CMD ["bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/${PORT:-8787} && printf 'GET /health HTTP/1.0\r\nHost: localhost\r\n\r\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"] diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh index fb91c16..ec68e96 100644 --- a/deploy/docker/entrypoint.sh +++ b/deploy/docker/entrypoint.sh @@ -1,9 +1,10 @@ #!/bin/sh # Container start (SPEC 15.2, issue #16). As root: prepare /data for PUID:PGID (99:100 on Unraid), then drop to that # user with setpriv, which is part of the base image (no gosu). Started as another user (docker run --user): go straight on. +# The process is the guard (SPEC 9.3, ADR 0012): it starts the engine, watches it and passes the stop on to it. set -eu -app="/app/Anubis.Server.dll" +app="/app/Anubis.Guard.dll" if [ "$(id -u)" != "0" ]; then exec dotnet "$app" "$@" @@ -19,7 +20,7 @@ case "$uid$gid" in esac mkdir -p /data -for folder in db models research import backups logs web-keys; do +for folder in db models research import backups logs web-keys guard; do mkdir -p "/data/$folder" done # Only what does not already belong to PUID:PGID, so a large data folder does not slow every start. diff --git a/docker-compose.yml b/docker-compose.yml index db8b85a..f78ab76 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,8 @@ services: context: . dockerfile: deploy/docker/Dockerfile image: gitea.encke-hake.ts.net/balbosoft/anubis:dev + # The guard gives the engine up to 55 seconds to close in order (SPEC 9.3). + stop_grace_period: 60s ports: - "8787:8787" environment: diff --git a/src/Anubis.Guard/Anubis.Guard.csproj b/src/Anubis.Guard/Anubis.Guard.csproj index c231f26..39dcc68 100644 --- a/src/Anubis.Guard/Anubis.Guard.csproj +++ b/src/Anubis.Guard/Anubis.Guard.csproj @@ -4,6 +4,8 @@ Anubis.Guard + + diff --git a/src/Anubis.Guard/Program.cs b/src/Anubis.Guard/Program.cs index 4dfc043..445c81b 100644 --- a/src/Anubis.Guard/Program.cs +++ b/src/Anubis.Guard/Program.cs @@ -1,3 +1,49 @@ -// Placeholder: the guard process (SPEC 9.3) arrives in phase 6 (ADR 0003). -await Console.Error.WriteLineAsync("Anubis.Guard non è ancora disponibile: arriva nella Fase 6.").ConfigureAwait(false); -return 2; +using System.Collections; +using System.Runtime.InteropServices; +using Anubis.Core; +using Anubis.Platform; +using Anubis.Supervision; + +// The guard (SPEC 9.3, ADR 0012): the main process of the container and the parent of the engine. Its arguments go to +// the engine as they are. "drill" runs the guard drill in a temporary folder instead. +if (args is ["drill", ..]) +{ + return await GuardDrill.RunAsync(GuardLaunch.Server(), Console.Out, CancellationToken.None).ConfigureAwait(false); +} + +var environment = Environment.GetEnvironmentVariables().Cast() + .ToDictionary(e => (string)e.Key, e => e.Value as string, StringComparer.Ordinal); +var paths = new GuardPaths(DataRoot(args, environment)); +var logger = new LineLogger(Console.Out.WriteLine); +using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(20) }; +var clock = SystemClock.Instance; +var guardian = new Guardian(paths, clock, new GuardOptions(), + start => ChildEngine.Start(GuardLaunch.Dotnet(), [GuardLaunch.Server(), .. args], environment, start), + (account, keys) => GuardBrokers.Create(http, clock, account, keys), + (text, keys, ct) => GuardBrokers.AlertAsync(http, logger, keys, text, ct), + logger); + +// The stop of the container (SIGTERM) or Ctrl+C: passed on to the engine, which closes in order and is not restarted. +using var shutdown = new CancellationTokenSource(); +using var terminate = PosixSignalRegistration.Create(PosixSignal.SIGTERM, Stop); +using var interrupt = PosixSignalRegistration.Create(PosixSignal.SIGINT, Stop); +return await guardian.RunAsync(shutdown.Token).ConfigureAwait(false); + +void Stop(PosixSignalContext context) +{ + context.Cancel = true; + shutdown.Cancel(); +} + +// The engine's data folder: --data-dir when given, /data in Docker, the user's folder on desktop (SPEC 15.2, 15.6). +static string DataRoot(string[] args, Dictionary environment) +{ + var index = Array.IndexOf(args, "--data-dir"); + if (index >= 0 && index + 1 < args.Length) + { + return Path.GetFullPath(args[index + 1]); + } + + var platform = Array.IndexOf(args, "--platform") is var p and >= 0 && p + 1 < args.Length ? args[p + 1] : environment.GetValueOrDefault("ANUBIS_PLATFORM"); + return platform == "docker" ? "/data" : DataDirectory.ForCurrentUser().Root; +} diff --git a/src/Anubis.Server/AnubisHost.cs b/src/Anubis.Server/AnubisHost.cs index cc5be12..88144f8 100644 --- a/src/Anubis.Server/AnubisHost.cs +++ b/src/Anubis.Server/AnubisHost.cs @@ -5,6 +5,7 @@ using Anubis.Data; using Anubis.Db; using Anubis.Demo; using Anubis.Platform; +using Anubis.Supervision; using Anubis.Trading; namespace Anubis.Server; @@ -72,6 +73,9 @@ internal sealed partial class AnubisHost( public StartupConfig Config { get; private set; } = initialConfig; + /// The channel to the guard that started this engine (SPEC 9.3); null without one. + public GuardLink? Guard { get; init; } + public bool IsDemo => options.Demo; public HostStatus Status => _status; @@ -176,6 +180,12 @@ internal sealed partial class AnubisHost( SetStatus(HostMode.DatabaseUnavailable, Redactor.Redact(ex.Message)); } + // With trading the beat of a running engine comes from the trading loop, so a blocked loop is noticed. + if (_status.Mode != HostMode.Ready || _trading is null) + { + Guard?.Beat(); + } + await Task.WhenAny(wake, Task.Delay(_interval, stoppingToken)).ConfigureAwait(false); } @@ -213,6 +223,7 @@ internal sealed partial class AnubisHost( } Config.Data.Ensure(); + Guard?.Step("database"); // A data folder that belongs to an installation never gets a new empty SQLite file by itself (trap 20). _db ??= Database.Open(Config.Db!, allowCreate: !File.Exists(Config.Data.InstallationFile)); @@ -234,6 +245,8 @@ internal sealed partial class AnubisHost( return; } + Guard?.Step("migrazioni e identità"); + // Identity first, migration after: a database that belongs elsewhere is never written to, and an empty one stays // empty, so a backup taken with an older schema can still be restored into it. var backups = new Backups(Config.Data, clock); @@ -271,6 +284,7 @@ internal sealed partial class AnubisHost( return; } + Guard?.Step("servizi"); var key = Config.SecretKey.GetOrCreateKey(); var settings = new SettingsStore(_db, clock, bus, StartupOverrides()); var secrets = new SecretStore(_db, clock, key, Config.EnvironmentKeys); @@ -288,12 +302,20 @@ internal sealed partial class AnubisHost( var scheduler = new Scheduler(clock, bus); _market = MarketDataEnabled - ? new MarketDataService(_db, clock, bus, settings, secrets, Config.Data, overrides?.MarketHttp, overrides?.MarketSockets, logger) + ? new MarketDataService(_db, clock, bus, settings, secrets, Config.Data, overrides?.MarketHttp, overrides?.MarketSockets, logger, + Guard is null ? null : GuardDrill.DrillOkx(environment)) : null; _trading = TradingEnabled && (identity.File ?? Installation.ReadFile(Config.Data)) is { } file ? new TradingService(_db, clock, bus, secrets, _market, new FxStore(_db, clock), TradingConsent.From(Config), file.InstallationId, overrides?.MarketHttp, overrides?.MarketSockets, logger, settings, (password, code, ct) => access.ReauthenticateAsync(password, code, ct), Path.Combine(Config.Data.Root, "STOP")) + { + Guard = Guard, + GuardFiles = Guard is null ? null : new GuardPaths(Config.Data.Root), + + // The drill's fault is honored only under a guard, which never passes it to an engine it restarts. + Fault = Guard is null ? null : environment.GetValueOrDefault(GuardChannel.FaultVariable), + } : null; if (_trading is not null) { @@ -610,6 +632,19 @@ internal sealed partial class AnubisHost( var attempts = previous.Mode == mode ? previous.Attempts + 1 : 1; var since = previous.Mode == mode ? previous.Since : clock.UtcNow; _status = new HostStatus(mode, detail, identity, since, attempts); + if (mode == HostMode.Ready) + { + // Trading tells the guard by itself when its own startup steps are done. + if (_trading is null || _trading.IsStarted) + { + Guard?.Set(GuardStages.Running); + } + } + else + { + Guard?.Set(mode == HostMode.Starting ? GuardStages.Starting : GuardStages.Waiting); + } + if (previous.Mode != mode) { LogMode(logger, mode); diff --git a/src/Anubis.Server/ServerApp.cs b/src/Anubis.Server/ServerApp.cs index 413755f..d03cff5 100644 --- a/src/Anubis.Server/ServerApp.cs +++ b/src/Anubis.Server/ServerApp.cs @@ -4,6 +4,7 @@ using Anubis.Core; using Anubis.Demo; using Anubis.Platform; using Anubis.Server.Components; +using Anubis.Supervision; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.DataProtection; @@ -52,8 +53,12 @@ internal static class ServerApp services.AddSingleton(clock); services.AddSingleton(overrides ?? new ServerOverrides()); services.AddSingleton(); + // Started by the guard (SPEC 9.3): the channel to it. Never in demo mode, which protects nothing. services.AddSingleton(sp => new AnubisHost(options, config, environment, clock, sp.GetRequiredService(), - sp.GetRequiredService>(), overrides)); + sp.GetRequiredService>(), overrides) + { + Guard = options.Demo ? null : GuardLink.FromEnvironment(environment, sp.GetRequiredService>()), + }); services.AddHostedService(sp => sp.GetRequiredService()); services.AddSingleton(new LoginLimiter(clock)); services.AddSingleton(); @@ -82,6 +87,13 @@ internal static class ServerApp var app = builder.Build(); var host = app.Services.GetRequiredService(); + if (host.Guard is { } guard) + { + // The guard asks for an orderly stop when the container stops; every stop of the engine is announced to it. + guard.StopRequested = app.Lifetime.StopApplication; + app.Lifetime.ApplicationStopping.Register(() => guard.AnnounceAsync("uscita").GetAwaiter().GetResult()); + app.Lifetime.ApplicationStopped.Register(() => guard.DisposeAsync().AsTask().GetAwaiter().GetResult()); + } app.Use(SecurityHeaders); app.Use((context, next) => LocalOnly(context, next, options, config)); diff --git a/src/Anubis/Data/MarketDataService.cs b/src/Anubis/Data/MarketDataService.cs index 8430066..16883dc 100644 --- a/src/Anubis/Data/MarketDataService.cs +++ b/src/Anubis/Data/MarketDataService.cs @@ -54,7 +54,7 @@ public sealed partial class MarketDataService : IDisposable private (string Key, string Secret, AlpacaClient Client)? _alpaca; public MarketDataService(Database db, IClock clock, EventBus bus, SettingsStore settings, SecretStore secrets, DataDirectory? data, - HttpMessageHandler? handler = null, Func? sockets = null, ILogger? logger = null) + HttpMessageHandler? handler = null, Func? sockets = null, ILogger? logger = null, OkxOptions? okx = null) { _db = db; _clock = clock; @@ -73,7 +73,7 @@ public sealed partial class MarketDataService : IDisposable _http.Timeout = TimeSpan.FromSeconds(60); _http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}"); - Okx = new OkxClient(_http, clock, null, sockets); + Okx = new OkxClient(_http, clock, okx, sockets); Freshness = new FreshnessMonitor(clock, bus); References = new ReferenceSources(new ApiHttp(_http, clock, "reference"), db, clock, settings, secrets.RevealAsync, Okx); UsCalendar = new StoredMarketCalendar(db, clock, "us_equity"); diff --git a/src/Anubis/Notifications/TelegramClient.cs b/src/Anubis/Notifications/TelegramClient.cs new file mode 100644 index 0000000..469d859 --- /dev/null +++ b/src/Anubis/Notifications/TelegramClient.cs @@ -0,0 +1,117 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Anubis.Core; + +namespace Anubis.Notifications; + +/// A refusal or failure of the Telegram Bot API. : the network, a 429 or a 5xx. +public sealed class TelegramException : Exception +{ + public TelegramException() + { + } + + public TelegramException(string message) + : base(message) + { + } + + public TelegramException(string message, Exception innerException) + : base(message, innerException) + { + } + + public TelegramException(string message, bool retryable, TimeSpan? retryAfter = null) + : base(message) + { + Retryable = retryable; + RetryAfter = retryAfter; + } + + public bool Retryable { get; } + + /// The wait Telegram asks for after a 429. + public TimeSpan? RetryAfter { get; } +} + +/// +/// The Telegram Bot API (SPEC 14): plain text messages to the user's chat. The token is part of the address, so the +/// address is never logged; errors carry only Telegram's own description. +/// +public sealed class TelegramClient +{ + public static readonly Uri DefaultBase = new("https://api.telegram.org/"); + + private readonly HttpClient _http; + private readonly string _token; + private readonly Uri _base; + + public TelegramClient(HttpClient http, string token, Uri? baseUrl = null) + { + _http = http ?? throw new ArgumentNullException(nameof(http)); + _token = token ?? throw new ArgumentNullException(nameof(token)); + _base = baseUrl ?? DefaultBase; + Redactor.Register(token); + } + + /// Telegram's limit for one message; longer texts are cut with an ellipsis. + public const int MaxLength = 4096; + + public async Task SendAsync(string chatId, string text, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(text); + if (text.Length > MaxLength) + { + text = text[..(MaxLength - 1)] + "…"; + } + + using var answer = await CallAsync("sendMessage", new Dictionary { ["chat_id"] = chatId, ["text"] = text, ["disable_web_page_preview"] = true }, + cancellationToken).ConfigureAwait(false); + } + + /// One call of the Bot API; the answer's result when Telegram says ok. + public async Task CallAsync(string method, object body, CancellationToken cancellationToken) + { + HttpResponseMessage response; + try + { + response = await _http.PostAsJsonAsync(new Uri(_base, $"bot{_token}/{method}"), body, cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + throw new TelegramException("Telegram non risponde: " + Redactor.Redact(ex.Message), retryable: true); + } + catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TelegramException("Telegram non ha risposto in tempo.", retryable: true); + } + + using (response) + { + var text = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + JsonDocument? json = null; + try + { + json = JsonDocument.Parse(text); + } + catch (JsonException) + { + } + + if (response.IsSuccessStatusCode && json is not null && json.RootElement.TryGetProperty("ok", out var ok) && ok.GetBoolean()) + { + return json; + } + + var description = json is not null && json.RootElement.TryGetProperty("description", out var d) ? d.GetString() : null; + TimeSpan? after = json is not null && json.RootElement.TryGetProperty("parameters", out var p) && p.TryGetProperty("retry_after", out var r) + ? TimeSpan.FromSeconds(r.GetInt32()) + : null; + json?.Dispose(); + var status = (int)response.StatusCode; + throw new TelegramException($"Telegram ha risposto {status}: {Redactor.Redact(description ?? response.ReasonPhrase ?? "")}", + retryable: response.StatusCode == HttpStatusCode.TooManyRequests || status >= 500, after); + } + } +} diff --git a/src/Anubis/Supervision/ChildEngine.cs b/src/Anubis/Supervision/ChildEngine.cs new file mode 100644 index 0000000..647d0f4 --- /dev/null +++ b/src/Anubis/Supervision/ChildEngine.cs @@ -0,0 +1,158 @@ +using System.Diagnostics; +using System.IO.Pipes; +using System.Security.Cryptography; +using System.Text; + +namespace Anubis.Supervision; + +/// +/// The engine as a child process of the guard, with its channel: a named pipe open to this user only, created +/// before the process starts. The child's output goes where the guard's goes (the container log). +/// +public sealed class ChildEngine : IEngineProcess +{ + private readonly Process _process; + private readonly NamedPipeServerStream _pipe; + private readonly CancellationTokenSource _stop = new(); + private readonly SemaphoreSlim _write = new(1, 1); + private readonly Task _reading; + private readonly TaskCompletionSource _connected = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _disposed; + + private ChildEngine(Process process, NamedPipeServerStream pipe, Action onMessage) + { + _process = process; + _pipe = pipe; + Exited = WaitAsync(process); + _reading = Task.Run(() => ReadAsync(onMessage, _stop.Token)); + } + + public Task Exited { get; } + + /// + /// Starts with and the guard's environment, plus the pipe's + /// name; the drill's fault only when asks for it. + /// + public static ChildEngine Start(string file, IEnumerable arguments, IReadOnlyDictionary environment, EngineStart start) + { + ArgumentNullException.ThrowIfNull(arguments); + ArgumentNullException.ThrowIfNull(environment); + ArgumentNullException.ThrowIfNull(start); + var name = "anubis-" + Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(8)); + var pipe = new NamedPipeServerStream(name, PipeDirection.InOut, 1, PipeTransmissionMode.Byte, PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly); + var info = new ProcessStartInfo(file) { UseShellExecute = false }; + foreach (var argument in arguments) + { + info.ArgumentList.Add(argument); + } + + info.Environment.Clear(); + foreach (var (key, value) in environment) + { + if (value is not null && key != GuardChannel.FaultVariable && key != GuardChannel.PipeVariable) + { + info.Environment[key] = value; + } + } + + info.Environment[GuardChannel.PipeVariable] = name; + if (start.Fault && environment.GetValueOrDefault(GuardChannel.FaultVariable) is { Length: > 0 } fault) + { + info.Environment[GuardChannel.FaultVariable] = fault; + } + + try + { + var process = Process.Start(info) ?? throw new InvalidOperationException("Il motore non è partito."); + return new ChildEngine(process, pipe, start.OnMessage); + } + catch + { + pipe.Dispose(); + throw; + } + } + + public async Task RequestStopAsync(CancellationToken cancellationToken) + { + await _connected.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + await _write.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var bytes = Encoding.UTF8.GetBytes(GuardChannel.Write(new GuardMessage(GuardMessageTypes.Stop)) + "\n"); + await _pipe.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + await _pipe.FlushAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + _write.Release(); + } + } + + public void Kill() + { + try + { + if (!_process.HasExited) + { + _process.Kill(entireProcessTree: true); + } + } + catch (InvalidOperationException) + { + // Already gone. + } + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + _stop.Cancel(); + try + { + _reading.Wait(TimeSpan.FromSeconds(5)); + } + catch (AggregateException) + { + } + + _pipe.Dispose(); + _process.Dispose(); + _stop.Dispose(); + _write.Dispose(); + } + + private static async Task WaitAsync(Process process) + { + await process.WaitForExitAsync().ConfigureAwait(false); + return process.ExitCode; + } + + private async Task ReadAsync(Action onMessage, CancellationToken cancellationToken) + { + try + { + await _pipe.WaitForConnectionAsync(cancellationToken).ConfigureAwait(false); + _connected.TrySetResult(); + using var reader = new StreamReader(_pipe, Encoding.UTF8, leaveOpen: true); + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + if (GuardChannel.Read(line) is { } message) + { + onMessage(message); + } + } + } + catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException) + { + } + finally + { + _connected.TrySetCanceled(CancellationToken.None); + } + } +} diff --git a/src/Anubis/Supervision/GuardBrokers.cs b/src/Anubis/Supervision/GuardBrokers.cs new file mode 100644 index 0000000..92f49c3 --- /dev/null +++ b/src/Anubis/Supervision/GuardBrokers.cs @@ -0,0 +1,53 @@ +using Anubis.Brokers; +using Anubis.Core; +using Anubis.Data; +using Anubis.Notifications; +using Anubis.Trading; +using Microsoft.Extensions.Logging; + +namespace Anubis.Supervision; + +/// The broker adapters and the warnings of the guard, from the keys the engine handed it in memory. +public static class GuardBrokers +{ + /// + /// The adapter of a demo account. Real accounts get none until the real-money phase gives the guard the startup + /// consent of its own (ADR 0012); the simulator needs none, the engine is its broker. + /// + public static IBrokerAccount? Create(HttpClient http, IClock clock, GuardAccount account, IReadOnlyList keys) + { + ArgumentNullException.ThrowIfNull(account); + ArgumentNullException.ThrowIfNull(keys); + if (account.Environment != TradingEnvironments.Demo || keys.FirstOrDefault(k => k.Service == account.Broker && k.Environment == account.Environment) is not { } key) + { + return null; + } + + return account.Broker switch + { + Venues.Alpaca when key.Values.Count >= 2 => new AlpacaTrading(http, clock, new AlpacaCredentials(key.Values[0], key.Values[1]), + new AlpacaTradingOptions { Environment = account.Environment }, TradingConsent.None), + Venues.Okx when key.Values.Count >= 3 => new OkxTrading(http, clock, new OkxCredentials(key.Values[0], key.Values[1], key.Values[2]), + new OkxTradingOptions { Environment = account.Environment }, TradingConsent.None), + _ => null, + }; + } + + /// The guard's warning: always in its log, and on Telegram when the engine handed it the bot's token and chat. + public static async Task AlertAsync(HttpClient http, ILogger logger, IReadOnlyList keys, string text, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(keys); + GuardLog.Alert(logger, text.ReplaceLineEndings(" ")); + if (keys.FirstOrDefault(k => k.Service == GuardKeys.Telegram) is { Values.Count: >= 2 } telegram) + { + await new TelegramClient(http, telegram.Values[0]).SendAsync(telegram.Values[1], "Anubis, guardia" + Environment.NewLine + text, cancellationToken) + .ConfigureAwait(false); + } + } +} + +internal static partial class GuardLog +{ + [LoggerMessage(Level = LogLevel.Warning, Message = "Guard alert: {Text}")] + public static partial void Alert(ILogger logger, string text); +} diff --git a/src/Anubis/Supervision/GuardDrill.cs b/src/Anubis/Supervision/GuardDrill.cs new file mode 100644 index 0000000..433ccbe --- /dev/null +++ b/src/Anubis/Supervision/GuardDrill.cs @@ -0,0 +1,419 @@ +using System.Globalization; +using System.Net; +using System.Net.Sockets; +using System.Security.Cryptography; +using System.Text; +using Anubis.Core; +using Anubis.Data; +using Anubis.Db; +using Anubis.Platform; +using Anubis.Trading; +using Microsoft.Extensions.Logging; + +namespace Anubis.Supervision; + +/// The outcome of the guard drill, line by line in Italian. +public sealed record DrillResult(bool Passed, IReadOnlyList Lines); + +/// +/// The guard drill (issue #48, ADR 0012): the real guard and the real engine, as child process, in a folder of their +/// own with a SQLite database. A position of 0.01 BTC with its stop on the internal simulator and a sale already +/// decided; the engine starts with the fault that blocks its main loop right after the sale leaves, with the stop +/// removed. The drill passes when the guard intervened, the engine restarted, the sale left open was canceled, the +/// stop came back for the whole quantity and the guard's note reached the diary. Prices come from a small local +/// server that answers like OKX: nothing leaves the computer. Same drill in the tests and in the Docker image. +/// +public static class GuardDrill +{ + public const decimal Quantity = 0.01m; + public const decimal StopLevel = 45000m; + + /// Times short enough for a drill of about two minutes. + public static GuardOptions Options { get; } = new() + { + Silence = TimeSpan.FromSeconds(20), + StalledStart = TimeSpan.FromMinutes(3), + GracefulStop = TimeSpan.FromSeconds(30), + }; + + /// The drill in a new temporary folder, removed at the end. Exit code 0 when it passes. + public static async Task RunAsync(string serverDll, TextWriter output, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(output); + var folder = Path.Combine(Path.GetTempPath(), "anubis-drill-" + Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(6))); + try + { + var result = await RunInAsync(folder, serverDll, Options, line => output.WriteLine(line), cancellationToken).ConfigureAwait(false); + await output.WriteLineAsync(result.Passed ? "Esercitazione della guardia superata." : "Esercitazione della guardia NON superata.").ConfigureAwait(false); + return result.Passed ? 0 : 1; + } + finally + { + try + { + Directory.Delete(folder, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // A leftover temporary folder is harmless. + } + } + } + + public static async Task RunInAsync(string folder, string serverDll, GuardOptions options, Action say, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(say); + var lines = new List(); + void Say(string line) + { + lines.Add(line); + say(line); + } + + var data = new DataDirectory(folder); + data.Ensure(); + using var okx = FakeOkx.Start(); + var held = await SeedAsync(data, cancellationToken).ConfigureAwait(false); + Say($"Preparato: {Format.Quantity(held)} BTC con lo stop a {Format.Price(StopLevel)} e una vendita già decisa."); + + var environment = ChildEnvironment(okx.Address); + var clock = SystemClock.Instance; + var logger = new LineLogger(Say); + var guardian = new Guardian(new GuardPaths(folder), clock, options, + start => ChildEngine.Start(GuardLaunch.Dotnet(), [serverDll, "--platform", "docker", "--data-dir", folder], environment, start), + (_, _) => null, (text, _, _) => + { + Say("Avviso della guardia: " + text.ReplaceLineEndings(" ")); + return Task.CompletedTask; + }, logger); + guardian.Start(fault: true); + using var stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var guarding = guardian.RunAsync(stop.Token); + + var intervened = await WaitAsync(() => guardian.Interventions > 0, options.Silence + TimeSpan.FromMinutes(3), cancellationToken).ConfigureAwait(false); + var restarted = intervened && await WaitAsync(() => guardian.EngineAlive && guardian.Stage == GuardStages.Running, TimeSpan.FromMinutes(3), cancellationToken) + .ConfigureAwait(false); + await stop.CancelAsync().ConfigureAwait(false); + var code = await guarding.ConfigureAwait(false); + Say($"Motore fermato in ordine alla fine dell'esercitazione (codice {code})."); + + var passed = Check(intervened, "La guardia è intervenuta dopo il silenzio del ciclo principale.", "La guardia non è intervenuta.", Say); + passed &= Check(restarted, "Il motore è ripartito e il suo ciclo batte di nuovo.", "Il motore non è ripartito.", Say); + var notes = guardian.Notes; + var at = notes.Count > 0 ? notes[0].At : DateTimeOffset.MaxValue; + await using (var db = Database.Open(new DbSettings { Engine = DbEngine.Sqlite, SqlitePath = data.SqliteFile }, allowCreate: false)) + { + var hung = await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM orders WHERE side = 'sell' AND purpose = 'exit' AND created_at < @t", ("t", at)), cancellationToken) + .ConfigureAwait(false); + var canceled = await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM executions WHERE side = 'sell' AND purpose = 'exit' AND status = 'canceled' AND reason = @r", + ("r", LiveReasons.Restart)), cancellationToken).ConfigureAwait(false); + var stops = await db.ReadAsync(s => s.QueryAsync("SELECT quantity FROM orders WHERE purpose = 'stop' AND created_at > @t", r => r.Money("quantity"), ("t", at)), + cancellationToken).ConfigureAwait(false); + var noted = await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM journal WHERE code = 'guard_intervention'"), cancellationToken).ConfigureAwait(false); + var again = await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM planned_orders WHERE side = 'sell' AND created_at > @t", ("t", at)), cancellationToken) + .ConfigureAwait(false); + passed &= Check(hung > 0, "Prima dell'intervento la vendita era partita, senza più lo stop.", "La vendita non era partita prima dell'intervento.", Say); + passed &= Check(canceled > 0, "Alla ripartenza la vendita rimasta a metà è stata annullata.", "La vendita rimasta a metà non è stata annullata.", Say); + passed &= Check(stops.Any(q => q == held), $"Lo stop è tornato per tutti i {Format.Quantity(held)} BTC.", "Lo stop non è tornato per l'intera quantità.", Say); + passed &= Check(again > 0, "La vendita già decisa è ripartita con un ordine nuovo.", "La vendita già decisa non è ripartita.", Say); + passed &= Check(noted > 0, "La nota della guardia è nel diario.", "La nota della guardia non è nel diario.", Say); + } + + return new DrillResult(passed, lines); + } + + private static bool Check(bool ok, string yes, string no, Action say) + { + say((ok ? "OK " : "NO ") + (ok ? yes : no)); + return ok; + } + + private static async Task WaitAsync(Func condition, TimeSpan timeout, CancellationToken cancellationToken) + { + var until = SystemClock.Instance.UtcNow + timeout; + while (!condition()) + { + if (SystemClock.Instance.UtcNow > until) + { + return false; + } + + await Task.Delay(250, cancellationToken).ConfigureAwait(false); + } + + return true; + } + + private static Dictionary ChildEnvironment(Uri okx) + { + var environment = new Dictionary(StringComparer.Ordinal); + foreach (System.Collections.DictionaryEntry entry in Environment.GetEnvironmentVariables()) + { + var key = (string)entry.Key; + if (!key.StartsWith("ANUBIS_", StringComparison.Ordinal) && !key.StartsWith("ALPACA_", StringComparison.Ordinal) && + !key.StartsWith("OKX_", StringComparison.Ordinal) && !key.StartsWith("TELEGRAM_", StringComparison.Ordinal)) + { + environment[key] = entry.Value as string; + } + } + + environment["ANUBIS_PLATFORM"] = "docker"; + environment["ANUBIS_DB_ENGINE"] = "sqlite"; + environment["ANUBIS_SECRET_KEY"] = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(24)); + environment["ANUBIS_ADMIN_PASSWORD"] = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(12)); + environment["PORT"] = FreePort().ToString(CultureInfo.InvariantCulture); + environment["TZ"] = "Europe/Rome"; + environment[DrillOkxVariable] = okx.ToString(); + environment[GuardChannel.FaultVariable] = GuardChannel.HangMidSale; + return environment; + } + + /// + /// The address of the local server that answers like OKX in the drill. Honored only on this computer's own + /// addresses (loopback): it can never point the bot at a price source elsewhere. + /// + public const string DrillOkxVariable = "ANUBIS_DRILL_OKX_URL"; + + /// The OKX addresses of the drill, or null when the variable is missing or not a loopback address. + public static Brokers.OkxOptions? DrillOkx(IReadOnlyDictionary environment) + { + ArgumentNullException.ThrowIfNull(environment); + if (environment.GetValueOrDefault(DrillOkxVariable) is not { Length: > 0 } text || !Uri.TryCreate(text, UriKind.Absolute, out var url) || !url.IsLoopback) + { + return null; + } + + var socket = new UriBuilder(url) { Scheme = "ws" }; + return new Brokers.OkxOptions + { + RestUrl = url, + PublicStreamUrl = new Uri(socket.Uri, "/ws/v5/public"), + BusinessStreamUrl = new Uri(socket.Uri, "/ws/v5/business"), + }; + } + + // The database the engine will find: installation, accounts, BTC, the capital, a filled purchase with its stop on + // the simulator, and the sale decided for now. Built with the same order manager the engine uses. + private static async Task SeedAsync(DataDirectory data, CancellationToken cancellationToken) + { + var clock = SystemClock.Instance; + await using var db = Database.Open(new DbSettings { Engine = DbEngine.Sqlite, SqlitePath = data.SqliteFile }, allowCreate: true); + await Migrations.MigrateAsync(db, clock, null, cancellationToken).ConfigureAwait(false); + var identity = await Installation.CheckAsync(db, data, clock, cancellationToken).ConfigureAwait(false); + var accounts = await AccountSetup.EnsureAsync(db, clock, alpacaDemo: false, okxDemo: false, cancellationToken).ConfigureAwait(false); + var account = accounts.First(a => a.Broker == Venues.Okx && a.Environment == TradingEnvironments.Simulation); + var btc = await new InstrumentCatalog(db, clock).EnsureAsync(new InstrumentSpec(Venues.Okx, "BTC-EUR", "BTC-EUR", AssetClasses.Crypto, "EUR") + { + BaseCurrency = "BTC", + TickSize = 0.1m, + LotSize = 0.00000001m, + MinSize = 0.00001m, + Status = "live", + }, cancellationToken).ConfigureAwait(false); + var ledger = new Ledger(db, clock); + await new Capital(db, clock, ledger, new FxStore(db, clock)).SetAsync(Venues.Okx, 1000m, "esercitazione", cancellationToken).ConfigureAwait(false); + + Task QuoteAsync(string symbol, CancellationToken _) => Task.FromResult(new Quote(symbol, FakeOkx.Bid, FakeOkx.Ask, clock.UtcNow)); + using var paper = new PaperBroker(PaperOptions.Okx() with { MinLatency = TimeSpan.Zero, MaxLatency = TimeSpan.Zero }, clock, QuoteAsync, + symbol => symbol == btc.Symbol ? btc.Spec : null, new PaperBookStore(db, clock, account.AccountId)); + await paper.DepositAsync(1000m, "EUR", cancellationToken).ConfigureAwait(false); + using var manager = new OrderManager(db, clock, ledger, paper, account, ClientOrderIds.InstallationCode(identity.File!.InstallationId), + (id, _) => Task.FromResult(id == btc.Id ? btc : null), QuoteAsync) + { + PassiveSteps = 0, + ConfirmPrice = QuoteAsync, + MarketOpen = (_, _) => Task.FromResult(true), + }; + var (_, refusal) = await manager.SubmitAsync(new TradeIntent(btc.Id, StrategyNames.CryptoTrend, OrderSide.Buy, Quantity, OrderPurpose.Entry, Ids.NewId(clock)) + { + StopLevel = StopLevel, + }, cancellationToken).ConfigureAwait(false); + if (refusal is not null) + { + throw new InvalidOperationException("Esercitazione: acquisto rifiutato (" + refusal + ")."); + } + + for (var i = 0; ; i++) + { + await paper.PumpAsync(cancellationToken).ConfigureAwait(false); + await manager.TickAsync(cancellationToken).ConfigureAwait(false); + var stops = await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM orders WHERE purpose = 'stop' AND status = 'submitted'"), cancellationToken).ConfigureAwait(false); + if (stops > 0) + { + break; + } + + if (i > 100) + { + throw new InvalidOperationException("Esercitazione: la posizione non ha avuto il suo stop."); + } + + await Task.Delay(100, cancellationToken).ConfigureAwait(false); + } + + var held = (await ledger.PositionsAsync(account.AccountId, cancellationToken).ConfigureAwait(false)).Single().Quantity; + var now = clock.UtcNow; + await db.WriteAsync(s => s.ExecuteAsync( + "INSERT INTO planned_orders (id, decision_id, strategy, account_id, instrument_id, side, purpose, quantity, exit_all, stop_level, stop_offset, reference_price, reason, " + + "due_at, valid_until, status, outcome, execution_id, created_at, updated_at) VALUES (@id, @d, @s, @a, @i, 'sell', 'exit', @q, @x, NULL, NULL, @r, 'esercitazione', " + + "@now, NULL, 'pending', NULL, NULL, @now, @now)", cancellationToken, + ("id", Ids.NewId(clock)), ("d", Ids.NewId(clock)), ("s", StrategyNames.CryptoTrend), ("a", account.AccountId), ("i", btc.Id), ("q", held), ("x", true), + ("r", FakeOkx.Bid), ("now", now)), cancellationToken).ConfigureAwait(false); + return held; + } + + private static int FreePort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return ((IPEndPoint)listener.LocalEndpoint).Port; + } + + /// + /// A local server that answers like OKX's REST interface with a fixed price for every ticker and nothing for the + /// rest; anything else (the streams) gets 404. Plain HTTP/1.1 on the loopback address, one request per connection. + /// + private sealed class FakeOkx : IDisposable + { + public const decimal Bid = 50000m; + public const decimal Ask = 50010m; + + private readonly TcpListener _listener; + private readonly CancellationTokenSource _stop = new(); + private readonly Task _loop; + + private FakeOkx() + { + _listener = new TcpListener(IPAddress.Loopback, 0); + _listener.Start(); + Address = new Uri($"http://127.0.0.1:{((IPEndPoint)_listener.LocalEndpoint).Port}/"); + _loop = Task.Run(() => AcceptAsync(_stop.Token)); + } + + public Uri Address { get; } + + public static FakeOkx Start() => new(); + + public void Dispose() + { + _stop.Cancel(); + _listener.Stop(); + try + { + _loop.Wait(TimeSpan.FromSeconds(5)); + } + catch (AggregateException) + { + } + + _stop.Dispose(); + } + + private async Task AcceptAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + TcpClient client; + try + { + client = await _listener.AcceptTcpClientAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException) + { + return; + } + + _ = Task.Run(() => AnswerAsync(client, cancellationToken), CancellationToken.None); + } + } + + private static async Task AnswerAsync(TcpClient client, CancellationToken cancellationToken) + { + using (client) + { + try + { + var stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.ASCII, leaveOpen: true); + var request = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) ?? ""; + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { Length: > 0 }) + { + } + + var target = request.Split(' ') is [_, var path, ..] ? path : "/"; + var (status, body) = Route(target); + var bytes = Encoding.UTF8.GetBytes(body); + var head = $"HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {bytes.Length}\r\nConnection: close\r\n\r\n"; + await stream.WriteAsync(Encoding.ASCII.GetBytes(head), cancellationToken).ConfigureAwait(false); + await stream.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or OperationCanceledException or SocketException) + { + } + } + } + + private static (string Status, string Body) Route(string target) + { + if (target.StartsWith("/ws", StringComparison.Ordinal)) + { + return ("404 Not Found", "{}"); + } + + if (target.StartsWith("/api/v5/market/ticker?instId=", StringComparison.Ordinal)) + { + var symbol = Uri.UnescapeDataString(target["/api/v5/market/ticker?instId=".Length..]); + var ts = SystemClock.Instance.UtcNow.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture); + return ("200 OK", $"{{\"code\":\"0\",\"msg\":\"\",\"data\":[{{\"instId\":\"{symbol}\",\"bidPx\":\"{Bid.ToString(CultureInfo.InvariantCulture)}\"," + + $"\"askPx\":\"{Ask.ToString(CultureInfo.InvariantCulture)}\",\"bidSz\":\"1\",\"askSz\":\"1\",\"ts\":\"{ts}\"}}]}}"); + } + + // The list of instruments has the drill's one: an instrument missing from it is taken as delisted. + if (target.StartsWith("/api/v5/public/instruments", StringComparison.Ordinal)) + { + return ("200 OK", "{\"code\":\"0\",\"msg\":\"\",\"data\":[{\"instId\":\"BTC-EUR\",\"baseCcy\":\"BTC\",\"quoteCcy\":\"EUR\",\"tickSz\":\"0.1\",\"lotSz\":\"0.00000001\",\"minSz\":\"0.00001\",\"state\":\"live\"}]}"); + } + + return ("200 OK", "{\"code\":\"0\",\"msg\":\"\",\"data\":[]}"); + } + } +} + +/// How the guard finds the .NET host and the engine next to itself. +public static class GuardLaunch +{ + /// The dotnet host that runs the engine: the one running now when it is dotnet itself, otherwise the one the SDK names. + public static string Dotnet() + { + if (Environment.ProcessPath is { } current && Path.GetFileNameWithoutExtension(current).Equals("dotnet", StringComparison.OrdinalIgnoreCase)) + { + return current; + } + + return Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") is { Length: > 0 } host ? host : "dotnet"; + } + + /// The engine's program next to the guard's. + public static string Server() => Path.Combine(AppContext.BaseDirectory, "Anubis.Server.dll"); +} + +/// A logger that writes one line per message: the guard's log goes where the container's goes. +public sealed class LineLogger(Action write) : ILogger +{ + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Information; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + ArgumentNullException.ThrowIfNull(formatter); + if (!IsEnabled(logLevel)) + { + return; + } + + var text = Redactor.Redact(formatter(state, exception) + (exception is null ? "" : " — " + exception.Message)); + write($"{SystemClock.Instance.UtcNow:yyyy-MM-dd HH:mm:ss} guardia {logLevel}: {text}"); + } +} diff --git a/src/Anubis/Supervision/GuardLink.cs b/src/Anubis/Supervision/GuardLink.cs new file mode 100644 index 0000000..365ad08 --- /dev/null +++ b/src/Anubis/Supervision/GuardLink.cs @@ -0,0 +1,170 @@ +using System.IO.Pipes; +using System.Text; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Anubis.Supervision; + +/// +/// The engine's end of the channel to the guard (SPEC 9.3): the beat of the main loop, the startup steps, the keys +/// and the stops it decides. Messages wait in a short queue and the oldest beats give way: a slow guard never slows +/// the engine. When the guard is gone the engine goes on by itself. +/// +public sealed partial class GuardLink : IAsyncDisposable +{ + private readonly Lock _gate = new(); + private readonly NamedPipeClientStream _pipe; + private readonly Channel _outgoing = Channel.CreateBounded(new BoundedChannelOptions(64) { FullMode = BoundedChannelFullMode.DropOldest }); + private readonly CancellationTokenSource _stop = new(); + private readonly ILogger _logger; + private readonly Task _loop; + private string _stage = GuardStages.Starting; + private string? _step; + + private GuardLink(string pipeName, ILogger logger) + { + _logger = logger; + _pipe = new NamedPipeClientStream(".", pipeName, PipeDirection.InOut, PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly); + _loop = Task.Run(() => RunAsync(_stop.Token)); + } + + /// The link when a guard started this engine (); null otherwise. + public static GuardLink? FromEnvironment(IReadOnlyDictionary environment, ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(environment); + return environment.GetValueOrDefault(GuardChannel.PipeVariable) is { Length: > 0 } name ? new GuardLink(name, logger ?? NullLogger.Instance) : null; + } + + /// The guard asked for an orderly stop. + public Action? StopRequested { get; set; } + + public string Stage + { + get + { + lock (_gate) + { + return _stage; + } + } + } + + /// A new stage or startup step: told to the guard at once. + public void Set(string stage, string? step = null) + { + lock (_gate) + { + if (_stage == stage && _step == step) + { + return; + } + + _stage = stage; + _step = step; + } + + Beat(); + } + + /// A step of the startup of SPEC 9.4. + public void Step(string step) => Set(GuardStages.Starting, step); + + /// Alive, at the current stage. + public void Beat() + { + GuardMessage message; + lock (_gate) + { + message = GuardMessage.Beat(_stage, _step); + } + + _outgoing.Writer.TryWrite(message); + } + + public void SendKeys(IReadOnlyList keys) => _outgoing.Writer.TryWrite(new GuardMessage(GuardMessageTypes.Keys) { Keys = keys }); + + /// A stop decided by the engine: the guard will not restart it. Waits a moment for the message to leave. + public async Task AnnounceAsync(string reason) + { + _outgoing.Writer.TryWrite(new GuardMessage(GuardMessageTypes.Stopping) { Reason = reason }); + for (var i = 0; i < 20 && _outgoing.Reader.Count > 0; i++) + { + await Task.Delay(50).ConfigureAwait(false); + } + } + + public async ValueTask DisposeAsync() + { + await _stop.CancelAsync().ConfigureAwait(false); + try + { + await _loop.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + + await _pipe.DisposeAsync().ConfigureAwait(false); + _stop.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + await _pipe.ConnectAsync(TimeSpan.FromSeconds(30), cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is TimeoutException or IOException or UnauthorizedAccessException) + { + LogNoGuard(_logger, ex.Message); + return; + } + + var reading = ReadAsync(cancellationToken); + var writer = new StreamWriter(_pipe, new UTF8Encoding(false), leaveOpen: true) { AutoFlush = true, NewLine = "\n" }; + await using (writer.ConfigureAwait(false)) + { + try + { + await foreach (var message in _outgoing.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + await writer.WriteLineAsync(GuardChannel.Write(message).AsMemory(), cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (IOException ex) + { + LogGuardLost(_logger, ex.Message); + } + } + + await reading.ConfigureAwait(false); + } + + private async Task ReadAsync(CancellationToken cancellationToken) + { + using var reader = new StreamReader(_pipe, Encoding.UTF8, leaveOpen: true); + try + { + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + if (GuardChannel.Read(line) is { Type: GuardMessageTypes.Stop }) + { + StopRequested?.Invoke(); + } + } + } + catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException) + { + } + } + + [LoggerMessage(Level = LogLevel.Warning, Message = "Guard channel not reached: {Reason}. The engine runs without its guard.")] + private static partial void LogNoGuard(ILogger logger, string reason); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Guard channel lost: {Reason}")] + private static partial void LogGuardLost(ILogger logger, string reason); +} diff --git a/src/Anubis/Supervision/GuardProtocol.cs b/src/Anubis/Supervision/GuardProtocol.cs new file mode 100644 index 0000000..3f3f4af --- /dev/null +++ b/src/Anubis/Supervision/GuardProtocol.cs @@ -0,0 +1,120 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Anubis.Supervision; + +/// +/// The channel between the guard and the engine (SPEC 9.3, ADR 0012): a local named pipe whose name the guard puts +/// in , one JSON message per line. The engine sends its beat, the startup steps, the broker +/// keys (in memory only) and the stops it decides; the guard sends only "stop". +/// +public static class GuardChannel +{ + /// The pipe the engine connects to. Without it the engine runs on its own (development, tests). + public const string PipeVariable = "ANUBIS_GUARD_PIPE"; + + /// + /// Fault injection for the guard drill (ADR 0012): blocks the main loop right after a sale + /// leaves. The guard never passes it to an engine it restarts. + /// + public const string FaultVariable = "ANUBIS_FAULT"; + + public const string HangMidSale = "hang-mid-sale"; + + public static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Converters = { new JsonStringEnumConverter() }, + }; + + public static string Write(GuardMessage message) => JsonSerializer.Serialize(message, Json); + + /// A line of the channel; null when it is not a message (a broken line is ignored, never fatal). + public static GuardMessage? Read(string? line) + { + if (string.IsNullOrWhiteSpace(line)) + { + return null; + } + + try + { + return JsonSerializer.Deserialize(line, Json) is { Type.Length: > 0 } message ? message : null; + } + catch (JsonException) + { + return null; + } + } +} + +public static class GuardMessageTypes +{ + /// Engine to guard: alive, with its stage and the startup step it is on. + public const string Beat = "beat"; + + /// Engine to guard: the keys of the broker accounts and of Telegram, kept in memory only. + public const string Keys = "keys"; + + /// Engine to guard: a stop decided by the engine (exit, restore, import): not restarted. + public const string Stopping = "stopping"; + + /// Guard to engine: stop in an orderly way (the container is stopping). + public const string Stop = "stop"; +} + +/// Stages of the engine as the guard sees them. +public static class GuardStages +{ + /// The startup steps of SPEC 9.4: watched for 10 minutes without progress. + public const string Starting = "starting"; + + /// The main loop runs: watched for 2 minutes of silence. + public const string Running = "running"; + + /// Error page or first run: waiting for the user, never a reason to intervene. + public const string Waiting = "waiting"; +} + +/// One message of the channel. +public sealed record GuardMessage(string Type) +{ + public string? Stage { get; init; } + + public string? Step { get; init; } + + public string? Reason { get; init; } + + public IReadOnlyList? Keys { get; init; } + + public static GuardMessage Beat(string stage, string? step) => new(GuardMessageTypes.Beat) { Stage = stage, Step = step }; + + // Never prints the keys. + public override string ToString() => $"GuardMessage({Type}, {Stage}, {Step})"; +} + +/// +/// The keys of one account, as the engine has them: Alpaca key and secret, OKX key, secret and passphrase, Telegram +/// token and chat. Held only in the guard's memory, never written (SPEC 9.3). +/// +public sealed record GuardKeys(string Service, string Environment, IReadOnlyList Values) +{ + public const string Telegram = "telegram"; + + public override string ToString() => $"GuardKeys({Service}, {Environment}, ***)"; +} + +/// Where the guard and the engine meet in the data folder. +public sealed record GuardPaths(string DataRoot) +{ + public string Folder => Path.Combine(DataRoot, "guard"); + + /// Positions, stop levels and open orders, kept current by the engine for the guard. + public string StateFile => Path.Combine(Folder, "state.json"); + + /// The guard does not write in the database: it leaves notes here, which the engine puts in the diary. + public string NotesFolder => Path.Combine(Folder, "notes"); + + /// The "Ferma tutto" file (SPEC 9.3). + public string StopFile => Path.Combine(DataRoot, "STOP"); +} diff --git a/src/Anubis/Supervision/GuardRepair.cs b/src/Anubis/Supervision/GuardRepair.cs new file mode 100644 index 0000000..c0ed4f6 --- /dev/null +++ b/src/Anubis/Supervision/GuardRepair.cs @@ -0,0 +1,130 @@ +using Anubis.Core; +using Anubis.Trading; + +namespace Anubis.Supervision; + +/// What the guard did on one account. +public sealed record RepairOutcome(IReadOnlyList Lines, IReadOnlyList Placed, bool Complete); + +/// +/// The guard's work on a broker account while the engine is stopped (SPEC 9.3): reads positions and the outcome of the +/// orders in the state file, cancels the purchases and sales left open, and puts back only the stops that are +/// missing and were not executed, for the quantity still held. Protective sales and stops are never touched. +/// +public static class GuardRepair +{ + /// + /// The guard's stops take sequences this far above the engine's next one: if a note were lost, the engine's next + /// stops would still not reuse their ids. + /// + public const int SequenceGap = 500; + + public static async Task RunAsync(GuardState state, GuardAccount account, IBrokerAccount broker, IClock clock, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(state); + ArgumentNullException.ThrowIfNull(account); + ArgumentNullException.ThrowIfNull(broker); + ArgumentNullException.ThrowIfNull(clock); + var lines = new List(); + var placed = new List(); + var complete = true; + + // 1. Purchases and sales still waiting: the open ones the broker shows, and those the state lists. + var open = (await broker.GetOpenOrdersAsync(cancellationToken).ConfigureAwait(false)) + .Where(o => ClientOrderIds.IsOurs(o.ClientOrderId, state.Installation)).ToList(); + var waiting = open.Where(o => ClientOrderIds.Parse(o.ClientOrderId)?.Purpose is OrderPurpose.Entry or OrderPurpose.Exit) + .Select(o => (o.ClientOrderId, o.Symbol, o.Side)).ToList(); + foreach (var listed in account.Orders.Where(o => o.Purpose is OrderPurpose.Entry or OrderPurpose.Exit && waiting.All(w => w.ClientOrderId != o.ClientOrderId))) + { + if (await broker.GetOrderAsync(listed.ClientOrderId, listed.Symbol, cancellationToken).ConfigureAwait(false) is { IsOpen: true } remote) + { + waiting.Add((remote.ClientOrderId, remote.Symbol, remote.Side)); + } + } + + foreach (var (clientId, symbol, side) in waiting) + { + await broker.CancelOrderAsync(clientId, symbol, cancellationToken).ConfigureAwait(false); + lines.Add($"Ho annullato {(side == OrderSide.Buy ? "l'acquisto" : "la vendita")} di {Shown(symbol)} rimast{(side == OrderSide.Buy ? "o" : "a")} in attesa."); + } + + // 2. What the bot holds now: the state's quantity plus the fills of its own orders since then, never more than the account has. + var fills = (await broker.GetFillsAsync(state.WrittenAt, cancellationToken).ConfigureAwait(false)) + .Where(f => f.ExecutedAt > state.WrittenAt && ClientOrderIds.IsOurs(f.ClientOrderId, state.Installation)).ToList(); + var positions = await broker.GetPositionsAsync(cancellationToken).ConfigureAwait(false); + var stops = (await broker.GetOpenOrdersAsync(cancellationToken).ConfigureAwait(false)) + .Where(o => o.IsOpen && o.Side == OrderSide.Sell && ClientOrderIds.Parse(o.ClientOrderId) is { Purpose: OrderPurpose.Stop } parts && parts.Installation == state.Installation) + .ToList(); + foreach (var bySymbol in account.Positions.Where(p => p.StopLevel is not null).GroupBy(p => p.Symbol, StringComparer.Ordinal)) + { + var available = positions.Where(p => p.Symbol == bySymbol.Key).Sum(p => p.Quantity); + foreach (var position in bySymbol.OrderBy(p => p.Strategy, StringComparer.Ordinal)) + { + var code = ClientOrderIds.StrategyCodes.GetValueOrDefault(position.Strategy); + var moved = fills.Where(f => f.Symbol == position.Symbol && ClientOrderIds.Parse(f.ClientOrderId)?.Strategy == position.Strategy) + .Sum(f => f.Side == OrderSide.Buy ? f.Quantity : -f.Quantity); + var held = Math.Min(Math.Max(0m, position.Quantity + moved), available); + available -= held; + var covered = stops.Where(o => o.Symbol == position.Symbol && code is not null && ClientOrderIds.Parse(o.ClientOrderId)?.Strategy == position.Strategy) + .Sum(o => o.Remaining); + var missing = OrderRules.RoundDown(held - covered, position.Lot); + if (missing <= 0m || code is null) + { + continue; + } + + var sequence = Math.Min(position.NextStopSequence + SequenceGap, ClientOrderIds.MaxSequence - 2); + foreach (var (quantity, timeInForce) in Shapes(position, missing, account.FractionalStopsAreDayOnly)) + { + var clientId = ClientOrderIds.Build(state.Installation, account.Environment, position.Strategy, OrderPurpose.Stop, position.StopKey, sequence++); + var request = new OrderRequest(clientId, position.Symbol, OrderSide.Sell, OrderKind.Stop, quantity, timeInForce) { StopPrice = position.StopLevel }; + try + { + var remote = await broker.SubmitOrderAsync(request, cancellationToken).ConfigureAwait(false); + placed.Add(new PlacedStop(account.AccountId, position.InstrumentId, position.Symbol, position.Strategy, clientId, remote.BrokerOrderId, quantity, + position.StopLevel!.Value, timeInForce)); + lines.Add($"Ho rimesso lo stop di {Shown(position.Symbol)} a {Format.Price(position.StopLevel.Value)} per {Format.Quantity(quantity)}."); + } + catch (OrderRejectedException ex) + { + complete = false; + lines.Add($"{account.Name} ha rifiutato lo stop di {Shown(position.Symbol)} ({ex.Code}): la posizione resta senza stop finché il motore non riparte."); + } + } + } + } + + if (lines.Count == 0) + { + lines.Add($"Su {account.Name} non c'era nulla da annullare e ogni posizione aveva il suo stop."); + } + + return new RepairOutcome(lines, placed, complete); + } + + /// The stops a quantity needs on the venue, as the engine places them (SPEC 7.2): fractions of shares only as day orders. + public static IReadOnlyList<(decimal Quantity, TimeInForce TimeInForce)> Shapes(GuardPosition position, decimal quantity, bool fractionalDayOnly) + { + ArgumentNullException.ThrowIfNull(position); + if (!fractionalDayOnly || position.IsCrypto) + { + return [(quantity, TimeInForce.Gtc)]; + } + + var (whole, fraction) = OrderRules.SplitWhole(quantity); + var shapes = new List<(decimal, TimeInForce)>(); + if (whole > 0m) + { + shapes.Add((whole, TimeInForce.Gtc)); + } + + if (fraction > 0m) + { + shapes.Add((fraction, TimeInForce.Day)); + } + + return shapes; + } + + private static string Shown(string symbol) => DiaryText.Shown(symbol); +} diff --git a/src/Anubis/Supervision/GuardState.cs b/src/Anubis/Supervision/GuardState.cs new file mode 100644 index 0000000..6ecf457 --- /dev/null +++ b/src/Anubis/Supervision/GuardState.cs @@ -0,0 +1,116 @@ +using System.Globalization; +using System.Text.Json; +using Anubis.Trading; + +namespace Anubis.Supervision; + +/// +/// What the guard needs to protect the positions while the engine is stopped (SPEC 9.3): written by the engine every +/// few seconds, read by the guard only after it stopped the engine. +/// +public sealed record GuardState(DateTimeOffset WrittenAt, string Installation, IReadOnlyList Accounts); + +public sealed record GuardAccount(Guid AccountId, string Name, string Broker, string Environment, bool FractionalStopsAreDayOnly, + IReadOnlyList Positions, IReadOnlyList Orders); + +/// +/// A position of a strategy: quantity, stop level already on the tick, lot, and what the guard needs to name a stop +/// the engine will recognize as its own (the key of the position's stops and the next free sequence). +/// +public sealed record GuardPosition(Guid InstrumentId, string Symbol, string Strategy, decimal Quantity, decimal? StopLevel, decimal Lot, bool IsCrypto, + string StopKey, int NextStopSequence); + +/// An order the engine believes open. +public sealed record GuardOrder(string ClientOrderId, Guid InstrumentId, string Symbol, string Strategy, OrderSide Side, OrderPurpose Purpose, decimal Quantity, + decimal? StopPrice); + +/// A stop the guard placed while the engine was stopped: the engine adopts it at the restart. +public sealed record PlacedStop(Guid AccountId, Guid InstrumentId, string Symbol, string Strategy, string ClientOrderId, string? BrokerOrderId, decimal Quantity, + decimal StopPrice, TimeInForce TimeInForce); + +/// +/// The note of one intervention: when, why, what the guard did, in the words of the diary. The engine writes it in the +/// diary at the restart and adopts the stops it lists. +/// +public sealed record GuardNote(DateTimeOffset At, string Reason, IReadOnlyList Lines, IReadOnlyList Placed) +{ + /// The engine was silent for the 2 minutes of SPEC 9.3. + public const string Silence = "silence"; + + /// The startup steps did not move for 10 minutes. + public const string Stalled = "stalled"; + + /// The engine ended without announcing it. + public const string Crash = "crash"; + + /// The intervention made the bot restart in "Ferma tutto" (third forced restart within an hour). + public bool StopFileCreated { get; init; } +} + +/// Reading and writing the guard's files: each write goes to a temporary file first, so a reader never sees half of it. +public static class GuardFiles +{ + public static void WriteState(GuardPaths paths, GuardState state) + { + ArgumentNullException.ThrowIfNull(paths); + Write(paths.StateFile, JsonSerializer.Serialize(state, GuardChannel.Json)); + } + + /// The last state the engine wrote; null when there is none or it cannot be read. + public static GuardState? ReadState(GuardPaths paths) + { + ArgumentNullException.ThrowIfNull(paths); + return Read(paths.StateFile); + } + + public static string WriteNote(GuardPaths paths, GuardNote note) + { + ArgumentNullException.ThrowIfNull(paths); + ArgumentNullException.ThrowIfNull(note); + var path = Path.Combine(paths.NotesFolder, note.At.UtcDateTime.ToString("yyyyMMdd'T'HHmmssfff'Z'", CultureInfo.InvariantCulture) + ".json"); + Write(path, JsonSerializer.Serialize(note, GuardChannel.Json)); + return path; + } + + /// The notes not yet taken by the engine, oldest first, with their files. + public static IReadOnlyList<(string Path, GuardNote Note)> ReadNotes(GuardPaths paths) + { + ArgumentNullException.ThrowIfNull(paths); + if (!Directory.Exists(paths.NotesFolder)) + { + return []; + } + + var notes = new List<(string, GuardNote)>(); + foreach (var file in Directory.GetFiles(paths.NotesFolder, "*.json").Order(StringComparer.Ordinal)) + { + if (Read(file) is { } note) + { + notes.Add((file, note)); + } + } + + return notes; + } + + private static T? Read(string path) + where T : class + { + try + { + return File.Exists(path) ? JsonSerializer.Deserialize(File.ReadAllText(path), GuardChannel.Json) : null; + } + catch (Exception ex) when (ex is IOException or JsonException or UnauthorizedAccessException) + { + return null; + } + } + + private static void Write(string path, string text) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var temporary = path + ".tmp"; + File.WriteAllText(temporary, text); + File.Move(temporary, path, overwrite: true); + } +} diff --git a/src/Anubis/Supervision/Guardian.cs b/src/Anubis/Supervision/Guardian.cs new file mode 100644 index 0000000..43fea09 --- /dev/null +++ b/src/Anubis/Supervision/Guardian.cs @@ -0,0 +1,499 @@ +using Anubis.Core; +using Anubis.Trading; +using Microsoft.Extensions.Logging; + +namespace Anubis.Supervision; + +/// The times of the guard (SPEC 9.3). The drill shortens them; the bot uses these. +public sealed record GuardOptions +{ + /// Silence of the main loop, after a successful start, that makes the guard intervene. + public TimeSpan Silence { get; init; } = TimeSpan.FromMinutes(2); + + /// Startup steps that do not move for this long make the guard intervene. + public TimeSpan StalledStart { get; init; } = TimeSpan.FromMinutes(10); + + public TimeSpan Poll { get; init; } = TimeSpan.FromSeconds(1); + + /// A round of the guard this late means its own clock jumped (the computer slept): the silence counts from now. + public TimeSpan ClockJump { get; init; } = TimeSpan.FromSeconds(30); + + /// How long an orderly stop may take before the engine is ended (the container waits 60 seconds). + public TimeSpan GracefulStop { get; init; } = TimeSpan.FromSeconds(55); + + public TimeSpan ForcedWindow { get; init; } = TimeSpan.FromHours(1); + + /// The forced restart, within , that restarts the engine in "Ferma tutto". + public int ForcedLimit { get; init; } = 3; + + public TimeSpan DownAlertEvery { get; init; } = TimeSpan.FromHours(1); + + /// Pauses before restarting an engine that keeps closing by itself: the first restart is immediate. + public IReadOnlyList CrashPauses { get; init; } = + [TimeSpan.Zero, TimeSpan.FromSeconds(10), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(15)]; +} + +/// How to start the engine: with the drill's fault or without, and where its messages go. +public sealed record EngineStart(bool Fault, Action OnMessage); + +/// The engine as a child process of the guard. +public interface IEngineProcess : IDisposable +{ + /// Completes with the exit code when the process ends. + Task Exited { get; } + + /// Asks for an orderly stop through the channel. + Task RequestStopAsync(CancellationToken cancellationToken); + + /// Ends the process at once, with everything it started. + void Kill(); +} + +/// +/// The guard (SPEC 9.3): parent of the engine, it listens to its beat and intervenes when the main loop is silent for +/// 2 minutes after a successful start, or the startup steps of 9.4 do not move for 10 minutes. Then, in this order: +/// ends the engine, reads positions and orders from the brokers, cancels the purchases and sales left open, puts back +/// the missing stops, leaves a note, warns, restarts the engine. Orderly stops are not reactions: they are passed on +/// and waited for, and the engine is not restarted. At the third forced restart within an hour the engine restarts in +/// "Ferma tutto". The guard never writes in the database. +/// +public sealed partial class Guardian(GuardPaths paths, IClock clock, GuardOptions options, Func start, + Func, IBrokerAccount?> brokers, Func, CancellationToken, Task> alert, ILogger logger) +{ + private readonly Lock _gate = new(); + private readonly List _forced = []; + private IEngineProcess? _engine; + private string _stage = GuardStages.Starting; + private string? _step; + private DateTimeOffset _lastBeat; + private DateTimeOffset _stepSince; + private bool _announced; + private IReadOnlyList _keys = []; + private DateTimeOffset _lastRound; + private DateTimeOffset? _downSince; + private DateTimeOffset? _nextDownAlert; + private DateTimeOffset? _restartAt; + private int _crashes; + + /// The stage the engine last declared. + public string Stage + { + get + { + lock (_gate) + { + return _stage; + } + } + } + + public int Interventions { get; private set; } + + /// The engine has a process now (not waiting for the pause after a crash). + public bool EngineAlive => _engine is { Exited.IsCompleted: false }; + + /// The notes written so far, for the drill and the tests. + public IReadOnlyList Notes => [.. _notes]; + + private readonly List _notes = []; + + /// Starts the engine; only for the drill. + public void Start(bool fault = false) => StartEngine(fault); + + /// A message from the engine. + public void Receive(GuardMessage message) + { + ArgumentNullException.ThrowIfNull(message); + var now = clock.UtcNow; + lock (_gate) + { + switch (message.Type) + { + case GuardMessageTypes.Beat: + var stage = message.Stage ?? _stage; + if (stage != _stage || message.Step != _step) + { + _stepSince = now; + } + + _stage = stage; + _step = message.Step; + _lastBeat = now; + if (stage is GuardStages.Running or GuardStages.Waiting) + { + // The engine is back: no more "it does not restart" alerts. + _downSince = null; + _nextDownAlert = null; + _crashes = 0; + } + + break; + case GuardMessageTypes.Keys: + _keys = message.Keys ?? []; + foreach (var value in _keys.SelectMany(k => k.Values)) + { + Redactor.Register(value); + } + + break; + case GuardMessageTypes.Stopping: + _announced = true; + LogAnnounced(logger, message.Reason ?? "-"); + break; + } + } + } + + /// Watches until the engine stops on purpose, or until asks the guard to stop it. + public async Task RunAsync(CancellationToken shutdown) + { + if (_engine is null && _restartAt is null) + { + StartEngine(fault: false); + } + + while (true) + { + var exited = _engine?.Exited ?? Task.FromResult(0); + try + { + await Task.WhenAny(Task.Delay(options.Poll, shutdown), _engine is null ? Task.Delay(Timeout.Infinite, shutdown) : exited).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + + if (shutdown.IsCancellationRequested) + { + return await StopAsync().ConfigureAwait(false); + } + + if (await RoundAsync(shutdown).ConfigureAwait(false) is { } code) + { + return code; + } + } + } + + /// One round of watching; the exit code when the engine stopped on purpose and the guard's work is over. + public async Task RoundAsync(CancellationToken cancellationToken) + { + var now = clock.UtcNow; + if (_lastRound != default && now - _lastRound > options.ClockJump) + { + // The guard's own clock jumped too (the computer slept): the engine's silence starts now, not then. + lock (_gate) + { + _lastBeat = now; + _stepSince = now; + } + } + + _lastRound = now; + if (_engine is null) + { + if (_restartAt is { } at && now >= at) + { + _restartAt = null; + StartEngine(fault: false); + } + + await DownAlertAsync(now, cancellationToken).ConfigureAwait(false); + return null; + } + + if (_engine.Exited.IsCompleted) + { + var code = await _engine.Exited.ConfigureAwait(false); + bool announced; + lock (_gate) + { + announced = _announced; + } + + if (announced) + { + LogEnded(logger, code); + return code; + } + + await InterveneAsync(GuardNote.Crash, cancellationToken).ConfigureAwait(false); + return null; + } + + string stage; + string? step; + DateTimeOffset beat; + DateTimeOffset since; + lock (_gate) + { + (stage, step, beat, since) = (_stage, _step, _lastBeat, _stepSince); + } + + if (stage == GuardStages.Running && now - beat > options.Silence) + { + await InterveneAsync(GuardNote.Silence, cancellationToken).ConfigureAwait(false); + } + else if (stage == GuardStages.Starting && now - since > options.StalledStart) + { + await InterveneAsync(GuardNote.Stalled, cancellationToken, step).ConfigureAwait(false); + } + + await DownAlertAsync(now, cancellationToken).ConfigureAwait(false); + return null; + } + + /// + /// The container is stopping: the engine gets "stop" and up to to close in + /// order, then it is ended. It is not restarted. + /// + public async Task StopAsync() + { + lock (_gate) + { + _announced = true; + } + + var engine = _engine; + if (engine is null) + { + return 0; + } + + try + { + using var asking = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await engine.RequestStopAsync(asking.Token).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or OperationCanceledException or ObjectDisposedException or InvalidOperationException) + { + // The channel is gone: the wait below ends the engine anyway. + } + + if (await Task.WhenAny(engine.Exited, Task.Delay(options.GracefulStop)).ConfigureAwait(false) != engine.Exited) + { + LogKilledAtStop(logger); + engine.Kill(); + } + + return await engine.Exited.ConfigureAwait(false); + } + + private void StartEngine(bool fault) + { + _engine?.Dispose(); + var now = clock.UtcNow; + lock (_gate) + { + _stage = GuardStages.Starting; + _step = null; + _lastBeat = now; + _stepSince = now; + _announced = false; + } + + _lastRound = now; + _engine = start(new EngineStart(fault, Receive)); + LogStarted(logger, fault); + } + + private async Task InterveneAsync(string reason, CancellationToken cancellationToken, string? step = null) + { + var engine = _engine!; + var now = clock.UtcNow; + Interventions++; + LogIntervention(logger, reason); + + // 1. The engine stops first: the guard touches orders only while it is stopped. + engine.Kill(); + try + { + await engine.Exited.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None).ConfigureAwait(false); + } + catch (TimeoutException) + { + LogStillAlive(logger); + } + + engine.Dispose(); + _engine = null; + + // 2-4. Each broker account: orders left waiting canceled, missing stops put back. + var lines = new List { Opening(reason, step) }; + var placed = new List(); + IReadOnlyList keys; + lock (_gate) + { + keys = _keys; + } + + var state = GuardFiles.ReadState(paths); + if (state is null) + { + lines.Add("Non ho trovato il file di stato del motore: posizioni e stop li rimette in ordine il motore alla ripartenza."); + } + else + { + foreach (var account in state.Accounts.Where(a => a.Positions.Count > 0 || a.Orders.Count > 0)) + { + lines.AddRange(await RepairAsync(state, account, keys, placed, cancellationToken).ConfigureAwait(false)); + } + } + + // The third forced restart within an hour: the engine restarts in "Ferma tutto" (SPEC 9.3). + _forced.RemoveAll(t => now - t >= options.ForcedWindow); + _forced.Add(now); + var stopAll = _forced.Count >= options.ForcedLimit; + if (stopAll) + { + WriteStopFile(now); + lines.Add("È il terzo riavvio forzato in un'ora: il motore riparte in \"Ferma tutto\". Per ripartire togli il file STOP e conferma dall'interfaccia."); + } + + // 5. The note for the diary, then the warning. + var note = new GuardNote(now, reason, lines, placed) { StopFileCreated = stopAll }; + try + { + GuardFiles.WriteNote(paths, note); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + LogNoteFailed(logger, ex); + } + + _notes.Add(note); + await AlertAsync(string.Join(Environment.NewLine, lines), cancellationToken).ConfigureAwait(false); + + // 6. The restart, never with the drill's fault. An engine that keeps closing by itself waits a little longer each time. + _downSince ??= now; + _nextDownAlert ??= now + options.DownAlertEvery; + var pause = TimeSpan.Zero; + if (reason == GuardNote.Crash) + { + _crashes++; + pause = options.CrashPauses[Math.Min(_crashes, options.CrashPauses.Count) - 1]; + } + + if (pause <= TimeSpan.Zero) + { + StartEngine(fault: false); + } + else + { + _restartAt = now + pause; + } + } + + private async Task> RepairAsync(GuardState state, GuardAccount account, IReadOnlyList keys, List placed, CancellationToken cancellationToken) + { + if (account.Environment == TradingEnvironments.Simulation) + { + return [$"{account.Name} è il simulatore interno: posizioni e stop li rimette in ordine il motore alla ripartenza."]; + } + + IBrokerAccount? broker; + try + { + broker = brokers(account, keys); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return [$"{account.Name}: non riesco a collegarmi al conto ({Redactor.Redact(ex.Message)}): ci pensa il motore alla ripartenza."]; + } + + if (broker is null) + { + return [$"{account.Name}: non ho le chiavi del conto, quindi non posso intervenire: ci pensa il motore alla ripartenza."]; + } + + try + { + var outcome = await GuardRepair.RunAsync(state, account, broker, clock, cancellationToken).ConfigureAwait(false); + placed.AddRange(outcome.Placed); + return [.. outcome.Lines]; + } + catch (BrokerException ex) + { + return [$"{account.Name}: non raggiungo il conto ({Redactor.Redact(ex.Message)}): ci pensa il motore alla ripartenza."]; + } + finally + { + (broker as IDisposable)?.Dispose(); + } + } + + private async Task DownAlertAsync(DateTimeOffset now, CancellationToken cancellationToken) + { + if (_downSince is not { } since || _nextDownAlert is not { } next || now < next) + { + return; + } + + _nextDownAlert = next + options.DownAlertEvery; + await AlertAsync($"Il motore non riparte da {DiaryText.Duration(now - since)}. Le posizioni hanno gli stop che ho rimesso io; " + + "quelli giornalieri sulle frazioni di azione scadono a fine seduta e senza il motore nessuno li rimette domani.", cancellationToken).ConfigureAwait(false); + } + + private async Task AlertAsync(string text, CancellationToken cancellationToken) + { + IReadOnlyList keys; + lock (_gate) + { + keys = _keys; + } + + try + { + await alert(text, keys, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogAlertFailed(logger, ex); + } + } + + private void WriteStopFile(DateTimeOffset now) + { + try + { + Directory.CreateDirectory(paths.DataRoot); + File.WriteAllText(paths.StopFile, $"Creato dalla guardia il {now:yyyy-MM-dd HH:mm} UTC: terzo riavvio forzato in un'ora.{Environment.NewLine}"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + LogNoteFailed(logger, ex); + } + } + + private string Opening(string reason, string? step) => reason switch + { + GuardNote.Silence => $"La guardia è intervenuta: il motore non dava segni di vita da {DiaryText.Duration(options.Silence)}. L'ho fermato, ho controllato i conti e l'ho riavviato.", + GuardNote.Stalled => $"La guardia è intervenuta: l'avvio del motore era fermo da {DiaryText.Duration(options.StalledStart)}" + + (step is null ? "" : $" al passo \"{step}\"") + ". L'ho fermato, ho controllato i conti e l'ho riavviato.", + _ => "La guardia è intervenuta: il motore si è chiuso senza avvisare. Ho controllato i conti e l'ho riavviato.", + }; + + [LoggerMessage(Level = LogLevel.Information, Message = "Engine started (drill fault: {Fault})")] + private static partial void LogStarted(ILogger logger, bool fault); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Guard intervention: {Reason}")] + private static partial void LogIntervention(ILogger logger, string reason); + + [LoggerMessage(Level = LogLevel.Information, Message = "Engine announced its stop: {Reason}")] + private static partial void LogAnnounced(ILogger logger, string reason); + + [LoggerMessage(Level = LogLevel.Information, Message = "Engine ended with code {Code}")] + private static partial void LogEnded(ILogger logger, int code); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Engine did not stop in time: ended")] + private static partial void LogKilledAtStop(ILogger logger); + + [LoggerMessage(Level = LogLevel.Error, Message = "Engine still alive after being ended")] + private static partial void LogStillAlive(ILogger logger); + + [LoggerMessage(Level = LogLevel.Error, Message = "Guard note or STOP file not written")] + private static partial void LogNoteFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Error, Message = "Guard alert not sent")] + private static partial void LogAlertFailed(ILogger logger, Exception exception); +} diff --git a/src/Anubis/Trading/DiaryText.cs b/src/Anubis/Trading/DiaryText.cs index c1103e7..58cec0a 100644 --- a/src/Anubis/Trading/DiaryText.cs +++ b/src/Anubis/Trading/DiaryText.cs @@ -150,6 +150,11 @@ public static class DiaryText /// A missed job left out at the restart, with its reason (SPEC 5.6). public static string Skipped(string what, string reason) => $"Saltato: {what}. {reason}"; + /// A purchase or sale left open at a start (SPEC 9.4): canceled, and proposed again. + public static string RestartedOrder(bool sale, string symbol) => sale + ? $"Alla ripartenza ho annullato la vendita di {Shown(symbol)} rimasta aperta: la vendita già decisa riparte subito con un ordine nuovo." + : $"Alla ripartenza ho annullato l'acquisto di {Shown(symbol)} rimasto aperto: lo ripropongo con le regole del recupero."; + /// An entry left out because the price moved too far from the planned one (SPEC 5.6). public static string EntryDropped(string symbol, decimal movedPercent, decimal limitPercent) => $"Scartato l'acquisto di {Shown(symbol)}: il prezzo si è mosso del {Format.Percent(movedPercent, 1)} dall'orario previsto, oltre il limite del {Format.Percent(limitPercent, 0)}."; diff --git a/src/Anubis/Trading/LiveEngine.cs b/src/Anubis/Trading/LiveEngine.cs index 932aace..9f58aaf 100644 --- a/src/Anubis/Trading/LiveEngine.cs +++ b/src/Anubis/Trading/LiveEngine.cs @@ -22,6 +22,9 @@ public static class LiveReasons public const string OrderInProgress = "order_in_progress"; public const string NothingHeld = "nothing_held"; public const string Recovered = "recovered"; + + /// A purchase or sale left open at a start, canceled by the steps of SPEC 9.4. + public const string Restart = "restart"; } /// States of a planned order. @@ -911,6 +914,55 @@ public sealed partial class LiveEngine : IDisposable await CloseAsync(order, refusal is null ? PlannedStates.Submitted : PlannedStates.Refused, refusal, execution, recovered, cancellationToken).ConfigureAwait(false); } + /// + /// The step of SPEC 9.4 that comes after the reconciliation: the purchases and sales left open on the account are + /// canceled (their stops come back at once), the exits already decided restart now with a new order, and the + /// interrupted entries are proposed again with their time and reference price, so the rules of SPEC 5.6 apply. + /// The answer is how many were proposed again. + /// + public async Task ResumeAsync(OrderManager manager, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(manager); + var canceled = await manager.CancelWorkingAsync(LiveReasons.Restart, cancellationToken: cancellationToken).ConfigureAwait(false); + var again = 0; + foreach (var execution in canceled) + { + var row = await _db.ReadAsync(s => s.FirstOrDefaultAsync( + "SELECT p.id, p.decision_id, p.strategy, p.account_id, p.instrument_id, p.side, p.purpose, p.quantity, p.exit_all, p.stop_level, p.stop_offset, " + + "p.reference_price, p.reason, p.due_at, p.valid_until, e.filled FROM planned_orders p JOIN executions e ON e.id = p.execution_id WHERE p.execution_id = @e", + r => new Interrupted(new Planned(r.Id("id"), r.Id("decision_id"), r.Text("strategy"), r.Id("account_id"), r.Id("instrument_id"), Wire.Side(r.Text("side")), + Wire.Purpose(r.Text("purpose")), r.Money("quantity"), r.Bool("exit_all"), r.MoneyOrNull("stop_level"), r.MoneyOrNull("stop_offset"), r.Money("reference_price"), + r.Text("reason"), r.Moment("due_at"), r.MomentOrNull("valid_until")), r.Money("filled")), ("e", execution)), cancellationToken).ConfigureAwait(false); + if (row is not { } found || (found.Order.Quantity - found.Filled <= 0m && !found.Order.ExitAll)) + { + continue; + } + + var (order, filled) = found; + var symbol = (await _catalog.GetAsync(order.InstrumentId, cancellationToken).ConfigureAwait(false))?.Symbol ?? "?"; + var now = _clock.UtcNow; + var sale = order.Side == OrderSide.Sell; + await _db.WriteAsync(async s => + { + await s.ExecuteAsync( + "INSERT INTO planned_orders (id, decision_id, strategy, account_id, instrument_id, side, purpose, quantity, exit_all, stop_level, stop_offset, reference_price, " + + "reason, due_at, valid_until, status, outcome, execution_id, created_at, updated_at) VALUES (@id, @d, @s, @a, @i, @side, @p, @q, @x, @sl, @so, @r, @reason, " + + "@due, @until, @st, NULL, NULL, @now, @now)", cancellationToken, + ("id", Ids.NewId(_clock)), ("d", order.DecisionId), ("s", order.Strategy), ("a", order.AccountId), ("i", order.InstrumentId), ("side", Wire.Text(order.Side)), + ("p", Wire.Text(order.Purpose)), ("q", Math.Max(0m, order.Quantity - filled)), ("x", order.ExitAll), ("sl", order.StopLevel), ("so", order.StopOffset), + ("r", order.Reference), ("reason", order.Reason), ("due", sale ? now : order.Due), ("until", order.ValidUntil), ("st", PlannedStates.Pending), ("now", now)) + .ConfigureAwait(false); + await Diary.WriteAsync(s, _clock, DiaryKinds.Orders, LiveReasons.Restart, DiaryText.RestartedOrder(sale, symbol), new { planned = order.Id, execution }, + cancellationToken).ConfigureAwait(false); + }, cancellationToken).ConfigureAwait(false); + again++; + } + + return again; + } + + private sealed record Interrupted(Planned Order, decimal Filled); + private static string OrderWords(Planned order, string symbol) => (order.Side == OrderSide.Buy ? "l'acquisto di " : "la vendita di ") + DiaryText.Shown(symbol); diff --git a/src/Anubis/Trading/OrderManager.Guard.cs b/src/Anubis/Trading/OrderManager.Guard.cs new file mode 100644 index 0000000..5e38687 --- /dev/null +++ b/src/Anubis/Trading/OrderManager.Guard.cs @@ -0,0 +1,84 @@ +using Anubis.Core; +using Anubis.Supervision; + +namespace Anubis.Trading; + +/// What the order manager gives to the guard and takes back from it (SPEC 9.3). +public sealed partial class OrderManager +{ + /// + /// Called right after a sale leaves, before the turn ends. Only the guard drill sets it, to block the main loop + /// with a sale halfway (ADR 0012). + /// + public Func? AfterSale { get; init; } + + /// + /// The account as the guard needs it: every position with its stop level on the tick and the names its stops + /// take, and the orders believed open. + /// + public async Task GuardAccountAsync(CancellationToken cancellationToken) + { + var positions = new List(); + foreach (var position in await _ledger.PositionsAsync(_account.AccountId, cancellationToken).ConfigureAwait(false)) + { + if (position.Quantity <= 0m || await _instruments(position.InstrumentId, cancellationToken).ConfigureAwait(false) is not { } instrument) + { + continue; + } + + var level = position.StopPrice is { } stop ? OrderRules.StopPrice(stop, OrderRules.TickOf(instrument.Spec, stop)) : (decimal?)null; + var key = ClientOrderIds.Key(StopKey(position.InstrumentId, position.Strategy)); + var next = await NextStopSequenceAsync(position.InstrumentId, position.Strategy, key, cancellationToken).ConfigureAwait(false); + positions.Add(new GuardPosition(position.InstrumentId, instrument.Symbol, position.Strategy, position.Quantity, level, OrderRules.LotOf(instrument.Spec), + instrument.IsCrypto, key, next)); + } + + var orders = new List(); + var open = await _db.ReadAsync(s => s.QueryAsync(OrderSelect + " WHERE account_id = @a AND status IN ('created', 'submitted', 'partially_filled')", ReadOrder, + ("a", _account.AccountId)), cancellationToken).ConfigureAwait(false); + foreach (var row in open) + { + var symbol = (await _instruments(row.InstrumentId, cancellationToken).ConfigureAwait(false))?.Symbol ?? ""; + orders.Add(new GuardOrder(row.ClientOrderId, row.InstrumentId, symbol, row.Strategy, row.Side, row.Purpose, row.Quantity, row.Stop)); + } + + return new GuardAccount(_account.AccountId, _account.Name, _account.Broker, _account.Environment, _broker.Capabilities.FractionalStopsAreDayOnly, positions, orders); + } + + /// + /// A stop the guard placed while the engine was stopped becomes one of the engine's stops: recorded, then read + /// again from the broker (it may have triggered meanwhile). False when it was already known. + /// + public async Task AdoptStopAsync(PlacedStop placed, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(placed); + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (await OrderByClientIdAsync(placed.ClientOrderId, cancellationToken).ConfigureAwait(false) is not null) + { + return false; + } + + var row = new OrderRow(Ids.NewId(_clock), null, placed.InstrumentId, placed.Strategy, placed.ClientOrderId, OrderSide.Sell, OrderKind.Stop, OrderPurpose.Stop, + placed.Quantity, null, placed.StopPrice, OrderStatus.Created, 0m, null, placed.StopPrice, null, placed.TimeInForce, false, _clock.UtcNow); + await _db.WriteAsync(async s => + { + await InsertOrderAsync(s, row, cancellationToken).ConfigureAwait(false); + await s.ExecuteAsync("UPDATE orders SET broker_order_id = @b, submitted_at = @now, reason = 'placed by the guard' WHERE id = @id", cancellationToken, + ("b", placed.BrokerOrderId), ("now", _clock.UtcNow), ("id", row.Id)).ConfigureAwait(false); + }, cancellationToken).ConfigureAwait(false); + if (await _broker.GetOrderAsync(placed.ClientOrderId, placed.Symbol, cancellationToken).ConfigureAwait(false) is { } remote) + { + await ApplyRemoteAsync(row, remote, "placed by the guard", cancellationToken).ConfigureAwait(false); + await SyncFillsCoreAsync(cancellationToken).ConfigureAwait(false); + } + + return true; + } + finally + { + _gate.Release(); + } + } +} diff --git a/src/Anubis/Trading/OrderManager.cs b/src/Anubis/Trading/OrderManager.cs index 8c68e48..c8dbb5d 100644 --- a/src/Anubis/Trading/OrderManager.cs +++ b/src/Anubis/Trading/OrderManager.cs @@ -375,14 +375,15 @@ public sealed partial class OrderManager : IDisposable /// /// Cancels the executions under way, except protective sales (SPEC 9.3, "Ferma tutto"): the open order of each is - /// canceled with the broker, and the stops it had removed come back at once for the quantity held. + /// canceled with the broker, and the stops it had removed come back at once for the quantity held. The answer is + /// the executions canceled. /// - public async Task CancelWorkingAsync(string reason, Func? strategy = null, CancellationToken cancellationToken = default) + public async Task> CancelWorkingAsync(string reason, Func? strategy = null, CancellationToken cancellationToken = default) { await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { - var count = 0; + var canceled = new List(); foreach (var exec in await WorkingAsync(cancellationToken).ConfigureAwait(false)) { if (exec.Purpose == OrderPurpose.Protective || (strategy is not null && !strategy(exec.Strategy))) @@ -400,10 +401,10 @@ public sealed partial class OrderManager : IDisposable await SyncFillsCoreAsync(cancellationToken).ConfigureAwait(false); await FinishAsync(exec, ExecutionStates.Canceled, reason, instrument, cancellationToken).ConfigureAwait(false); - count++; + canceled.Add(exec.Id); } - return count; + return canceled; } finally { @@ -626,6 +627,10 @@ public sealed partial class OrderManager : IDisposable } }, cancellationToken).ConfigureAwait(false); await SendAsync(exec, row, instrument, cancellationToken).ConfigureAwait(false); + if (exec.Side == OrderSide.Sell && exec.Purpose == OrderPurpose.Exit && AfterSale is { } afterSale) + { + await afterSale(cancellationToken).ConfigureAwait(false); + } } private async Task SendAsync(ExecRow exec, OrderRow row, Instrument instrument, CancellationToken cancellationToken) diff --git a/src/Anubis/Trading/ReasonCatalog.cs b/src/Anubis/Trading/ReasonCatalog.cs index 7571e11..951a1eb 100644 --- a/src/Anubis/Trading/ReasonCatalog.cs +++ b/src/Anubis/Trading/ReasonCatalog.cs @@ -83,6 +83,7 @@ public static class ReasonCatalog ["superseded"] = "Sostituito da una decisione più recente.", ["order_in_progress"] = "Un ordine sullo stesso strumento era ancora in corso.", ["nothing_held"] = "Niente da vendere: la posizione era già chiusa.", + ["restart"] = "Annullato alla ripartenza del bot: se era ancora dovuto, è stato riproposto.", ["recovered"] = "Recuperato dopo un'assenza.", }; diff --git a/src/Anubis/Trading/TradingService.cs b/src/Anubis/Trading/TradingService.cs index 14a45bc..bd44a26 100644 --- a/src/Anubis/Trading/TradingService.cs +++ b/src/Anubis/Trading/TradingService.cs @@ -4,6 +4,7 @@ using Anubis.Core; using Anubis.Data; using Anubis.Db; using Anubis.Risk; +using Anubis.Supervision; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -192,6 +193,9 @@ public sealed partial class TradingService : IDisposable private volatile bool _started; + /// The startup steps of SPEC 9.4 are done and the loop runs. + public bool IsStarted => _started; + /// The live engine (SPEC 5.5, issue #47): the decisions at their times, the planned orders, the recovery. public LiveEngine Engine { get; } @@ -216,6 +220,23 @@ public sealed partial class TradingService : IDisposable public TimeSpan TickInterval { get; init; } = TimeSpan.FromSeconds(5); + /// The channel to the guard (SPEC 9.3): startup steps and the beat of the loop. Null without a guard. + public GuardLink? Guard { get; init; } + + /// Where the state file for the guard is written and its notes are read. Null without a guard. + public GuardPaths? GuardFiles { get; init; } + + /// The guard drill's fault (ADR 0012): or nothing. + public string? Fault { get; init; } + + /// How often the loop writes the state file for the guard. + public TimeSpan GuardStateEvery { get; init; } = TimeSpan.FromSeconds(10); + + /// A broker clock further than this from the computer's is reported at the start (SPEC 9.4). + public TimeSpan ClockTolerance { get; init; } = TimeSpan.FromSeconds(5); + + private DateTimeOffset _guardStateAt; + public IReadOnlyCollection Managers => [.. _managers.Values]; public OrderManager? ManagerOf(Guid accountId) => _managers.GetValueOrDefault(accountId); @@ -233,14 +254,21 @@ public sealed partial class TradingService : IDisposable return accounts; } - /// Accounts, brokers and order managers; then the simulator catches up, keys are checked and every account reconciled. + /// + /// The startup of SPEC 9.4 after the migrations: brokers, clock, corporate actions, then for every account the + /// complete reconciliation, the purchases and sales left open canceled (exits restart, entries are proposed again) + /// and every position checked for its stop. Each step is told to the guard. + /// public async Task StartAsync(CancellationToken cancellationToken) { + Guard?.Step("conti e strategie"); var accounts = await PrepareAsync(cancellationToken).ConfigureAwait(false); var alpaca = await AlpacaKeysAsync(cancellationToken).ConfigureAwait(false); var okx = await OkxKeysAsync(cancellationToken).ConfigureAwait(false); await RefreshSpecsAsync(cancellationToken).ConfigureAwait(false); await RefreshLimitsAsync(cancellationToken).ConfigureAwait(false); + Guard?.Step("connessione ai broker"); + var hang = Fault == GuardChannel.HangMidSale; foreach (var account in accounts) { var broker = Build(account, alpaca, okx); @@ -259,10 +287,16 @@ public sealed partial class TradingService : IDisposable Gate = Risk is null ? null : Risk.GateAsync, LivePermit = Stages.LiveRefusalAsync, PurchasePriceCheck = (limit, quote) => RiskRules.TechnicalRefusal(_clock.UtcNow, quote.At, quote.Mid, limit, null, _limits), + + // The guard drill only: the main loop stops here for good, with a sale halfway. + AfterSale = hang ? _ => Task.Delay(Timeout.Infinite, CancellationToken.None) : null, }; } await GuardAsync("simulatore", () => FundSimulatorsAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + Guard?.Step("orologio"); + await GuardAsync("orologio", () => CheckClocksAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + Guard?.Step("azioni societarie"); // SPEC 5.6: corporate actions come first, so the stops the reconciliation puts back already know about a split. if (_market is not null) @@ -271,6 +305,7 @@ public sealed partial class TradingService : IDisposable } await GuardAsync("azioni societarie nei registri", () => CorporateActionsAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + var notes = GuardFiles is null ? [] : Supervision.GuardFiles.ReadNotes(GuardFiles); foreach (var manager in _managers.Values) { await GuardAsync("avvio " + manager.Account.Name, async () => @@ -284,12 +319,104 @@ public sealed partial class TradingService : IDisposable await Reconciler.CheckKeyAsync(manager, cancellationToken).ConfigureAwait(false); } + // The stops the guard put back while the engine was stopped become the engine's own, before anything compares. + foreach (var placed in notes.SelectMany(n => n.Note.Placed).Where(p => p.AccountId == manager.Account.AccountId)) + { + await manager.AdoptStopAsync(placed, cancellationToken).ConfigureAwait(false); + } + + Guard?.Step("riconciliazione " + manager.Account.Name); await Reconciler.ReconcileAsync(manager, cancellationToken).ConfigureAwait(false); + Guard?.Step("ordini rimasti aperti " + manager.Account.Name); + await Engine.ResumeAsync(manager, cancellationToken).ConfigureAwait(false); + Guard?.Step("stop " + manager.Account.Name); + await manager.AlignStopsAsync(cancellationToken).ConfigureAwait(false); }, cancellationToken).ConfigureAwait(false); } + await GuardAsync("note della guardia", () => TakeGuardNotesAsync(notes, cancellationToken), cancellationToken).ConfigureAwait(false); await GuardAsync("patrimonio", () => PhotographAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + await GuardAsync("stato per la guardia", () => WriteGuardStateAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + await GuardAsync("chiavi per la guardia", () => SendGuardKeysAsync(alpaca, okx, cancellationToken), cancellationToken).ConfigureAwait(false); _started = true; + Guard?.Set(GuardStages.Running); + } + + /// + /// What the guard needs if the engine stops answering (SPEC 9.3): positions, stop levels and open orders of every + /// account. Written to a temporary file first, so the guard never reads half of it. + /// + public async Task WriteGuardStateAsync(CancellationToken cancellationToken) + { + if (GuardFiles is null) + { + return; + } + + var accounts = new List(); + foreach (var manager in _managers.Values) + { + accounts.Add(await manager.GuardAccountAsync(cancellationToken).ConfigureAwait(false)); + } + + Supervision.GuardFiles.WriteState(GuardFiles, new GuardState(_clock.UtcNow, InstallationCode, accounts)); + _guardStateAt = _clock.UtcNow; + } + + // The guard's notes go in the diary once their stops are adopted; then they are gone. + private async Task TakeGuardNotesAsync(IReadOnlyList<(string Path, GuardNote Note)> notes, CancellationToken cancellationToken) + { + foreach (var (path, note) in notes) + { + await _db.WriteAsync(s => Diary.WriteAsync(s, _clock, DiaryKinds.System, "guard_intervention", string.Join(" ", note.Lines), + new { at = note.At, reason = note.Reason, stops = note.Placed.Count, stopFile = note.StopFileCreated }, cancellationToken), cancellationToken).ConfigureAwait(false); + File.Delete(path); + } + } + + // The keys reach the guard in memory only: it needs them to protect the positions while the engine is stopped. + private async Task SendGuardKeysAsync(AlpacaCredentials? alpaca, OkxCredentials? okx, CancellationToken cancellationToken) + { + if (Guard is null) + { + return; + } + + var keys = new List(); + if (alpaca is not null) + { + keys.Add(new GuardKeys(Venues.Alpaca, TradingEnvironments.Demo, [alpaca.KeyId, alpaca.Secret])); + } + + if (okx is not null) + { + keys.Add(new GuardKeys(Venues.Okx, TradingEnvironments.Demo, [okx.Key, okx.Secret, okx.Passphrase])); + } + + var token = await _secrets.RevealAsync("TELEGRAM_BOT_TOKEN", cancellationToken).ConfigureAwait(false); + var chat = await _secrets.RevealAsync("TELEGRAM_CHAT_ID", cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(token) && !string.IsNullOrWhiteSpace(chat)) + { + keys.Add(new GuardKeys(GuardKeys.Telegram, "", [token, chat])); + } + + Guard.SendKeys(keys); + } + + // SPEC 9.4: the clock of each broker against the computer's. A large gap is said, not corrected: the computer's clock is the system's. + private async Task CheckClocksAsync(CancellationToken cancellationToken) + { + foreach (var manager in _managers.Values.Where(m => m.Broker is not PaperBroker)) + { + var remote = await manager.Broker.ServerTimeAsync(cancellationToken).ConfigureAwait(false); + var gap = (remote - _clock.UtcNow).Duration(); + if (gap > ClockTolerance) + { + await _db.WriteAsync(s => Diary.NotifyAsync(s, _clock, "clock_gap", + $"L'orologio di questo computer differisce di {Math.Round(gap.TotalSeconds)} secondi da quello di {manager.Account.Name}: controlla la sincronizzazione dell'ora.", + cancellationToken), cancellationToken).ConfigureAwait(false); + } + } } /// @@ -377,6 +504,12 @@ public sealed partial class TradingService : IDisposable } await TickAllAsync(cancellationToken).ConfigureAwait(false); + Guard?.Beat(); + if (GuardFiles is not null && _clock.UtcNow - _guardStateAt >= GuardStateEvery) + { + await GuardAsync("stato per la guardia", () => WriteGuardStateAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + } + await Task.Delay(TickInterval, cancellationToken).ConfigureAwait(false); } } diff --git a/tests/Anubis.Tests/GuardTests.cs b/tests/Anubis.Tests/GuardTests.cs new file mode 100644 index 0000000..036cb29 --- /dev/null +++ b/tests/Anubis.Tests/GuardTests.cs @@ -0,0 +1,414 @@ +using Anubis.Core; +using Anubis.Data; +using Anubis.Supervision; +using Anubis.Trading; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Anubis.Tests; + +/// The guard (issue #48, SPEC 9.3, 9.4): when it intervenes, what it does on the broker, and the restart. +[TestClass] +public sealed class GuardTests +{ + private static readonly CancellationToken _none = CancellationToken.None; + private static readonly TimeSpan _minute = TimeSpan.FromMinutes(1); + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task TwoMinutesOfSilenceAfterAStartMakeTheGuardInterveneAndRestartWithoutTheFault() + { + using var g = new GuardRig(fault: true); + g.Engine.Beat(GuardStages.Starting, "database"); + await g.PassAsync(TimeSpan.FromMinutes(5)).ConfigureAwait(false); + g.Engine.Beat(GuardStages.Running); + await g.PassAsync(TimeSpan.FromMinutes(10), () => g.Engine.Beat(GuardStages.Running)).ConfigureAwait(false); + await g.PassAsync(TimeSpan.FromSeconds(110)).ConfigureAwait(false); + Assert.AreEqual(0, g.Guardian.Interventions, "Five minutes of startup, a loop that beats, then 1:50 of silence: nothing yet."); + + await g.PassAsync(TimeSpan.FromSeconds(20)).ConfigureAwait(false); + Assert.AreEqual(1, g.Guardian.Interventions); + Assert.IsTrue(g.Engines[0].Killed); + Assert.HasCount(2, g.Engines); + Assert.IsTrue(g.Engines[0].Start.Fault, "The first engine had the drill's fault…"); + Assert.IsFalse(g.Engines[1].Start.Fault, "…the one the guard restarts never has it."); + Assert.Contains("non dava segni di vita da 2 minuti", g.Alerts.Single()); + var note = GuardFiles.ReadNotes(g.Paths).Single().Note; + Assert.AreEqual(GuardNote.Silence, note.Reason); + Assert.Contains("Non ho trovato il file di stato", string.Join(" ", note.Lines)); + Assert.IsFalse(File.Exists(g.Paths.StopFile)); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task TheErrorPageIsNeverAReasonToIntervene() + { + using var g = new GuardRig(); + g.Engine.Beat(GuardStages.Waiting); + await g.PassAsync(TimeSpan.FromHours(1)).ConfigureAwait(false); + Assert.AreEqual(0, g.Guardian.Interventions); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task StartupStepsThatDoNotMoveForTenMinutesMakeTheGuardIntervene() + { + using var g = new GuardRig(); + g.Engine.Beat(GuardStages.Starting, "migrazioni e identità"); + await g.PassAsync(TimeSpan.FromMinutes(9)).ConfigureAwait(false); + g.Engine.Beat(GuardStages.Starting, "riconciliazione OKX demo"); + await g.PassAsync(TimeSpan.FromMinutes(9), () => g.Engine.Beat(GuardStages.Starting, "riconciliazione OKX demo")).ConfigureAwait(false); + Assert.AreEqual(0, g.Guardian.Interventions, "A step that moves resets the count; the same step repeated does not."); + + await g.PassAsync(TimeSpan.FromMinutes(2)).ConfigureAwait(false); + Assert.AreEqual(1, g.Guardian.Interventions); + Assert.Contains("al passo \"riconciliazione OKX demo\"", g.Alerts.Single()); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task AJumpOfTheGuardsOwnClockIsNotSilence() + { + using var g = new GuardRig(); + g.Engine.Beat(GuardStages.Running); + await g.PassAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); + + // The computer slept for a quarter of an hour: guard and engine both stopped. + g.Clock.Advance(TimeSpan.FromMinutes(15)); + await g.Guardian.RoundAsync(_none).ConfigureAwait(false); + await g.PassAsync(TimeSpan.FromSeconds(110)).ConfigureAwait(false); + Assert.AreEqual(0, g.Guardian.Interventions); + + await g.PassAsync(TimeSpan.FromSeconds(20)).ConfigureAwait(false); + Assert.AreEqual(1, g.Guardian.Interventions, "After the jump the silence counts again from the wake-up."); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task TheContainersStopIsPassedOnAndNeverRestarted() + { + using var g = new GuardRig(); + g.Engine.Beat(GuardStages.Running); + Assert.AreEqual(0, await g.Guardian.StopAsync().ConfigureAwait(false)); + Assert.IsTrue(g.Engine.StopAsked); + Assert.IsFalse(g.Engine.Killed); + Assert.HasCount(1, g.Engines); + + // An engine that does not close in time is ended. + using var slow = new GuardRig(options: new GuardOptions { GracefulStop = TimeSpan.FromMilliseconds(200) }); + slow.Engine.ExitOnStop = false; + Assert.AreEqual(137, await slow.Guardian.StopAsync().ConfigureAwait(false)); + Assert.IsTrue(slow.Engine.Killed); + Assert.AreEqual(0, slow.Guardian.Interventions); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task AStopTheEngineAnnouncesEndsTheGuardWithoutARestart() + { + using var g = new GuardRig(); + g.Engine.Beat(GuardStages.Running); + g.Engine.Start.OnMessage(new GuardMessage(GuardMessageTypes.Stopping) { Reason = "uscita" }); + g.Engine.Exit(0); + Assert.AreEqual(0, await g.Guardian.RoundAsync(_none).ConfigureAwait(false)); + Assert.HasCount(1, g.Engines); + Assert.AreEqual(0, g.Guardian.Interventions); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task TheThirdForcedRestartWithinAnHourRestartsInStopEverything() + { + using var g = new GuardRig(); + for (var i = 1; i <= 3; i++) + { + g.Engine.Beat(GuardStages.Running); + await g.PassAsync(TimeSpan.FromMinutes(3)).ConfigureAwait(false); + Assert.AreEqual(i, g.Guardian.Interventions); + Assert.AreEqual(i == 3, File.Exists(g.Paths.StopFile), $"After intervention {i}."); + } + + var last = GuardFiles.ReadNotes(g.Paths)[^1].Note; + Assert.IsTrue(last.StopFileCreated); + Assert.Contains("\"Ferma tutto\"", g.Alerts[^1]); + + // Spread over more than an hour, three are not three within an hour. + using var spread = new GuardRig(); + for (var i = 0; i < 3; i++) + { + spread.Engine.Beat(GuardStages.Running); + await spread.PassAsync(TimeSpan.FromMinutes(3)).ConfigureAwait(false); + spread.Engine.Beat(GuardStages.Running); + await spread.PassAsync(TimeSpan.FromMinutes(30), () => spread.Engine.Beat(GuardStages.Running)).ConfigureAwait(false); + } + + Assert.AreEqual(3, spread.Guardian.Interventions); + Assert.IsFalse(File.Exists(spread.Paths.StopFile)); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task AnEngineThatKeepsClosingIsRestartedWithPausesAndTheUserIsRemindedEveryHour() + { + using var g = new GuardRig(options: new GuardOptions { StalledStart = TimeSpan.FromHours(5) }); + g.Engine.Exit(1); + await g.Guardian.RoundAsync(_none).ConfigureAwait(false); + Assert.HasCount(2, g.Engines, "The first restart is immediate."); + Assert.Contains("si è chiuso senza avvisare", g.Alerts[0]); + + g.Engine.Exit(1); + await g.Guardian.RoundAsync(_none).ConfigureAwait(false); + Assert.HasCount(2, g.Engines, "The second waits ten seconds."); + await g.PassAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); + Assert.HasCount(3, g.Engines); + + // It starts, but never gets to run: every hour a reminder about the stops on fractions of shares. + await g.PassAsync(TimeSpan.FromMinutes(61)).ConfigureAwait(false); + var reminder = g.Alerts.Single(a => a.Contains("non riparte", StringComparison.Ordinal)); + Assert.Contains("frazioni di azione scadono a fine seduta", reminder); + g.Engine.Beat(GuardStages.Running); + await g.PassAsync(TimeSpan.FromMinutes(70), () => g.Engine.Beat(GuardStages.Running)).ConfigureAwait(false); + Assert.AreEqual(1, g.Alerts.Count(a => a.Contains("non riparte", StringComparison.Ordinal)), "Once it runs, the reminders stop."); + } + + /// + /// The acceptance test of issue #48 on a broker account: the main loop blocks right after a sale leaves, with the + /// stop already removed. The guard stops the engine, cancels the sale, puts the stop back for the quantity held and + /// restarts the engine, which adopts the guard's stop and finds the account in order. + /// + [TestMethod] + [TestCategory(Category.Simulation)] + public async Task ALoopBlockedWithASaleHalfwayGetsItsStopBackAndTheEngineRestarts() + { + await using var rig = await TradingRig.CreateAsync().ConfigureAwait(false); + var account = await rig.AccountAsync(Venues.Okx, TradingEnvironments.Demo).ConfigureAwait(false); + var btc = await rig.InstrumentAsync(TradingRig.Btc).ConfigureAwait(false); + var paper = rig.Paper(PaperOptions.Okx(), account.AccountId); + await paper.DepositAsync(100_000m, "EUR", _none).ConfigureAwait(false); + await rig.Ledger.RecordCapitalAsync(account.AccountId, StrategyNames.CryptoTrend, 100_000m, "EUR", "test", _none).ConfigureAwait(false); + rig.Quote(btc.Symbol, 60000m, 60010m); + + // The demo account at the broker, which the guard reaches on its own. + var remote = new FaultyBroker(paper, TradingEnvironments.Demo); + using var hang = new CancellationTokenSource(); + var sold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + OrderManager Engine(Func? afterSale) => new(rig.Db, rig.Clock, rig.Ledger, remote, account, TradingRig.Installation, rig.FindAsync, rig.QuoteAsync) + { + ConfirmPrice = rig.QuoteAsync, + MarketOpen = (_, _) => Task.FromResult(true), + AfterSale = afterSale, + }; + + using var first = Engine(async _ => + { + sold.TrySetResult(); + await Task.Delay(Timeout.Infinite, hang.Token).ConfigureAwait(false); + }); + await first.SubmitAsync(new TradeIntent(btc.Id, StrategyNames.CryptoTrend, OrderSide.Buy, 0.01m, OrderPurpose.Entry, Ids.NewId(rig.Clock)) + { + StopLevel = 57000m, + StartAggressive = true, + }, _none).ConfigureAwait(false); + await rig.TurnsAsync(first, 3, _minute).ConfigureAwait(false); + var held = (await rig.Ledger.PositionAsync(account.AccountId, btc.Id, StrategyNames.CryptoTrend, _none).ConfigureAwait(false))!.Quantity; + Assert.AreEqual(held, (await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false)).Sum(o => o.Quantity)); + + var folder = Directory.CreateTempSubdirectory("anubis-guard-").FullName; + try + { + var paths = new GuardPaths(folder); + GuardFiles.WriteState(paths, new GuardState(rig.Clock.UtcNow, TradingRig.Installation, [await first.GuardAccountAsync(_none).ConfigureAwait(false)])); + + // The sale: the stop goes, the order leaves, and the loop never comes back. + await first.SubmitAsync(new TradeIntent(btc.Id, StrategyNames.CryptoTrend, OrderSide.Sell, held, OrderPurpose.Exit, Ids.NewId(rig.Clock)), _none).ConfigureAwait(false); + var blocked = Task.Run(() => first.TickAsync(hang.Token)); + await sold.Task.WaitAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); + Assert.IsEmpty(await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false), "Halfway: the stop is gone…"); + Assert.HasCount(1, await OpenAsync(remote, OrderPurpose.Exit).ConfigureAwait(false), "…and the sale waits on the book."); + + // The guard: two minutes of silence, then it works on the account with the keys it was given. + var engines = new List(); + var guardian = new Guardian(paths, rig.Clock, new GuardOptions(), s => + { + var engine = new FakeEngine(s); + engines.Add(engine); + return engine; + }, (a, keys) => a.AccountId == account.AccountId && keys.Count > 0 ? remote : null, (_, _, _) => Task.CompletedTask, NullLogger.Instance); + guardian.Start(); + engines[0].Start.OnMessage(new GuardMessage(GuardMessageTypes.Keys) { Keys = [new GuardKeys(Venues.Okx, TradingEnvironments.Demo, ["k", "s", "p"])] }); + engines[0].Beat(GuardStages.Running); + for (var i = 0; i < 13 && guardian.Interventions == 0; i++) + { + rig.Clock.Advance(TimeSpan.FromSeconds(10)); + await guardian.RoundAsync(_none).ConfigureAwait(false); + } + + Assert.AreEqual(1, guardian.Interventions); + Assert.IsTrue(engines[0].Killed); + Assert.HasCount(2, engines, "The guard restarted the engine."); + Assert.IsEmpty(await OpenAsync(remote, OrderPurpose.Exit).ConfigureAwait(false), "The sale left waiting is canceled."); + var stops = await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false); + Assert.AreEqual((held, 57000m), (stops.Sum(o => o.Quantity), stops.Single().StopPrice), "The stop is back for the whole quantity."); + var note = GuardFiles.ReadNotes(paths).Single().Note; + Assert.AreEqual(stops.Single().ClientOrderId, note.Placed.Single().ClientOrderId); + Assert.IsTrue(note.Lines.Any(l => l.Contains("Ho annullato la vendita di BTC", StringComparison.Ordinal)), string.Join(" | ", note.Lines)); + + // The engine starts again (SPEC 9.4): it adopts the guard's stop, reconciles, cancels what it had left open. + using var second = Engine(null); + using var live = new LiveEngine(rig.Db, rig.Clock, rig.Ledger, new StageBook(rig.Db, rig.Clock, TradingConsent.None, (_, _, _) => Task.FromResult(false)), + _ => second, rig.QuoteAsync); + Assert.IsTrue(await second.AdoptStopAsync(note.Placed.Single(), _none).ConfigureAwait(false)); + var report = await new Reconciler(rig.Db, rig.Clock, rig.Ledger).ReconcileAsync(second, _none).ConfigureAwait(false); + await live.ResumeAsync(second, _none).ConfigureAwait(false); + await second.AlignStopsAsync(_none).ConfigureAwait(false); + engines[1].Beat(GuardStages.Running); + await rig.TurnsAsync(second, 2, _minute).ConfigureAwait(false); + + // The only thing the engine finds changed is the sale the guard canceled; nothing is left different. + Assert.IsEmpty(report.Differences, string.Join(" | ", report.Differences.Select(d => d.Message))); + Assert.AreEqual(OrderPurpose.Exit, ClientOrderIds.Parse(report.Healed.Single().Split(':')[1])?.Purpose, report.Healed.Single()); + Assert.AreEqual(GuardStages.Running, guardian.Stage); + var after = await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false); + Assert.AreEqual(stops.Single().ClientOrderId, after.Single().ClientOrderId, "The guard's stop stays: no second stop on the same quantity."); + var sale = await rig.QueryAsync("SELECT status, reason FROM executions WHERE side = 'sell'", r => r.Text("status") + "/" + r.TextOrNull("reason")).ConfigureAwait(false); + Assert.AreEqual("canceled/" + LiveReasons.Restart, sale.Single()); + var adopted = await rig.QueryAsync("SELECT status, reason FROM orders WHERE client_order_id = @c", r => r.Text("status") + "/" + r.TextOrNull("reason"), + ("c", after.Single().ClientOrderId)).ConfigureAwait(false); + Assert.AreEqual("submitted/placed by the guard", adopted.Single()); + Assert.AreEqual((held, 57000m), await rig.Ledger.PositionAsync(account.AccountId, btc.Id, StrategyNames.CryptoTrend, _none).ConfigureAwait(false) is { } p + ? (p.Quantity, p.StopPrice!.Value) : default); + + await hang.CancelAsync().ConfigureAwait(false); + await Assert.ThrowsAsync(() => blocked).ConfigureAwait(false); + } + finally + { + Directory.Delete(folder, recursive: true); + } + } + + /// + /// The same proof with real processes (the drill that also runs in the Docker image): the real guard, the real + /// engine as its child, a sale blocked halfway on the internal simulator, then the restart by SPEC 9.4. + /// + [TestMethod] + [TestCategory(Category.Integration)] + [Timeout(600_000, CooperativeCancellation = true)] + public async Task TheGuardDrillPassesWithTheRealEngineAsChildProcess() + { + if (TestDatabase.Engine == "postgres") + { + Assert.Inconclusive("The drill has its own SQLite database: it runs in the SQLite pass."); + } + + var folder = Directory.CreateTempSubdirectory("anubis-drill-").FullName; + try + { + var result = await GuardDrill.RunInAsync(folder, Path.Combine(AppContext.BaseDirectory, "Anubis.Server.dll"), GuardDrill.Options, + line => Console.WriteLine(line), _none).ConfigureAwait(false); + Assert.IsTrue(result.Passed, string.Join(Environment.NewLine, result.Lines)); + } + finally + { + try + { + Directory.Delete(folder, recursive: true); + } + catch (IOException) + { + } + } + } + + private static async Task> OpenAsync(FaultyBroker broker, OrderPurpose purpose) => + [.. (await broker.GetOpenOrdersAsync(_none).ConfigureAwait(false)).Where(o => ClientOrderIds.Parse(o.ClientOrderId)?.Purpose == purpose)]; + + /// The engine process as the guard sees it, driven by the test. + private sealed class FakeEngine(EngineStart start) : IEngineProcess + { + private readonly TaskCompletionSource _exit = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public EngineStart Start { get; } = start; + + public Task Exited => _exit.Task; + + public bool StopAsked { get; private set; } + + public bool Killed { get; private set; } + + public bool ExitOnStop { get; set; } = true; + + public Task RequestStopAsync(CancellationToken cancellationToken) + { + StopAsked = true; + if (ExitOnStop) + { + Start.OnMessage(new GuardMessage(GuardMessageTypes.Stopping) { Reason = "uscita" }); + _exit.TrySetResult(0); + } + + return Task.CompletedTask; + } + + public void Kill() + { + Killed = true; + _exit.TrySetResult(137); + } + + public void Exit(int code) => _exit.TrySetResult(code); + + public void Beat(string stage, string? step = null) => Start.OnMessage(GuardMessage.Beat(stage, step)); + + public void Dispose() + { + } + } + + /// A guard on a manual clock with fake engines, a data folder of its own and the alerts it sends. + private sealed class GuardRig : IDisposable + { + public GuardRig(bool fault = false, GuardOptions? options = null) + { + Folder = Directory.CreateTempSubdirectory("anubis-guard-").FullName; + Paths = new GuardPaths(Folder); + Guardian = new Guardian(Paths, Clock, options ?? new GuardOptions(), s => + { + var engine = new FakeEngine(s); + Engines.Add(engine); + return engine; + }, (_, _) => null, (text, _, _) => + { + Alerts.Add(text); + return Task.CompletedTask; + }, NullLogger.Instance); + Guardian.Start(fault); + } + + public ManualClock Clock { get; } = new(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero)); + + public string Folder { get; } + + public GuardPaths Paths { get; } + + public Guardian Guardian { get; } + + public List Engines { get; } = []; + + public List Alerts { get; } = []; + + public FakeEngine Engine => Engines[^1]; + + /// Time passes in rounds of ten seconds; runs before every round. + public async Task PassAsync(TimeSpan span, Action? each = null) + { + for (var t = TimeSpan.Zero; t < span; t += TimeSpan.FromSeconds(10)) + { + Clock.Advance(TimeSpan.FromSeconds(10)); + each?.Invoke(); + await Guardian.RoundAsync(_none).ConfigureAwait(false); + } + } + + public void Dispose() => Directory.Delete(Folder, recursive: true); + } +} diff --git a/tools/Anubis.Tasks/Program.cs b/tools/Anubis.Tasks/Program.cs index 2393bb6..e981ce2 100644 --- a/tools/Anubis.Tasks/Program.cs +++ b/tools/Anubis.Tasks/Program.cs @@ -39,7 +39,7 @@ int Test(string engine, string[] extra) ["test", "Anubis.slnx", "--filter", "TestCategory!=LiveDemo", "--logger", "console;verbosity=normal", .. extra], env) ?? 0; } -// What the remote CI does, run locally: check on SQLite, tests on PostgreSQL, image build. +// What the remote CI does, run locally: check on SQLite, tests on PostgreSQL, image build, the guard drill in the image. int Ci() { var failed = Check(); @@ -70,9 +70,14 @@ int Ci() } } - return failed != 0 ? failed : BuildImage("ci"); + failed = failed != 0 ? failed : BuildImage("ci"); + return failed != 0 ? failed : GuardDrill("ci"); } +// The guard drill inside the image (issue #48): the real guard and the real engine, a sale blocked halfway, the restart. +int GuardDrill(string tag) => + Step("Esercitazione della guardia nell'immagine", "docker", ["run", "--rm", $"gitea.encke-hake.ts.net/balbosoft/anubis:{tag}", "drill"]) ?? 0; + int BuildImage(string tag) => Step("Costruzione dell'immagine Docker", "docker", ["build", "-f", "deploy/docker/Dockerfile", "-t", $"gitea.encke-hake.ts.net/balbosoft/anubis:{tag}", "."]) ?? 0;