diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 96bc28f..5e424b3 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -63,3 +63,6 @@ jobs: git checkout -q FETCH_HEAD - name: Build image (no publish) run: docker build -f deploy/docker/Dockerfile -t anubis:ci . + # The guard drill inside the image (issue #48): offline, its own SQLite database in a temporary folder. + - name: Guard drill in the image + run: docker run --rm anubis:ci drill diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index f1bc33a..4dcff4e 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -6,14 +6,19 @@ WORKDIR /src COPY global.json Directory.Build.props Directory.Packages.props ./ COPY src/Anubis/Anubis.csproj src/Anubis/packages.lock.json src/Anubis/ COPY src/Anubis.Server/Anubis.Server.csproj src/Anubis.Server/packages.lock.json src/Anubis.Server/ -RUN dotnet restore src/Anubis.Server/Anubis.Server.csproj --locked-mode +COPY src/Anubis.Guard/Anubis.Guard.csproj src/Anubis.Guard/packages.lock.json src/Anubis.Guard/ +RUN dotnet restore src/Anubis.Server/Anubis.Server.csproj --locked-mode && \ + dotnet restore src/Anubis.Guard/Anubis.Guard.csproj --locked-mode COPY src/Anubis/ src/Anubis/ COPY src/Anubis.Server/ src/Anubis.Server/ +COPY src/Anubis.Guard/ src/Anubis.Guard/ # The logo is embedded from docs/design/logo, its only source (ADR 0005). COPY docs/design/logo/ docs/design/logo/ # The backtest outcomes are read at the first start (SPEC 12.3); .dockerignore lets only their JSON files through. COPY docs/backtests/ docs/backtests/ RUN dotnet publish src/Anubis.Server/Anubis.Server.csproj -c Release --no-restore -o /app +# The guard (SPEC 9.3, ADR 0012) is the main process and starts the engine next to it, from the same folder. +RUN dotnet publish src/Anubis.Guard/Anubis.Guard.csproj -c Release --no-restore -o /app FROM mcr.microsoft.com/dotnet/aspnet:10.0 WORKDIR /app @@ -27,7 +32,8 @@ ENV ANUBIS_PLATFORM=docker \ PGID=100 EXPOSE 8787 VOLUME /data -# Starts as root only to give /data to PUID:PGID, then runs the app as that user (entrypoint.sh). +# Starts as root only to give /data to PUID:PGID, then runs the guard as that user (entrypoint.sh); the guard starts +# the engine. "docker run --rm drill" runs the guard drill instead (issue #48). # Health through bash's /dev/tcp: no curl or wget in the image. 503 (database down) counts as unhealthy. HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ CMD ["bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/${PORT:-8787} && printf 'GET /health HTTP/1.0\r\nHost: localhost\r\n\r\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"] diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh index fb91c16..ec68e96 100644 --- a/deploy/docker/entrypoint.sh +++ b/deploy/docker/entrypoint.sh @@ -1,9 +1,10 @@ #!/bin/sh # Container start (SPEC 15.2, issue #16). As root: prepare /data for PUID:PGID (99:100 on Unraid), then drop to that # user with setpriv, which is part of the base image (no gosu). Started as another user (docker run --user): go straight on. +# The process is the guard (SPEC 9.3, ADR 0012): it starts the engine, watches it and passes the stop on to it. set -eu -app="/app/Anubis.Server.dll" +app="/app/Anubis.Guard.dll" if [ "$(id -u)" != "0" ]; then exec dotnet "$app" "$@" @@ -19,7 +20,7 @@ case "$uid$gid" in esac mkdir -p /data -for folder in db models research import backups logs web-keys; do +for folder in db models research import backups logs web-keys guard; do mkdir -p "/data/$folder" done # Only what does not already belong to PUID:PGID, so a large data folder does not slow every start. diff --git a/docker-compose.yml b/docker-compose.yml index db8b85a..f78ab76 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,8 @@ services: context: . dockerfile: deploy/docker/Dockerfile image: gitea.encke-hake.ts.net/balbosoft/anubis:dev + # The guard gives the engine up to 55 seconds to close in order (SPEC 9.3). + stop_grace_period: 60s ports: - "8787:8787" environment: diff --git a/src/Anubis.Guard/Anubis.Guard.csproj b/src/Anubis.Guard/Anubis.Guard.csproj index c231f26..39dcc68 100644 --- a/src/Anubis.Guard/Anubis.Guard.csproj +++ b/src/Anubis.Guard/Anubis.Guard.csproj @@ -4,6 +4,8 @@ Anubis.Guard + + diff --git a/src/Anubis.Guard/Program.cs b/src/Anubis.Guard/Program.cs index 4dfc043..445c81b 100644 --- a/src/Anubis.Guard/Program.cs +++ b/src/Anubis.Guard/Program.cs @@ -1,3 +1,49 @@ -// Placeholder: the guard process (SPEC 9.3) arrives in phase 6 (ADR 0003). -await Console.Error.WriteLineAsync("Anubis.Guard non è ancora disponibile: arriva nella Fase 6.").ConfigureAwait(false); -return 2; +using System.Collections; +using System.Runtime.InteropServices; +using Anubis.Core; +using Anubis.Platform; +using Anubis.Supervision; + +// The guard (SPEC 9.3, ADR 0012): the main process of the container and the parent of the engine. Its arguments go to +// the engine as they are. "drill" runs the guard drill in a temporary folder instead. +if (args is ["drill", ..]) +{ + return await GuardDrill.RunAsync(GuardLaunch.Server(), Console.Out, CancellationToken.None).ConfigureAwait(false); +} + +var environment = Environment.GetEnvironmentVariables().Cast() + .ToDictionary(e => (string)e.Key, e => e.Value as string, StringComparer.Ordinal); +var paths = new GuardPaths(DataRoot(args, environment)); +var logger = new LineLogger(Console.Out.WriteLine); +using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(20) }; +var clock = SystemClock.Instance; +var guardian = new Guardian(paths, clock, new GuardOptions(), + start => ChildEngine.Start(GuardLaunch.Dotnet(), [GuardLaunch.Server(), .. args], environment, start), + (account, keys) => GuardBrokers.Create(http, clock, account, keys), + (text, keys, ct) => GuardBrokers.AlertAsync(http, logger, keys, text, ct), + logger); + +// The stop of the container (SIGTERM) or Ctrl+C: passed on to the engine, which closes in order and is not restarted. +using var shutdown = new CancellationTokenSource(); +using var terminate = PosixSignalRegistration.Create(PosixSignal.SIGTERM, Stop); +using var interrupt = PosixSignalRegistration.Create(PosixSignal.SIGINT, Stop); +return await guardian.RunAsync(shutdown.Token).ConfigureAwait(false); + +void Stop(PosixSignalContext context) +{ + context.Cancel = true; + shutdown.Cancel(); +} + +// The engine's data folder: --data-dir when given, /data in Docker, the user's folder on desktop (SPEC 15.2, 15.6). +static string DataRoot(string[] args, Dictionary environment) +{ + var index = Array.IndexOf(args, "--data-dir"); + if (index >= 0 && index + 1 < args.Length) + { + return Path.GetFullPath(args[index + 1]); + } + + var platform = Array.IndexOf(args, "--platform") is var p and >= 0 && p + 1 < args.Length ? args[p + 1] : environment.GetValueOrDefault("ANUBIS_PLATFORM"); + return platform == "docker" ? "/data" : DataDirectory.ForCurrentUser().Root; +} diff --git a/src/Anubis.Server/AnubisHost.cs b/src/Anubis.Server/AnubisHost.cs index cc5be12..88144f8 100644 --- a/src/Anubis.Server/AnubisHost.cs +++ b/src/Anubis.Server/AnubisHost.cs @@ -5,6 +5,7 @@ using Anubis.Data; using Anubis.Db; using Anubis.Demo; using Anubis.Platform; +using Anubis.Supervision; using Anubis.Trading; namespace Anubis.Server; @@ -72,6 +73,9 @@ internal sealed partial class AnubisHost( public StartupConfig Config { get; private set; } = initialConfig; + /// The channel to the guard that started this engine (SPEC 9.3); null without one. + public GuardLink? Guard { get; init; } + public bool IsDemo => options.Demo; public HostStatus Status => _status; @@ -176,6 +180,12 @@ internal sealed partial class AnubisHost( SetStatus(HostMode.DatabaseUnavailable, Redactor.Redact(ex.Message)); } + // With trading the beat of a running engine comes from the trading loop, so a blocked loop is noticed. + if (_status.Mode != HostMode.Ready || _trading is null) + { + Guard?.Beat(); + } + await Task.WhenAny(wake, Task.Delay(_interval, stoppingToken)).ConfigureAwait(false); } @@ -213,6 +223,7 @@ internal sealed partial class AnubisHost( } Config.Data.Ensure(); + Guard?.Step("database"); // A data folder that belongs to an installation never gets a new empty SQLite file by itself (trap 20). _db ??= Database.Open(Config.Db!, allowCreate: !File.Exists(Config.Data.InstallationFile)); @@ -234,6 +245,8 @@ internal sealed partial class AnubisHost( return; } + Guard?.Step("migrazioni e identità"); + // Identity first, migration after: a database that belongs elsewhere is never written to, and an empty one stays // empty, so a backup taken with an older schema can still be restored into it. var backups = new Backups(Config.Data, clock); @@ -271,6 +284,7 @@ internal sealed partial class AnubisHost( return; } + Guard?.Step("servizi"); var key = Config.SecretKey.GetOrCreateKey(); var settings = new SettingsStore(_db, clock, bus, StartupOverrides()); var secrets = new SecretStore(_db, clock, key, Config.EnvironmentKeys); @@ -288,12 +302,20 @@ internal sealed partial class AnubisHost( var scheduler = new Scheduler(clock, bus); _market = MarketDataEnabled - ? new MarketDataService(_db, clock, bus, settings, secrets, Config.Data, overrides?.MarketHttp, overrides?.MarketSockets, logger) + ? new MarketDataService(_db, clock, bus, settings, secrets, Config.Data, overrides?.MarketHttp, overrides?.MarketSockets, logger, + Guard is null ? null : GuardDrill.DrillOkx(environment)) : null; _trading = TradingEnabled && (identity.File ?? Installation.ReadFile(Config.Data)) is { } file ? new TradingService(_db, clock, bus, secrets, _market, new FxStore(_db, clock), TradingConsent.From(Config), file.InstallationId, overrides?.MarketHttp, overrides?.MarketSockets, logger, settings, (password, code, ct) => access.ReauthenticateAsync(password, code, ct), Path.Combine(Config.Data.Root, "STOP")) + { + Guard = Guard, + GuardFiles = Guard is null ? null : new GuardPaths(Config.Data.Root), + + // The drill's fault is honored only under a guard, which never passes it to an engine it restarts. + Fault = Guard is null ? null : environment.GetValueOrDefault(GuardChannel.FaultVariable), + } : null; if (_trading is not null) { @@ -610,6 +632,19 @@ internal sealed partial class AnubisHost( var attempts = previous.Mode == mode ? previous.Attempts + 1 : 1; var since = previous.Mode == mode ? previous.Since : clock.UtcNow; _status = new HostStatus(mode, detail, identity, since, attempts); + if (mode == HostMode.Ready) + { + // Trading tells the guard by itself when its own startup steps are done. + if (_trading is null || _trading.IsStarted) + { + Guard?.Set(GuardStages.Running); + } + } + else + { + Guard?.Set(mode == HostMode.Starting ? GuardStages.Starting : GuardStages.Waiting); + } + if (previous.Mode != mode) { LogMode(logger, mode); diff --git a/src/Anubis.Server/ServerApp.cs b/src/Anubis.Server/ServerApp.cs index 413755f..d03cff5 100644 --- a/src/Anubis.Server/ServerApp.cs +++ b/src/Anubis.Server/ServerApp.cs @@ -4,6 +4,7 @@ using Anubis.Core; using Anubis.Demo; using Anubis.Platform; using Anubis.Server.Components; +using Anubis.Supervision; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.DataProtection; @@ -52,8 +53,12 @@ internal static class ServerApp services.AddSingleton(clock); services.AddSingleton(overrides ?? new ServerOverrides()); services.AddSingleton(); + // Started by the guard (SPEC 9.3): the channel to it. Never in demo mode, which protects nothing. services.AddSingleton(sp => new AnubisHost(options, config, environment, clock, sp.GetRequiredService(), - sp.GetRequiredService>(), overrides)); + sp.GetRequiredService>(), overrides) + { + Guard = options.Demo ? null : GuardLink.FromEnvironment(environment, sp.GetRequiredService>()), + }); services.AddHostedService(sp => sp.GetRequiredService()); services.AddSingleton(new LoginLimiter(clock)); services.AddSingleton(); @@ -82,6 +87,13 @@ internal static class ServerApp var app = builder.Build(); var host = app.Services.GetRequiredService(); + if (host.Guard is { } guard) + { + // The guard asks for an orderly stop when the container stops; every stop of the engine is announced to it. + guard.StopRequested = app.Lifetime.StopApplication; + app.Lifetime.ApplicationStopping.Register(() => guard.AnnounceAsync("uscita").GetAwaiter().GetResult()); + app.Lifetime.ApplicationStopped.Register(() => guard.DisposeAsync().AsTask().GetAwaiter().GetResult()); + } app.Use(SecurityHeaders); app.Use((context, next) => LocalOnly(context, next, options, config)); diff --git a/src/Anubis/Data/MarketDataService.cs b/src/Anubis/Data/MarketDataService.cs index 8430066..16883dc 100644 --- a/src/Anubis/Data/MarketDataService.cs +++ b/src/Anubis/Data/MarketDataService.cs @@ -54,7 +54,7 @@ public sealed partial class MarketDataService : IDisposable private (string Key, string Secret, AlpacaClient Client)? _alpaca; public MarketDataService(Database db, IClock clock, EventBus bus, SettingsStore settings, SecretStore secrets, DataDirectory? data, - HttpMessageHandler? handler = null, Func? sockets = null, ILogger? logger = null) + HttpMessageHandler? handler = null, Func? sockets = null, ILogger? logger = null, OkxOptions? okx = null) { _db = db; _clock = clock; @@ -73,7 +73,7 @@ public sealed partial class MarketDataService : IDisposable _http.Timeout = TimeSpan.FromSeconds(60); _http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}"); - Okx = new OkxClient(_http, clock, null, sockets); + Okx = new OkxClient(_http, clock, okx, sockets); Freshness = new FreshnessMonitor(clock, bus); References = new ReferenceSources(new ApiHttp(_http, clock, "reference"), db, clock, settings, secrets.RevealAsync, Okx); UsCalendar = new StoredMarketCalendar(db, clock, "us_equity"); diff --git a/src/Anubis/Notifications/TelegramClient.cs b/src/Anubis/Notifications/TelegramClient.cs new file mode 100644 index 0000000..469d859 --- /dev/null +++ b/src/Anubis/Notifications/TelegramClient.cs @@ -0,0 +1,117 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Anubis.Core; + +namespace Anubis.Notifications; + +/// A refusal or failure of the Telegram Bot API. : the network, a 429 or a 5xx. +public sealed class TelegramException : Exception +{ + public TelegramException() + { + } + + public TelegramException(string message) + : base(message) + { + } + + public TelegramException(string message, Exception innerException) + : base(message, innerException) + { + } + + public TelegramException(string message, bool retryable, TimeSpan? retryAfter = null) + : base(message) + { + Retryable = retryable; + RetryAfter = retryAfter; + } + + public bool Retryable { get; } + + /// The wait Telegram asks for after a 429. + public TimeSpan? RetryAfter { get; } +} + +/// +/// The Telegram Bot API (SPEC 14): plain text messages to the user's chat. The token is part of the address, so the +/// address is never logged; errors carry only Telegram's own description. +/// +public sealed class TelegramClient +{ + public static readonly Uri DefaultBase = new("https://api.telegram.org/"); + + private readonly HttpClient _http; + private readonly string _token; + private readonly Uri _base; + + public TelegramClient(HttpClient http, string token, Uri? baseUrl = null) + { + _http = http ?? throw new ArgumentNullException(nameof(http)); + _token = token ?? throw new ArgumentNullException(nameof(token)); + _base = baseUrl ?? DefaultBase; + Redactor.Register(token); + } + + /// Telegram's limit for one message; longer texts are cut with an ellipsis. + public const int MaxLength = 4096; + + public async Task SendAsync(string chatId, string text, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(text); + if (text.Length > MaxLength) + { + text = text[..(MaxLength - 1)] + "…"; + } + + using var answer = await CallAsync("sendMessage", new Dictionary { ["chat_id"] = chatId, ["text"] = text, ["disable_web_page_preview"] = true }, + cancellationToken).ConfigureAwait(false); + } + + /// One call of the Bot API; the answer's result when Telegram says ok. + public async Task CallAsync(string method, object body, CancellationToken cancellationToken) + { + HttpResponseMessage response; + try + { + response = await _http.PostAsJsonAsync(new Uri(_base, $"bot{_token}/{method}"), body, cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + throw new TelegramException("Telegram non risponde: " + Redactor.Redact(ex.Message), retryable: true); + } + catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TelegramException("Telegram non ha risposto in tempo.", retryable: true); + } + + using (response) + { + var text = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + JsonDocument? json = null; + try + { + json = JsonDocument.Parse(text); + } + catch (JsonException) + { + } + + if (response.IsSuccessStatusCode && json is not null && json.RootElement.TryGetProperty("ok", out var ok) && ok.GetBoolean()) + { + return json; + } + + var description = json is not null && json.RootElement.TryGetProperty("description", out var d) ? d.GetString() : null; + TimeSpan? after = json is not null && json.RootElement.TryGetProperty("parameters", out var p) && p.TryGetProperty("retry_after", out var r) + ? TimeSpan.FromSeconds(r.GetInt32()) + : null; + json?.Dispose(); + var status = (int)response.StatusCode; + throw new TelegramException($"Telegram ha risposto {status}: {Redactor.Redact(description ?? response.ReasonPhrase ?? "")}", + retryable: response.StatusCode == HttpStatusCode.TooManyRequests || status >= 500, after); + } + } +} diff --git a/src/Anubis/Supervision/ChildEngine.cs b/src/Anubis/Supervision/ChildEngine.cs new file mode 100644 index 0000000..647d0f4 --- /dev/null +++ b/src/Anubis/Supervision/ChildEngine.cs @@ -0,0 +1,158 @@ +using System.Diagnostics; +using System.IO.Pipes; +using System.Security.Cryptography; +using System.Text; + +namespace Anubis.Supervision; + +/// +/// The engine as a child process of the guard, with its channel: a named pipe open to this user only, created +/// before the process starts. The child's output goes where the guard's goes (the container log). +/// +public sealed class ChildEngine : IEngineProcess +{ + private readonly Process _process; + private readonly NamedPipeServerStream _pipe; + private readonly CancellationTokenSource _stop = new(); + private readonly SemaphoreSlim _write = new(1, 1); + private readonly Task _reading; + private readonly TaskCompletionSource _connected = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _disposed; + + private ChildEngine(Process process, NamedPipeServerStream pipe, Action onMessage) + { + _process = process; + _pipe = pipe; + Exited = WaitAsync(process); + _reading = Task.Run(() => ReadAsync(onMessage, _stop.Token)); + } + + public Task Exited { get; } + + /// + /// Starts with and the guard's environment, plus the pipe's + /// name; the drill's fault only when asks for it. + /// + public static ChildEngine Start(string file, IEnumerable arguments, IReadOnlyDictionary environment, EngineStart start) + { + ArgumentNullException.ThrowIfNull(arguments); + ArgumentNullException.ThrowIfNull(environment); + ArgumentNullException.ThrowIfNull(start); + var name = "anubis-" + Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(8)); + var pipe = new NamedPipeServerStream(name, PipeDirection.InOut, 1, PipeTransmissionMode.Byte, PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly); + var info = new ProcessStartInfo(file) { UseShellExecute = false }; + foreach (var argument in arguments) + { + info.ArgumentList.Add(argument); + } + + info.Environment.Clear(); + foreach (var (key, value) in environment) + { + if (value is not null && key != GuardChannel.FaultVariable && key != GuardChannel.PipeVariable) + { + info.Environment[key] = value; + } + } + + info.Environment[GuardChannel.PipeVariable] = name; + if (start.Fault && environment.GetValueOrDefault(GuardChannel.FaultVariable) is { Length: > 0 } fault) + { + info.Environment[GuardChannel.FaultVariable] = fault; + } + + try + { + var process = Process.Start(info) ?? throw new InvalidOperationException("Il motore non è partito."); + return new ChildEngine(process, pipe, start.OnMessage); + } + catch + { + pipe.Dispose(); + throw; + } + } + + public async Task RequestStopAsync(CancellationToken cancellationToken) + { + await _connected.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + await _write.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var bytes = Encoding.UTF8.GetBytes(GuardChannel.Write(new GuardMessage(GuardMessageTypes.Stop)) + "\n"); + await _pipe.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + await _pipe.FlushAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + _write.Release(); + } + } + + public void Kill() + { + try + { + if (!_process.HasExited) + { + _process.Kill(entireProcessTree: true); + } + } + catch (InvalidOperationException) + { + // Already gone. + } + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + _stop.Cancel(); + try + { + _reading.Wait(TimeSpan.FromSeconds(5)); + } + catch (AggregateException) + { + } + + _pipe.Dispose(); + _process.Dispose(); + _stop.Dispose(); + _write.Dispose(); + } + + private static async Task WaitAsync(Process process) + { + await process.WaitForExitAsync().ConfigureAwait(false); + return process.ExitCode; + } + + private async Task ReadAsync(Action onMessage, CancellationToken cancellationToken) + { + try + { + await _pipe.WaitForConnectionAsync(cancellationToken).ConfigureAwait(false); + _connected.TrySetResult(); + using var reader = new StreamReader(_pipe, Encoding.UTF8, leaveOpen: true); + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + if (GuardChannel.Read(line) is { } message) + { + onMessage(message); + } + } + } + catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException) + { + } + finally + { + _connected.TrySetCanceled(CancellationToken.None); + } + } +} diff --git a/src/Anubis/Supervision/GuardBrokers.cs b/src/Anubis/Supervision/GuardBrokers.cs new file mode 100644 index 0000000..92f49c3 --- /dev/null +++ b/src/Anubis/Supervision/GuardBrokers.cs @@ -0,0 +1,53 @@ +using Anubis.Brokers; +using Anubis.Core; +using Anubis.Data; +using Anubis.Notifications; +using Anubis.Trading; +using Microsoft.Extensions.Logging; + +namespace Anubis.Supervision; + +/// The broker adapters and the warnings of the guard, from the keys the engine handed it in memory. +public static class GuardBrokers +{ + /// + /// The adapter of a demo account. Real accounts get none until the real-money phase gives the guard the startup + /// consent of its own (ADR 0012); the simulator needs none, the engine is its broker. + /// + public static IBrokerAccount? Create(HttpClient http, IClock clock, GuardAccount account, IReadOnlyList keys) + { + ArgumentNullException.ThrowIfNull(account); + ArgumentNullException.ThrowIfNull(keys); + if (account.Environment != TradingEnvironments.Demo || keys.FirstOrDefault(k => k.Service == account.Broker && k.Environment == account.Environment) is not { } key) + { + return null; + } + + return account.Broker switch + { + Venues.Alpaca when key.Values.Count >= 2 => new AlpacaTrading(http, clock, new AlpacaCredentials(key.Values[0], key.Values[1]), + new AlpacaTradingOptions { Environment = account.Environment }, TradingConsent.None), + Venues.Okx when key.Values.Count >= 3 => new OkxTrading(http, clock, new OkxCredentials(key.Values[0], key.Values[1], key.Values[2]), + new OkxTradingOptions { Environment = account.Environment }, TradingConsent.None), + _ => null, + }; + } + + /// The guard's warning: always in its log, and on Telegram when the engine handed it the bot's token and chat. + public static async Task AlertAsync(HttpClient http, ILogger logger, IReadOnlyList keys, string text, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(keys); + GuardLog.Alert(logger, text.ReplaceLineEndings(" ")); + if (keys.FirstOrDefault(k => k.Service == GuardKeys.Telegram) is { Values.Count: >= 2 } telegram) + { + await new TelegramClient(http, telegram.Values[0]).SendAsync(telegram.Values[1], "Anubis, guardia" + Environment.NewLine + text, cancellationToken) + .ConfigureAwait(false); + } + } +} + +internal static partial class GuardLog +{ + [LoggerMessage(Level = LogLevel.Warning, Message = "Guard alert: {Text}")] + public static partial void Alert(ILogger logger, string text); +} diff --git a/src/Anubis/Supervision/GuardDrill.cs b/src/Anubis/Supervision/GuardDrill.cs new file mode 100644 index 0000000..433ccbe --- /dev/null +++ b/src/Anubis/Supervision/GuardDrill.cs @@ -0,0 +1,419 @@ +using System.Globalization; +using System.Net; +using System.Net.Sockets; +using System.Security.Cryptography; +using System.Text; +using Anubis.Core; +using Anubis.Data; +using Anubis.Db; +using Anubis.Platform; +using Anubis.Trading; +using Microsoft.Extensions.Logging; + +namespace Anubis.Supervision; + +/// The outcome of the guard drill, line by line in Italian. +public sealed record DrillResult(bool Passed, IReadOnlyList Lines); + +/// +/// The guard drill (issue #48, ADR 0012): the real guard and the real engine, as child process, in a folder of their +/// own with a SQLite database. A position of 0.01 BTC with its stop on the internal simulator and a sale already +/// decided; the engine starts with the fault that blocks its main loop right after the sale leaves, with the stop +/// removed. The drill passes when the guard intervened, the engine restarted, the sale left open was canceled, the +/// stop came back for the whole quantity and the guard's note reached the diary. Prices come from a small local +/// server that answers like OKX: nothing leaves the computer. Same drill in the tests and in the Docker image. +/// +public static class GuardDrill +{ + public const decimal Quantity = 0.01m; + public const decimal StopLevel = 45000m; + + /// Times short enough for a drill of about two minutes. + public static GuardOptions Options { get; } = new() + { + Silence = TimeSpan.FromSeconds(20), + StalledStart = TimeSpan.FromMinutes(3), + GracefulStop = TimeSpan.FromSeconds(30), + }; + + /// The drill in a new temporary folder, removed at the end. Exit code 0 when it passes. + public static async Task RunAsync(string serverDll, TextWriter output, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(output); + var folder = Path.Combine(Path.GetTempPath(), "anubis-drill-" + Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(6))); + try + { + var result = await RunInAsync(folder, serverDll, Options, line => output.WriteLine(line), cancellationToken).ConfigureAwait(false); + await output.WriteLineAsync(result.Passed ? "Esercitazione della guardia superata." : "Esercitazione della guardia NON superata.").ConfigureAwait(false); + return result.Passed ? 0 : 1; + } + finally + { + try + { + Directory.Delete(folder, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // A leftover temporary folder is harmless. + } + } + } + + public static async Task RunInAsync(string folder, string serverDll, GuardOptions options, Action say, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(say); + var lines = new List(); + void Say(string line) + { + lines.Add(line); + say(line); + } + + var data = new DataDirectory(folder); + data.Ensure(); + using var okx = FakeOkx.Start(); + var held = await SeedAsync(data, cancellationToken).ConfigureAwait(false); + Say($"Preparato: {Format.Quantity(held)} BTC con lo stop a {Format.Price(StopLevel)} e una vendita già decisa."); + + var environment = ChildEnvironment(okx.Address); + var clock = SystemClock.Instance; + var logger = new LineLogger(Say); + var guardian = new Guardian(new GuardPaths(folder), clock, options, + start => ChildEngine.Start(GuardLaunch.Dotnet(), [serverDll, "--platform", "docker", "--data-dir", folder], environment, start), + (_, _) => null, (text, _, _) => + { + Say("Avviso della guardia: " + text.ReplaceLineEndings(" ")); + return Task.CompletedTask; + }, logger); + guardian.Start(fault: true); + using var stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var guarding = guardian.RunAsync(stop.Token); + + var intervened = await WaitAsync(() => guardian.Interventions > 0, options.Silence + TimeSpan.FromMinutes(3), cancellationToken).ConfigureAwait(false); + var restarted = intervened && await WaitAsync(() => guardian.EngineAlive && guardian.Stage == GuardStages.Running, TimeSpan.FromMinutes(3), cancellationToken) + .ConfigureAwait(false); + await stop.CancelAsync().ConfigureAwait(false); + var code = await guarding.ConfigureAwait(false); + Say($"Motore fermato in ordine alla fine dell'esercitazione (codice {code})."); + + var passed = Check(intervened, "La guardia è intervenuta dopo il silenzio del ciclo principale.", "La guardia non è intervenuta.", Say); + passed &= Check(restarted, "Il motore è ripartito e il suo ciclo batte di nuovo.", "Il motore non è ripartito.", Say); + var notes = guardian.Notes; + var at = notes.Count > 0 ? notes[0].At : DateTimeOffset.MaxValue; + await using (var db = Database.Open(new DbSettings { Engine = DbEngine.Sqlite, SqlitePath = data.SqliteFile }, allowCreate: false)) + { + var hung = await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM orders WHERE side = 'sell' AND purpose = 'exit' AND created_at < @t", ("t", at)), cancellationToken) + .ConfigureAwait(false); + var canceled = await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM executions WHERE side = 'sell' AND purpose = 'exit' AND status = 'canceled' AND reason = @r", + ("r", LiveReasons.Restart)), cancellationToken).ConfigureAwait(false); + var stops = await db.ReadAsync(s => s.QueryAsync("SELECT quantity FROM orders WHERE purpose = 'stop' AND created_at > @t", r => r.Money("quantity"), ("t", at)), + cancellationToken).ConfigureAwait(false); + var noted = await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM journal WHERE code = 'guard_intervention'"), cancellationToken).ConfigureAwait(false); + var again = await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM planned_orders WHERE side = 'sell' AND created_at > @t", ("t", at)), cancellationToken) + .ConfigureAwait(false); + passed &= Check(hung > 0, "Prima dell'intervento la vendita era partita, senza più lo stop.", "La vendita non era partita prima dell'intervento.", Say); + passed &= Check(canceled > 0, "Alla ripartenza la vendita rimasta a metà è stata annullata.", "La vendita rimasta a metà non è stata annullata.", Say); + passed &= Check(stops.Any(q => q == held), $"Lo stop è tornato per tutti i {Format.Quantity(held)} BTC.", "Lo stop non è tornato per l'intera quantità.", Say); + passed &= Check(again > 0, "La vendita già decisa è ripartita con un ordine nuovo.", "La vendita già decisa non è ripartita.", Say); + passed &= Check(noted > 0, "La nota della guardia è nel diario.", "La nota della guardia non è nel diario.", Say); + } + + return new DrillResult(passed, lines); + } + + private static bool Check(bool ok, string yes, string no, Action say) + { + say((ok ? "OK " : "NO ") + (ok ? yes : no)); + return ok; + } + + private static async Task WaitAsync(Func condition, TimeSpan timeout, CancellationToken cancellationToken) + { + var until = SystemClock.Instance.UtcNow + timeout; + while (!condition()) + { + if (SystemClock.Instance.UtcNow > until) + { + return false; + } + + await Task.Delay(250, cancellationToken).ConfigureAwait(false); + } + + return true; + } + + private static Dictionary ChildEnvironment(Uri okx) + { + var environment = new Dictionary(StringComparer.Ordinal); + foreach (System.Collections.DictionaryEntry entry in Environment.GetEnvironmentVariables()) + { + var key = (string)entry.Key; + if (!key.StartsWith("ANUBIS_", StringComparison.Ordinal) && !key.StartsWith("ALPACA_", StringComparison.Ordinal) && + !key.StartsWith("OKX_", StringComparison.Ordinal) && !key.StartsWith("TELEGRAM_", StringComparison.Ordinal)) + { + environment[key] = entry.Value as string; + } + } + + environment["ANUBIS_PLATFORM"] = "docker"; + environment["ANUBIS_DB_ENGINE"] = "sqlite"; + environment["ANUBIS_SECRET_KEY"] = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(24)); + environment["ANUBIS_ADMIN_PASSWORD"] = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(12)); + environment["PORT"] = FreePort().ToString(CultureInfo.InvariantCulture); + environment["TZ"] = "Europe/Rome"; + environment[DrillOkxVariable] = okx.ToString(); + environment[GuardChannel.FaultVariable] = GuardChannel.HangMidSale; + return environment; + } + + /// + /// The address of the local server that answers like OKX in the drill. Honored only on this computer's own + /// addresses (loopback): it can never point the bot at a price source elsewhere. + /// + public const string DrillOkxVariable = "ANUBIS_DRILL_OKX_URL"; + + /// The OKX addresses of the drill, or null when the variable is missing or not a loopback address. + public static Brokers.OkxOptions? DrillOkx(IReadOnlyDictionary environment) + { + ArgumentNullException.ThrowIfNull(environment); + if (environment.GetValueOrDefault(DrillOkxVariable) is not { Length: > 0 } text || !Uri.TryCreate(text, UriKind.Absolute, out var url) || !url.IsLoopback) + { + return null; + } + + var socket = new UriBuilder(url) { Scheme = "ws" }; + return new Brokers.OkxOptions + { + RestUrl = url, + PublicStreamUrl = new Uri(socket.Uri, "/ws/v5/public"), + BusinessStreamUrl = new Uri(socket.Uri, "/ws/v5/business"), + }; + } + + // The database the engine will find: installation, accounts, BTC, the capital, a filled purchase with its stop on + // the simulator, and the sale decided for now. Built with the same order manager the engine uses. + private static async Task SeedAsync(DataDirectory data, CancellationToken cancellationToken) + { + var clock = SystemClock.Instance; + await using var db = Database.Open(new DbSettings { Engine = DbEngine.Sqlite, SqlitePath = data.SqliteFile }, allowCreate: true); + await Migrations.MigrateAsync(db, clock, null, cancellationToken).ConfigureAwait(false); + var identity = await Installation.CheckAsync(db, data, clock, cancellationToken).ConfigureAwait(false); + var accounts = await AccountSetup.EnsureAsync(db, clock, alpacaDemo: false, okxDemo: false, cancellationToken).ConfigureAwait(false); + var account = accounts.First(a => a.Broker == Venues.Okx && a.Environment == TradingEnvironments.Simulation); + var btc = await new InstrumentCatalog(db, clock).EnsureAsync(new InstrumentSpec(Venues.Okx, "BTC-EUR", "BTC-EUR", AssetClasses.Crypto, "EUR") + { + BaseCurrency = "BTC", + TickSize = 0.1m, + LotSize = 0.00000001m, + MinSize = 0.00001m, + Status = "live", + }, cancellationToken).ConfigureAwait(false); + var ledger = new Ledger(db, clock); + await new Capital(db, clock, ledger, new FxStore(db, clock)).SetAsync(Venues.Okx, 1000m, "esercitazione", cancellationToken).ConfigureAwait(false); + + Task QuoteAsync(string symbol, CancellationToken _) => Task.FromResult(new Quote(symbol, FakeOkx.Bid, FakeOkx.Ask, clock.UtcNow)); + using var paper = new PaperBroker(PaperOptions.Okx() with { MinLatency = TimeSpan.Zero, MaxLatency = TimeSpan.Zero }, clock, QuoteAsync, + symbol => symbol == btc.Symbol ? btc.Spec : null, new PaperBookStore(db, clock, account.AccountId)); + await paper.DepositAsync(1000m, "EUR", cancellationToken).ConfigureAwait(false); + using var manager = new OrderManager(db, clock, ledger, paper, account, ClientOrderIds.InstallationCode(identity.File!.InstallationId), + (id, _) => Task.FromResult(id == btc.Id ? btc : null), QuoteAsync) + { + PassiveSteps = 0, + ConfirmPrice = QuoteAsync, + MarketOpen = (_, _) => Task.FromResult(true), + }; + var (_, refusal) = await manager.SubmitAsync(new TradeIntent(btc.Id, StrategyNames.CryptoTrend, OrderSide.Buy, Quantity, OrderPurpose.Entry, Ids.NewId(clock)) + { + StopLevel = StopLevel, + }, cancellationToken).ConfigureAwait(false); + if (refusal is not null) + { + throw new InvalidOperationException("Esercitazione: acquisto rifiutato (" + refusal + ")."); + } + + for (var i = 0; ; i++) + { + await paper.PumpAsync(cancellationToken).ConfigureAwait(false); + await manager.TickAsync(cancellationToken).ConfigureAwait(false); + var stops = await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM orders WHERE purpose = 'stop' AND status = 'submitted'"), cancellationToken).ConfigureAwait(false); + if (stops > 0) + { + break; + } + + if (i > 100) + { + throw new InvalidOperationException("Esercitazione: la posizione non ha avuto il suo stop."); + } + + await Task.Delay(100, cancellationToken).ConfigureAwait(false); + } + + var held = (await ledger.PositionsAsync(account.AccountId, cancellationToken).ConfigureAwait(false)).Single().Quantity; + var now = clock.UtcNow; + await db.WriteAsync(s => s.ExecuteAsync( + "INSERT INTO planned_orders (id, decision_id, strategy, account_id, instrument_id, side, purpose, quantity, exit_all, stop_level, stop_offset, reference_price, reason, " + + "due_at, valid_until, status, outcome, execution_id, created_at, updated_at) VALUES (@id, @d, @s, @a, @i, 'sell', 'exit', @q, @x, NULL, NULL, @r, 'esercitazione', " + + "@now, NULL, 'pending', NULL, NULL, @now, @now)", cancellationToken, + ("id", Ids.NewId(clock)), ("d", Ids.NewId(clock)), ("s", StrategyNames.CryptoTrend), ("a", account.AccountId), ("i", btc.Id), ("q", held), ("x", true), + ("r", FakeOkx.Bid), ("now", now)), cancellationToken).ConfigureAwait(false); + return held; + } + + private static int FreePort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return ((IPEndPoint)listener.LocalEndpoint).Port; + } + + /// + /// A local server that answers like OKX's REST interface with a fixed price for every ticker and nothing for the + /// rest; anything else (the streams) gets 404. Plain HTTP/1.1 on the loopback address, one request per connection. + /// + private sealed class FakeOkx : IDisposable + { + public const decimal Bid = 50000m; + public const decimal Ask = 50010m; + + private readonly TcpListener _listener; + private readonly CancellationTokenSource _stop = new(); + private readonly Task _loop; + + private FakeOkx() + { + _listener = new TcpListener(IPAddress.Loopback, 0); + _listener.Start(); + Address = new Uri($"http://127.0.0.1:{((IPEndPoint)_listener.LocalEndpoint).Port}/"); + _loop = Task.Run(() => AcceptAsync(_stop.Token)); + } + + public Uri Address { get; } + + public static FakeOkx Start() => new(); + + public void Dispose() + { + _stop.Cancel(); + _listener.Stop(); + try + { + _loop.Wait(TimeSpan.FromSeconds(5)); + } + catch (AggregateException) + { + } + + _stop.Dispose(); + } + + private async Task AcceptAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + TcpClient client; + try + { + client = await _listener.AcceptTcpClientAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException) + { + return; + } + + _ = Task.Run(() => AnswerAsync(client, cancellationToken), CancellationToken.None); + } + } + + private static async Task AnswerAsync(TcpClient client, CancellationToken cancellationToken) + { + using (client) + { + try + { + var stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.ASCII, leaveOpen: true); + var request = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) ?? ""; + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { Length: > 0 }) + { + } + + var target = request.Split(' ') is [_, var path, ..] ? path : "/"; + var (status, body) = Route(target); + var bytes = Encoding.UTF8.GetBytes(body); + var head = $"HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {bytes.Length}\r\nConnection: close\r\n\r\n"; + await stream.WriteAsync(Encoding.ASCII.GetBytes(head), cancellationToken).ConfigureAwait(false); + await stream.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or OperationCanceledException or SocketException) + { + } + } + } + + private static (string Status, string Body) Route(string target) + { + if (target.StartsWith("/ws", StringComparison.Ordinal)) + { + return ("404 Not Found", "{}"); + } + + if (target.StartsWith("/api/v5/market/ticker?instId=", StringComparison.Ordinal)) + { + var symbol = Uri.UnescapeDataString(target["/api/v5/market/ticker?instId=".Length..]); + var ts = SystemClock.Instance.UtcNow.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture); + return ("200 OK", $"{{\"code\":\"0\",\"msg\":\"\",\"data\":[{{\"instId\":\"{symbol}\",\"bidPx\":\"{Bid.ToString(CultureInfo.InvariantCulture)}\"," + + $"\"askPx\":\"{Ask.ToString(CultureInfo.InvariantCulture)}\",\"bidSz\":\"1\",\"askSz\":\"1\",\"ts\":\"{ts}\"}}]}}"); + } + + // The list of instruments has the drill's one: an instrument missing from it is taken as delisted. + if (target.StartsWith("/api/v5/public/instruments", StringComparison.Ordinal)) + { + return ("200 OK", "{\"code\":\"0\",\"msg\":\"\",\"data\":[{\"instId\":\"BTC-EUR\",\"baseCcy\":\"BTC\",\"quoteCcy\":\"EUR\",\"tickSz\":\"0.1\",\"lotSz\":\"0.00000001\",\"minSz\":\"0.00001\",\"state\":\"live\"}]}"); + } + + return ("200 OK", "{\"code\":\"0\",\"msg\":\"\",\"data\":[]}"); + } + } +} + +/// How the guard finds the .NET host and the engine next to itself. +public static class GuardLaunch +{ + /// The dotnet host that runs the engine: the one running now when it is dotnet itself, otherwise the one the SDK names. + public static string Dotnet() + { + if (Environment.ProcessPath is { } current && Path.GetFileNameWithoutExtension(current).Equals("dotnet", StringComparison.OrdinalIgnoreCase)) + { + return current; + } + + return Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") is { Length: > 0 } host ? host : "dotnet"; + } + + /// The engine's program next to the guard's. + public static string Server() => Path.Combine(AppContext.BaseDirectory, "Anubis.Server.dll"); +} + +/// A logger that writes one line per message: the guard's log goes where the container's goes. +public sealed class LineLogger(Action write) : ILogger +{ + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Information; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + ArgumentNullException.ThrowIfNull(formatter); + if (!IsEnabled(logLevel)) + { + return; + } + + var text = Redactor.Redact(formatter(state, exception) + (exception is null ? "" : " — " + exception.Message)); + write($"{SystemClock.Instance.UtcNow:yyyy-MM-dd HH:mm:ss} guardia {logLevel}: {text}"); + } +} diff --git a/src/Anubis/Supervision/GuardLink.cs b/src/Anubis/Supervision/GuardLink.cs new file mode 100644 index 0000000..365ad08 --- /dev/null +++ b/src/Anubis/Supervision/GuardLink.cs @@ -0,0 +1,170 @@ +using System.IO.Pipes; +using System.Text; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Anubis.Supervision; + +/// +/// The engine's end of the channel to the guard (SPEC 9.3): the beat of the main loop, the startup steps, the keys +/// and the stops it decides. Messages wait in a short queue and the oldest beats give way: a slow guard never slows +/// the engine. When the guard is gone the engine goes on by itself. +/// +public sealed partial class GuardLink : IAsyncDisposable +{ + private readonly Lock _gate = new(); + private readonly NamedPipeClientStream _pipe; + private readonly Channel _outgoing = Channel.CreateBounded(new BoundedChannelOptions(64) { FullMode = BoundedChannelFullMode.DropOldest }); + private readonly CancellationTokenSource _stop = new(); + private readonly ILogger _logger; + private readonly Task _loop; + private string _stage = GuardStages.Starting; + private string? _step; + + private GuardLink(string pipeName, ILogger logger) + { + _logger = logger; + _pipe = new NamedPipeClientStream(".", pipeName, PipeDirection.InOut, PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly); + _loop = Task.Run(() => RunAsync(_stop.Token)); + } + + /// The link when a guard started this engine (); null otherwise. + public static GuardLink? FromEnvironment(IReadOnlyDictionary environment, ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(environment); + return environment.GetValueOrDefault(GuardChannel.PipeVariable) is { Length: > 0 } name ? new GuardLink(name, logger ?? NullLogger.Instance) : null; + } + + /// The guard asked for an orderly stop. + public Action? StopRequested { get; set; } + + public string Stage + { + get + { + lock (_gate) + { + return _stage; + } + } + } + + /// A new stage or startup step: told to the guard at once. + public void Set(string stage, string? step = null) + { + lock (_gate) + { + if (_stage == stage && _step == step) + { + return; + } + + _stage = stage; + _step = step; + } + + Beat(); + } + + /// A step of the startup of SPEC 9.4. + public void Step(string step) => Set(GuardStages.Starting, step); + + /// Alive, at the current stage. + public void Beat() + { + GuardMessage message; + lock (_gate) + { + message = GuardMessage.Beat(_stage, _step); + } + + _outgoing.Writer.TryWrite(message); + } + + public void SendKeys(IReadOnlyList keys) => _outgoing.Writer.TryWrite(new GuardMessage(GuardMessageTypes.Keys) { Keys = keys }); + + /// A stop decided by the engine: the guard will not restart it. Waits a moment for the message to leave. + public async Task AnnounceAsync(string reason) + { + _outgoing.Writer.TryWrite(new GuardMessage(GuardMessageTypes.Stopping) { Reason = reason }); + for (var i = 0; i < 20 && _outgoing.Reader.Count > 0; i++) + { + await Task.Delay(50).ConfigureAwait(false); + } + } + + public async ValueTask DisposeAsync() + { + await _stop.CancelAsync().ConfigureAwait(false); + try + { + await _loop.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + + await _pipe.DisposeAsync().ConfigureAwait(false); + _stop.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + await _pipe.ConnectAsync(TimeSpan.FromSeconds(30), cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is TimeoutException or IOException or UnauthorizedAccessException) + { + LogNoGuard(_logger, ex.Message); + return; + } + + var reading = ReadAsync(cancellationToken); + var writer = new StreamWriter(_pipe, new UTF8Encoding(false), leaveOpen: true) { AutoFlush = true, NewLine = "\n" }; + await using (writer.ConfigureAwait(false)) + { + try + { + await foreach (var message in _outgoing.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + await writer.WriteLineAsync(GuardChannel.Write(message).AsMemory(), cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (IOException ex) + { + LogGuardLost(_logger, ex.Message); + } + } + + await reading.ConfigureAwait(false); + } + + private async Task ReadAsync(CancellationToken cancellationToken) + { + using var reader = new StreamReader(_pipe, Encoding.UTF8, leaveOpen: true); + try + { + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + if (GuardChannel.Read(line) is { Type: GuardMessageTypes.Stop }) + { + StopRequested?.Invoke(); + } + } + } + catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException) + { + } + } + + [LoggerMessage(Level = LogLevel.Warning, Message = "Guard channel not reached: {Reason}. The engine runs without its guard.")] + private static partial void LogNoGuard(ILogger logger, string reason); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Guard channel lost: {Reason}")] + private static partial void LogGuardLost(ILogger logger, string reason); +} diff --git a/src/Anubis/Supervision/GuardProtocol.cs b/src/Anubis/Supervision/GuardProtocol.cs new file mode 100644 index 0000000..3f3f4af --- /dev/null +++ b/src/Anubis/Supervision/GuardProtocol.cs @@ -0,0 +1,120 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Anubis.Supervision; + +/// +/// The channel between the guard and the engine (SPEC 9.3, ADR 0012): a local named pipe whose name the guard puts +/// in , one JSON message per line. The engine sends its beat, the startup steps, the broker +/// keys (in memory only) and the stops it decides; the guard sends only "stop". +/// +public static class GuardChannel +{ + /// The pipe the engine connects to. Without it the engine runs on its own (development, tests). + public const string PipeVariable = "ANUBIS_GUARD_PIPE"; + + /// + /// Fault injection for the guard drill (ADR 0012): blocks the main loop right after a sale + /// leaves. The guard never passes it to an engine it restarts. + /// + public const string FaultVariable = "ANUBIS_FAULT"; + + public const string HangMidSale = "hang-mid-sale"; + + public static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Converters = { new JsonStringEnumConverter() }, + }; + + public static string Write(GuardMessage message) => JsonSerializer.Serialize(message, Json); + + /// A line of the channel; null when it is not a message (a broken line is ignored, never fatal). + public static GuardMessage? Read(string? line) + { + if (string.IsNullOrWhiteSpace(line)) + { + return null; + } + + try + { + return JsonSerializer.Deserialize(line, Json) is { Type.Length: > 0 } message ? message : null; + } + catch (JsonException) + { + return null; + } + } +} + +public static class GuardMessageTypes +{ + /// Engine to guard: alive, with its stage and the startup step it is on. + public const string Beat = "beat"; + + /// Engine to guard: the keys of the broker accounts and of Telegram, kept in memory only. + public const string Keys = "keys"; + + /// Engine to guard: a stop decided by the engine (exit, restore, import): not restarted. + public const string Stopping = "stopping"; + + /// Guard to engine: stop in an orderly way (the container is stopping). + public const string Stop = "stop"; +} + +/// Stages of the engine as the guard sees them. +public static class GuardStages +{ + /// The startup steps of SPEC 9.4: watched for 10 minutes without progress. + public const string Starting = "starting"; + + /// The main loop runs: watched for 2 minutes of silence. + public const string Running = "running"; + + /// Error page or first run: waiting for the user, never a reason to intervene. + public const string Waiting = "waiting"; +} + +/// One message of the channel. +public sealed record GuardMessage(string Type) +{ + public string? Stage { get; init; } + + public string? Step { get; init; } + + public string? Reason { get; init; } + + public IReadOnlyList? Keys { get; init; } + + public static GuardMessage Beat(string stage, string? step) => new(GuardMessageTypes.Beat) { Stage = stage, Step = step }; + + // Never prints the keys. + public override string ToString() => $"GuardMessage({Type}, {Stage}, {Step})"; +} + +/// +/// The keys of one account, as the engine has them: Alpaca key and secret, OKX key, secret and passphrase, Telegram +/// token and chat. Held only in the guard's memory, never written (SPEC 9.3). +/// +public sealed record GuardKeys(string Service, string Environment, IReadOnlyList Values) +{ + public const string Telegram = "telegram"; + + public override string ToString() => $"GuardKeys({Service}, {Environment}, ***)"; +} + +/// Where the guard and the engine meet in the data folder. +public sealed record GuardPaths(string DataRoot) +{ + public string Folder => Path.Combine(DataRoot, "guard"); + + /// Positions, stop levels and open orders, kept current by the engine for the guard. + public string StateFile => Path.Combine(Folder, "state.json"); + + /// The guard does not write in the database: it leaves notes here, which the engine puts in the diary. + public string NotesFolder => Path.Combine(Folder, "notes"); + + /// The "Ferma tutto" file (SPEC 9.3). + public string StopFile => Path.Combine(DataRoot, "STOP"); +} diff --git a/src/Anubis/Supervision/GuardRepair.cs b/src/Anubis/Supervision/GuardRepair.cs new file mode 100644 index 0000000..c0ed4f6 --- /dev/null +++ b/src/Anubis/Supervision/GuardRepair.cs @@ -0,0 +1,130 @@ +using Anubis.Core; +using Anubis.Trading; + +namespace Anubis.Supervision; + +/// What the guard did on one account. +public sealed record RepairOutcome(IReadOnlyList Lines, IReadOnlyList Placed, bool Complete); + +/// +/// The guard's work on a broker account while the engine is stopped (SPEC 9.3): reads positions and the outcome of the +/// orders in the state file, cancels the purchases and sales left open, and puts back only the stops that are +/// missing and were not executed, for the quantity still held. Protective sales and stops are never touched. +/// +public static class GuardRepair +{ + /// + /// The guard's stops take sequences this far above the engine's next one: if a note were lost, the engine's next + /// stops would still not reuse their ids. + /// + public const int SequenceGap = 500; + + public static async Task RunAsync(GuardState state, GuardAccount account, IBrokerAccount broker, IClock clock, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(state); + ArgumentNullException.ThrowIfNull(account); + ArgumentNullException.ThrowIfNull(broker); + ArgumentNullException.ThrowIfNull(clock); + var lines = new List(); + var placed = new List(); + var complete = true; + + // 1. Purchases and sales still waiting: the open ones the broker shows, and those the state lists. + var open = (await broker.GetOpenOrdersAsync(cancellationToken).ConfigureAwait(false)) + .Where(o => ClientOrderIds.IsOurs(o.ClientOrderId, state.Installation)).ToList(); + var waiting = open.Where(o => ClientOrderIds.Parse(o.ClientOrderId)?.Purpose is OrderPurpose.Entry or OrderPurpose.Exit) + .Select(o => (o.ClientOrderId, o.Symbol, o.Side)).ToList(); + foreach (var listed in account.Orders.Where(o => o.Purpose is OrderPurpose.Entry or OrderPurpose.Exit && waiting.All(w => w.ClientOrderId != o.ClientOrderId))) + { + if (await broker.GetOrderAsync(listed.ClientOrderId, listed.Symbol, cancellationToken).ConfigureAwait(false) is { IsOpen: true } remote) + { + waiting.Add((remote.ClientOrderId, remote.Symbol, remote.Side)); + } + } + + foreach (var (clientId, symbol, side) in waiting) + { + await broker.CancelOrderAsync(clientId, symbol, cancellationToken).ConfigureAwait(false); + lines.Add($"Ho annullato {(side == OrderSide.Buy ? "l'acquisto" : "la vendita")} di {Shown(symbol)} rimast{(side == OrderSide.Buy ? "o" : "a")} in attesa."); + } + + // 2. What the bot holds now: the state's quantity plus the fills of its own orders since then, never more than the account has. + var fills = (await broker.GetFillsAsync(state.WrittenAt, cancellationToken).ConfigureAwait(false)) + .Where(f => f.ExecutedAt > state.WrittenAt && ClientOrderIds.IsOurs(f.ClientOrderId, state.Installation)).ToList(); + var positions = await broker.GetPositionsAsync(cancellationToken).ConfigureAwait(false); + var stops = (await broker.GetOpenOrdersAsync(cancellationToken).ConfigureAwait(false)) + .Where(o => o.IsOpen && o.Side == OrderSide.Sell && ClientOrderIds.Parse(o.ClientOrderId) is { Purpose: OrderPurpose.Stop } parts && parts.Installation == state.Installation) + .ToList(); + foreach (var bySymbol in account.Positions.Where(p => p.StopLevel is not null).GroupBy(p => p.Symbol, StringComparer.Ordinal)) + { + var available = positions.Where(p => p.Symbol == bySymbol.Key).Sum(p => p.Quantity); + foreach (var position in bySymbol.OrderBy(p => p.Strategy, StringComparer.Ordinal)) + { + var code = ClientOrderIds.StrategyCodes.GetValueOrDefault(position.Strategy); + var moved = fills.Where(f => f.Symbol == position.Symbol && ClientOrderIds.Parse(f.ClientOrderId)?.Strategy == position.Strategy) + .Sum(f => f.Side == OrderSide.Buy ? f.Quantity : -f.Quantity); + var held = Math.Min(Math.Max(0m, position.Quantity + moved), available); + available -= held; + var covered = stops.Where(o => o.Symbol == position.Symbol && code is not null && ClientOrderIds.Parse(o.ClientOrderId)?.Strategy == position.Strategy) + .Sum(o => o.Remaining); + var missing = OrderRules.RoundDown(held - covered, position.Lot); + if (missing <= 0m || code is null) + { + continue; + } + + var sequence = Math.Min(position.NextStopSequence + SequenceGap, ClientOrderIds.MaxSequence - 2); + foreach (var (quantity, timeInForce) in Shapes(position, missing, account.FractionalStopsAreDayOnly)) + { + var clientId = ClientOrderIds.Build(state.Installation, account.Environment, position.Strategy, OrderPurpose.Stop, position.StopKey, sequence++); + var request = new OrderRequest(clientId, position.Symbol, OrderSide.Sell, OrderKind.Stop, quantity, timeInForce) { StopPrice = position.StopLevel }; + try + { + var remote = await broker.SubmitOrderAsync(request, cancellationToken).ConfigureAwait(false); + placed.Add(new PlacedStop(account.AccountId, position.InstrumentId, position.Symbol, position.Strategy, clientId, remote.BrokerOrderId, quantity, + position.StopLevel!.Value, timeInForce)); + lines.Add($"Ho rimesso lo stop di {Shown(position.Symbol)} a {Format.Price(position.StopLevel.Value)} per {Format.Quantity(quantity)}."); + } + catch (OrderRejectedException ex) + { + complete = false; + lines.Add($"{account.Name} ha rifiutato lo stop di {Shown(position.Symbol)} ({ex.Code}): la posizione resta senza stop finché il motore non riparte."); + } + } + } + } + + if (lines.Count == 0) + { + lines.Add($"Su {account.Name} non c'era nulla da annullare e ogni posizione aveva il suo stop."); + } + + return new RepairOutcome(lines, placed, complete); + } + + /// The stops a quantity needs on the venue, as the engine places them (SPEC 7.2): fractions of shares only as day orders. + public static IReadOnlyList<(decimal Quantity, TimeInForce TimeInForce)> Shapes(GuardPosition position, decimal quantity, bool fractionalDayOnly) + { + ArgumentNullException.ThrowIfNull(position); + if (!fractionalDayOnly || position.IsCrypto) + { + return [(quantity, TimeInForce.Gtc)]; + } + + var (whole, fraction) = OrderRules.SplitWhole(quantity); + var shapes = new List<(decimal, TimeInForce)>(); + if (whole > 0m) + { + shapes.Add((whole, TimeInForce.Gtc)); + } + + if (fraction > 0m) + { + shapes.Add((fraction, TimeInForce.Day)); + } + + return shapes; + } + + private static string Shown(string symbol) => DiaryText.Shown(symbol); +} diff --git a/src/Anubis/Supervision/GuardState.cs b/src/Anubis/Supervision/GuardState.cs new file mode 100644 index 0000000..6ecf457 --- /dev/null +++ b/src/Anubis/Supervision/GuardState.cs @@ -0,0 +1,116 @@ +using System.Globalization; +using System.Text.Json; +using Anubis.Trading; + +namespace Anubis.Supervision; + +/// +/// What the guard needs to protect the positions while the engine is stopped (SPEC 9.3): written by the engine every +/// few seconds, read by the guard only after it stopped the engine. +/// +public sealed record GuardState(DateTimeOffset WrittenAt, string Installation, IReadOnlyList Accounts); + +public sealed record GuardAccount(Guid AccountId, string Name, string Broker, string Environment, bool FractionalStopsAreDayOnly, + IReadOnlyList Positions, IReadOnlyList Orders); + +/// +/// A position of a strategy: quantity, stop level already on the tick, lot, and what the guard needs to name a stop +/// the engine will recognize as its own (the key of the position's stops and the next free sequence). +/// +public sealed record GuardPosition(Guid InstrumentId, string Symbol, string Strategy, decimal Quantity, decimal? StopLevel, decimal Lot, bool IsCrypto, + string StopKey, int NextStopSequence); + +/// An order the engine believes open. +public sealed record GuardOrder(string ClientOrderId, Guid InstrumentId, string Symbol, string Strategy, OrderSide Side, OrderPurpose Purpose, decimal Quantity, + decimal? StopPrice); + +/// A stop the guard placed while the engine was stopped: the engine adopts it at the restart. +public sealed record PlacedStop(Guid AccountId, Guid InstrumentId, string Symbol, string Strategy, string ClientOrderId, string? BrokerOrderId, decimal Quantity, + decimal StopPrice, TimeInForce TimeInForce); + +/// +/// The note of one intervention: when, why, what the guard did, in the words of the diary. The engine writes it in the +/// diary at the restart and adopts the stops it lists. +/// +public sealed record GuardNote(DateTimeOffset At, string Reason, IReadOnlyList Lines, IReadOnlyList Placed) +{ + /// The engine was silent for the 2 minutes of SPEC 9.3. + public const string Silence = "silence"; + + /// The startup steps did not move for 10 minutes. + public const string Stalled = "stalled"; + + /// The engine ended without announcing it. + public const string Crash = "crash"; + + /// The intervention made the bot restart in "Ferma tutto" (third forced restart within an hour). + public bool StopFileCreated { get; init; } +} + +/// Reading and writing the guard's files: each write goes to a temporary file first, so a reader never sees half of it. +public static class GuardFiles +{ + public static void WriteState(GuardPaths paths, GuardState state) + { + ArgumentNullException.ThrowIfNull(paths); + Write(paths.StateFile, JsonSerializer.Serialize(state, GuardChannel.Json)); + } + + /// The last state the engine wrote; null when there is none or it cannot be read. + public static GuardState? ReadState(GuardPaths paths) + { + ArgumentNullException.ThrowIfNull(paths); + return Read(paths.StateFile); + } + + public static string WriteNote(GuardPaths paths, GuardNote note) + { + ArgumentNullException.ThrowIfNull(paths); + ArgumentNullException.ThrowIfNull(note); + var path = Path.Combine(paths.NotesFolder, note.At.UtcDateTime.ToString("yyyyMMdd'T'HHmmssfff'Z'", CultureInfo.InvariantCulture) + ".json"); + Write(path, JsonSerializer.Serialize(note, GuardChannel.Json)); + return path; + } + + /// The notes not yet taken by the engine, oldest first, with their files. + public static IReadOnlyList<(string Path, GuardNote Note)> ReadNotes(GuardPaths paths) + { + ArgumentNullException.ThrowIfNull(paths); + if (!Directory.Exists(paths.NotesFolder)) + { + return []; + } + + var notes = new List<(string, GuardNote)>(); + foreach (var file in Directory.GetFiles(paths.NotesFolder, "*.json").Order(StringComparer.Ordinal)) + { + if (Read(file) is { } note) + { + notes.Add((file, note)); + } + } + + return notes; + } + + private static T? Read(string path) + where T : class + { + try + { + return File.Exists(path) ? JsonSerializer.Deserialize(File.ReadAllText(path), GuardChannel.Json) : null; + } + catch (Exception ex) when (ex is IOException or JsonException or UnauthorizedAccessException) + { + return null; + } + } + + private static void Write(string path, string text) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var temporary = path + ".tmp"; + File.WriteAllText(temporary, text); + File.Move(temporary, path, overwrite: true); + } +} diff --git a/src/Anubis/Supervision/Guardian.cs b/src/Anubis/Supervision/Guardian.cs new file mode 100644 index 0000000..43fea09 --- /dev/null +++ b/src/Anubis/Supervision/Guardian.cs @@ -0,0 +1,499 @@ +using Anubis.Core; +using Anubis.Trading; +using Microsoft.Extensions.Logging; + +namespace Anubis.Supervision; + +/// The times of the guard (SPEC 9.3). The drill shortens them; the bot uses these. +public sealed record GuardOptions +{ + /// Silence of the main loop, after a successful start, that makes the guard intervene. + public TimeSpan Silence { get; init; } = TimeSpan.FromMinutes(2); + + /// Startup steps that do not move for this long make the guard intervene. + public TimeSpan StalledStart { get; init; } = TimeSpan.FromMinutes(10); + + public TimeSpan Poll { get; init; } = TimeSpan.FromSeconds(1); + + /// A round of the guard this late means its own clock jumped (the computer slept): the silence counts from now. + public TimeSpan ClockJump { get; init; } = TimeSpan.FromSeconds(30); + + /// How long an orderly stop may take before the engine is ended (the container waits 60 seconds). + public TimeSpan GracefulStop { get; init; } = TimeSpan.FromSeconds(55); + + public TimeSpan ForcedWindow { get; init; } = TimeSpan.FromHours(1); + + /// The forced restart, within , that restarts the engine in "Ferma tutto". + public int ForcedLimit { get; init; } = 3; + + public TimeSpan DownAlertEvery { get; init; } = TimeSpan.FromHours(1); + + /// Pauses before restarting an engine that keeps closing by itself: the first restart is immediate. + public IReadOnlyList CrashPauses { get; init; } = + [TimeSpan.Zero, TimeSpan.FromSeconds(10), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(15)]; +} + +/// How to start the engine: with the drill's fault or without, and where its messages go. +public sealed record EngineStart(bool Fault, Action OnMessage); + +/// The engine as a child process of the guard. +public interface IEngineProcess : IDisposable +{ + /// Completes with the exit code when the process ends. + Task Exited { get; } + + /// Asks for an orderly stop through the channel. + Task RequestStopAsync(CancellationToken cancellationToken); + + /// Ends the process at once, with everything it started. + void Kill(); +} + +/// +/// The guard (SPEC 9.3): parent of the engine, it listens to its beat and intervenes when the main loop is silent for +/// 2 minutes after a successful start, or the startup steps of 9.4 do not move for 10 minutes. Then, in this order: +/// ends the engine, reads positions and orders from the brokers, cancels the purchases and sales left open, puts back +/// the missing stops, leaves a note, warns, restarts the engine. Orderly stops are not reactions: they are passed on +/// and waited for, and the engine is not restarted. At the third forced restart within an hour the engine restarts in +/// "Ferma tutto". The guard never writes in the database. +/// +public sealed partial class Guardian(GuardPaths paths, IClock clock, GuardOptions options, Func start, + Func, IBrokerAccount?> brokers, Func, CancellationToken, Task> alert, ILogger logger) +{ + private readonly Lock _gate = new(); + private readonly List _forced = []; + private IEngineProcess? _engine; + private string _stage = GuardStages.Starting; + private string? _step; + private DateTimeOffset _lastBeat; + private DateTimeOffset _stepSince; + private bool _announced; + private IReadOnlyList _keys = []; + private DateTimeOffset _lastRound; + private DateTimeOffset? _downSince; + private DateTimeOffset? _nextDownAlert; + private DateTimeOffset? _restartAt; + private int _crashes; + + /// The stage the engine last declared. + public string Stage + { + get + { + lock (_gate) + { + return _stage; + } + } + } + + public int Interventions { get; private set; } + + /// The engine has a process now (not waiting for the pause after a crash). + public bool EngineAlive => _engine is { Exited.IsCompleted: false }; + + /// The notes written so far, for the drill and the tests. + public IReadOnlyList Notes => [.. _notes]; + + private readonly List _notes = []; + + /// Starts the engine; only for the drill. + public void Start(bool fault = false) => StartEngine(fault); + + /// A message from the engine. + public void Receive(GuardMessage message) + { + ArgumentNullException.ThrowIfNull(message); + var now = clock.UtcNow; + lock (_gate) + { + switch (message.Type) + { + case GuardMessageTypes.Beat: + var stage = message.Stage ?? _stage; + if (stage != _stage || message.Step != _step) + { + _stepSince = now; + } + + _stage = stage; + _step = message.Step; + _lastBeat = now; + if (stage is GuardStages.Running or GuardStages.Waiting) + { + // The engine is back: no more "it does not restart" alerts. + _downSince = null; + _nextDownAlert = null; + _crashes = 0; + } + + break; + case GuardMessageTypes.Keys: + _keys = message.Keys ?? []; + foreach (var value in _keys.SelectMany(k => k.Values)) + { + Redactor.Register(value); + } + + break; + case GuardMessageTypes.Stopping: + _announced = true; + LogAnnounced(logger, message.Reason ?? "-"); + break; + } + } + } + + /// Watches until the engine stops on purpose, or until asks the guard to stop it. + public async Task RunAsync(CancellationToken shutdown) + { + if (_engine is null && _restartAt is null) + { + StartEngine(fault: false); + } + + while (true) + { + var exited = _engine?.Exited ?? Task.FromResult(0); + try + { + await Task.WhenAny(Task.Delay(options.Poll, shutdown), _engine is null ? Task.Delay(Timeout.Infinite, shutdown) : exited).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + + if (shutdown.IsCancellationRequested) + { + return await StopAsync().ConfigureAwait(false); + } + + if (await RoundAsync(shutdown).ConfigureAwait(false) is { } code) + { + return code; + } + } + } + + /// One round of watching; the exit code when the engine stopped on purpose and the guard's work is over. + public async Task RoundAsync(CancellationToken cancellationToken) + { + var now = clock.UtcNow; + if (_lastRound != default && now - _lastRound > options.ClockJump) + { + // The guard's own clock jumped too (the computer slept): the engine's silence starts now, not then. + lock (_gate) + { + _lastBeat = now; + _stepSince = now; + } + } + + _lastRound = now; + if (_engine is null) + { + if (_restartAt is { } at && now >= at) + { + _restartAt = null; + StartEngine(fault: false); + } + + await DownAlertAsync(now, cancellationToken).ConfigureAwait(false); + return null; + } + + if (_engine.Exited.IsCompleted) + { + var code = await _engine.Exited.ConfigureAwait(false); + bool announced; + lock (_gate) + { + announced = _announced; + } + + if (announced) + { + LogEnded(logger, code); + return code; + } + + await InterveneAsync(GuardNote.Crash, cancellationToken).ConfigureAwait(false); + return null; + } + + string stage; + string? step; + DateTimeOffset beat; + DateTimeOffset since; + lock (_gate) + { + (stage, step, beat, since) = (_stage, _step, _lastBeat, _stepSince); + } + + if (stage == GuardStages.Running && now - beat > options.Silence) + { + await InterveneAsync(GuardNote.Silence, cancellationToken).ConfigureAwait(false); + } + else if (stage == GuardStages.Starting && now - since > options.StalledStart) + { + await InterveneAsync(GuardNote.Stalled, cancellationToken, step).ConfigureAwait(false); + } + + await DownAlertAsync(now, cancellationToken).ConfigureAwait(false); + return null; + } + + /// + /// The container is stopping: the engine gets "stop" and up to to close in + /// order, then it is ended. It is not restarted. + /// + public async Task StopAsync() + { + lock (_gate) + { + _announced = true; + } + + var engine = _engine; + if (engine is null) + { + return 0; + } + + try + { + using var asking = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await engine.RequestStopAsync(asking.Token).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or OperationCanceledException or ObjectDisposedException or InvalidOperationException) + { + // The channel is gone: the wait below ends the engine anyway. + } + + if (await Task.WhenAny(engine.Exited, Task.Delay(options.GracefulStop)).ConfigureAwait(false) != engine.Exited) + { + LogKilledAtStop(logger); + engine.Kill(); + } + + return await engine.Exited.ConfigureAwait(false); + } + + private void StartEngine(bool fault) + { + _engine?.Dispose(); + var now = clock.UtcNow; + lock (_gate) + { + _stage = GuardStages.Starting; + _step = null; + _lastBeat = now; + _stepSince = now; + _announced = false; + } + + _lastRound = now; + _engine = start(new EngineStart(fault, Receive)); + LogStarted(logger, fault); + } + + private async Task InterveneAsync(string reason, CancellationToken cancellationToken, string? step = null) + { + var engine = _engine!; + var now = clock.UtcNow; + Interventions++; + LogIntervention(logger, reason); + + // 1. The engine stops first: the guard touches orders only while it is stopped. + engine.Kill(); + try + { + await engine.Exited.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None).ConfigureAwait(false); + } + catch (TimeoutException) + { + LogStillAlive(logger); + } + + engine.Dispose(); + _engine = null; + + // 2-4. Each broker account: orders left waiting canceled, missing stops put back. + var lines = new List { Opening(reason, step) }; + var placed = new List(); + IReadOnlyList keys; + lock (_gate) + { + keys = _keys; + } + + var state = GuardFiles.ReadState(paths); + if (state is null) + { + lines.Add("Non ho trovato il file di stato del motore: posizioni e stop li rimette in ordine il motore alla ripartenza."); + } + else + { + foreach (var account in state.Accounts.Where(a => a.Positions.Count > 0 || a.Orders.Count > 0)) + { + lines.AddRange(await RepairAsync(state, account, keys, placed, cancellationToken).ConfigureAwait(false)); + } + } + + // The third forced restart within an hour: the engine restarts in "Ferma tutto" (SPEC 9.3). + _forced.RemoveAll(t => now - t >= options.ForcedWindow); + _forced.Add(now); + var stopAll = _forced.Count >= options.ForcedLimit; + if (stopAll) + { + WriteStopFile(now); + lines.Add("È il terzo riavvio forzato in un'ora: il motore riparte in \"Ferma tutto\". Per ripartire togli il file STOP e conferma dall'interfaccia."); + } + + // 5. The note for the diary, then the warning. + var note = new GuardNote(now, reason, lines, placed) { StopFileCreated = stopAll }; + try + { + GuardFiles.WriteNote(paths, note); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + LogNoteFailed(logger, ex); + } + + _notes.Add(note); + await AlertAsync(string.Join(Environment.NewLine, lines), cancellationToken).ConfigureAwait(false); + + // 6. The restart, never with the drill's fault. An engine that keeps closing by itself waits a little longer each time. + _downSince ??= now; + _nextDownAlert ??= now + options.DownAlertEvery; + var pause = TimeSpan.Zero; + if (reason == GuardNote.Crash) + { + _crashes++; + pause = options.CrashPauses[Math.Min(_crashes, options.CrashPauses.Count) - 1]; + } + + if (pause <= TimeSpan.Zero) + { + StartEngine(fault: false); + } + else + { + _restartAt = now + pause; + } + } + + private async Task> RepairAsync(GuardState state, GuardAccount account, IReadOnlyList keys, List placed, CancellationToken cancellationToken) + { + if (account.Environment == TradingEnvironments.Simulation) + { + return [$"{account.Name} è il simulatore interno: posizioni e stop li rimette in ordine il motore alla ripartenza."]; + } + + IBrokerAccount? broker; + try + { + broker = brokers(account, keys); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return [$"{account.Name}: non riesco a collegarmi al conto ({Redactor.Redact(ex.Message)}): ci pensa il motore alla ripartenza."]; + } + + if (broker is null) + { + return [$"{account.Name}: non ho le chiavi del conto, quindi non posso intervenire: ci pensa il motore alla ripartenza."]; + } + + try + { + var outcome = await GuardRepair.RunAsync(state, account, broker, clock, cancellationToken).ConfigureAwait(false); + placed.AddRange(outcome.Placed); + return [.. outcome.Lines]; + } + catch (BrokerException ex) + { + return [$"{account.Name}: non raggiungo il conto ({Redactor.Redact(ex.Message)}): ci pensa il motore alla ripartenza."]; + } + finally + { + (broker as IDisposable)?.Dispose(); + } + } + + private async Task DownAlertAsync(DateTimeOffset now, CancellationToken cancellationToken) + { + if (_downSince is not { } since || _nextDownAlert is not { } next || now < next) + { + return; + } + + _nextDownAlert = next + options.DownAlertEvery; + await AlertAsync($"Il motore non riparte da {DiaryText.Duration(now - since)}. Le posizioni hanno gli stop che ho rimesso io; " + + "quelli giornalieri sulle frazioni di azione scadono a fine seduta e senza il motore nessuno li rimette domani.", cancellationToken).ConfigureAwait(false); + } + + private async Task AlertAsync(string text, CancellationToken cancellationToken) + { + IReadOnlyList keys; + lock (_gate) + { + keys = _keys; + } + + try + { + await alert(text, keys, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogAlertFailed(logger, ex); + } + } + + private void WriteStopFile(DateTimeOffset now) + { + try + { + Directory.CreateDirectory(paths.DataRoot); + File.WriteAllText(paths.StopFile, $"Creato dalla guardia il {now:yyyy-MM-dd HH:mm} UTC: terzo riavvio forzato in un'ora.{Environment.NewLine}"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + LogNoteFailed(logger, ex); + } + } + + private string Opening(string reason, string? step) => reason switch + { + GuardNote.Silence => $"La guardia è intervenuta: il motore non dava segni di vita da {DiaryText.Duration(options.Silence)}. L'ho fermato, ho controllato i conti e l'ho riavviato.", + GuardNote.Stalled => $"La guardia è intervenuta: l'avvio del motore era fermo da {DiaryText.Duration(options.StalledStart)}" + + (step is null ? "" : $" al passo \"{step}\"") + ". L'ho fermato, ho controllato i conti e l'ho riavviato.", + _ => "La guardia è intervenuta: il motore si è chiuso senza avvisare. Ho controllato i conti e l'ho riavviato.", + }; + + [LoggerMessage(Level = LogLevel.Information, Message = "Engine started (drill fault: {Fault})")] + private static partial void LogStarted(ILogger logger, bool fault); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Guard intervention: {Reason}")] + private static partial void LogIntervention(ILogger logger, string reason); + + [LoggerMessage(Level = LogLevel.Information, Message = "Engine announced its stop: {Reason}")] + private static partial void LogAnnounced(ILogger logger, string reason); + + [LoggerMessage(Level = LogLevel.Information, Message = "Engine ended with code {Code}")] + private static partial void LogEnded(ILogger logger, int code); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Engine did not stop in time: ended")] + private static partial void LogKilledAtStop(ILogger logger); + + [LoggerMessage(Level = LogLevel.Error, Message = "Engine still alive after being ended")] + private static partial void LogStillAlive(ILogger logger); + + [LoggerMessage(Level = LogLevel.Error, Message = "Guard note or STOP file not written")] + private static partial void LogNoteFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Error, Message = "Guard alert not sent")] + private static partial void LogAlertFailed(ILogger logger, Exception exception); +} diff --git a/src/Anubis/Trading/DiaryText.cs b/src/Anubis/Trading/DiaryText.cs index c1103e7..58cec0a 100644 --- a/src/Anubis/Trading/DiaryText.cs +++ b/src/Anubis/Trading/DiaryText.cs @@ -150,6 +150,11 @@ public static class DiaryText /// A missed job left out at the restart, with its reason (SPEC 5.6). public static string Skipped(string what, string reason) => $"Saltato: {what}. {reason}"; + /// A purchase or sale left open at a start (SPEC 9.4): canceled, and proposed again. + public static string RestartedOrder(bool sale, string symbol) => sale + ? $"Alla ripartenza ho annullato la vendita di {Shown(symbol)} rimasta aperta: la vendita già decisa riparte subito con un ordine nuovo." + : $"Alla ripartenza ho annullato l'acquisto di {Shown(symbol)} rimasto aperto: lo ripropongo con le regole del recupero."; + /// An entry left out because the price moved too far from the planned one (SPEC 5.6). public static string EntryDropped(string symbol, decimal movedPercent, decimal limitPercent) => $"Scartato l'acquisto di {Shown(symbol)}: il prezzo si è mosso del {Format.Percent(movedPercent, 1)} dall'orario previsto, oltre il limite del {Format.Percent(limitPercent, 0)}."; diff --git a/src/Anubis/Trading/LiveEngine.cs b/src/Anubis/Trading/LiveEngine.cs index 932aace..9f58aaf 100644 --- a/src/Anubis/Trading/LiveEngine.cs +++ b/src/Anubis/Trading/LiveEngine.cs @@ -22,6 +22,9 @@ public static class LiveReasons public const string OrderInProgress = "order_in_progress"; public const string NothingHeld = "nothing_held"; public const string Recovered = "recovered"; + + /// A purchase or sale left open at a start, canceled by the steps of SPEC 9.4. + public const string Restart = "restart"; } /// States of a planned order. @@ -911,6 +914,55 @@ public sealed partial class LiveEngine : IDisposable await CloseAsync(order, refusal is null ? PlannedStates.Submitted : PlannedStates.Refused, refusal, execution, recovered, cancellationToken).ConfigureAwait(false); } + /// + /// The step of SPEC 9.4 that comes after the reconciliation: the purchases and sales left open on the account are + /// canceled (their stops come back at once), the exits already decided restart now with a new order, and the + /// interrupted entries are proposed again with their time and reference price, so the rules of SPEC 5.6 apply. + /// The answer is how many were proposed again. + /// + public async Task ResumeAsync(OrderManager manager, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(manager); + var canceled = await manager.CancelWorkingAsync(LiveReasons.Restart, cancellationToken: cancellationToken).ConfigureAwait(false); + var again = 0; + foreach (var execution in canceled) + { + var row = await _db.ReadAsync(s => s.FirstOrDefaultAsync( + "SELECT p.id, p.decision_id, p.strategy, p.account_id, p.instrument_id, p.side, p.purpose, p.quantity, p.exit_all, p.stop_level, p.stop_offset, " + + "p.reference_price, p.reason, p.due_at, p.valid_until, e.filled FROM planned_orders p JOIN executions e ON e.id = p.execution_id WHERE p.execution_id = @e", + r => new Interrupted(new Planned(r.Id("id"), r.Id("decision_id"), r.Text("strategy"), r.Id("account_id"), r.Id("instrument_id"), Wire.Side(r.Text("side")), + Wire.Purpose(r.Text("purpose")), r.Money("quantity"), r.Bool("exit_all"), r.MoneyOrNull("stop_level"), r.MoneyOrNull("stop_offset"), r.Money("reference_price"), + r.Text("reason"), r.Moment("due_at"), r.MomentOrNull("valid_until")), r.Money("filled")), ("e", execution)), cancellationToken).ConfigureAwait(false); + if (row is not { } found || (found.Order.Quantity - found.Filled <= 0m && !found.Order.ExitAll)) + { + continue; + } + + var (order, filled) = found; + var symbol = (await _catalog.GetAsync(order.InstrumentId, cancellationToken).ConfigureAwait(false))?.Symbol ?? "?"; + var now = _clock.UtcNow; + var sale = order.Side == OrderSide.Sell; + await _db.WriteAsync(async s => + { + await s.ExecuteAsync( + "INSERT INTO planned_orders (id, decision_id, strategy, account_id, instrument_id, side, purpose, quantity, exit_all, stop_level, stop_offset, reference_price, " + + "reason, due_at, valid_until, status, outcome, execution_id, created_at, updated_at) VALUES (@id, @d, @s, @a, @i, @side, @p, @q, @x, @sl, @so, @r, @reason, " + + "@due, @until, @st, NULL, NULL, @now, @now)", cancellationToken, + ("id", Ids.NewId(_clock)), ("d", order.DecisionId), ("s", order.Strategy), ("a", order.AccountId), ("i", order.InstrumentId), ("side", Wire.Text(order.Side)), + ("p", Wire.Text(order.Purpose)), ("q", Math.Max(0m, order.Quantity - filled)), ("x", order.ExitAll), ("sl", order.StopLevel), ("so", order.StopOffset), + ("r", order.Reference), ("reason", order.Reason), ("due", sale ? now : order.Due), ("until", order.ValidUntil), ("st", PlannedStates.Pending), ("now", now)) + .ConfigureAwait(false); + await Diary.WriteAsync(s, _clock, DiaryKinds.Orders, LiveReasons.Restart, DiaryText.RestartedOrder(sale, symbol), new { planned = order.Id, execution }, + cancellationToken).ConfigureAwait(false); + }, cancellationToken).ConfigureAwait(false); + again++; + } + + return again; + } + + private sealed record Interrupted(Planned Order, decimal Filled); + private static string OrderWords(Planned order, string symbol) => (order.Side == OrderSide.Buy ? "l'acquisto di " : "la vendita di ") + DiaryText.Shown(symbol); diff --git a/src/Anubis/Trading/OrderManager.Guard.cs b/src/Anubis/Trading/OrderManager.Guard.cs new file mode 100644 index 0000000..5e38687 --- /dev/null +++ b/src/Anubis/Trading/OrderManager.Guard.cs @@ -0,0 +1,84 @@ +using Anubis.Core; +using Anubis.Supervision; + +namespace Anubis.Trading; + +/// What the order manager gives to the guard and takes back from it (SPEC 9.3). +public sealed partial class OrderManager +{ + /// + /// Called right after a sale leaves, before the turn ends. Only the guard drill sets it, to block the main loop + /// with a sale halfway (ADR 0012). + /// + public Func? AfterSale { get; init; } + + /// + /// The account as the guard needs it: every position with its stop level on the tick and the names its stops + /// take, and the orders believed open. + /// + public async Task GuardAccountAsync(CancellationToken cancellationToken) + { + var positions = new List(); + foreach (var position in await _ledger.PositionsAsync(_account.AccountId, cancellationToken).ConfigureAwait(false)) + { + if (position.Quantity <= 0m || await _instruments(position.InstrumentId, cancellationToken).ConfigureAwait(false) is not { } instrument) + { + continue; + } + + var level = position.StopPrice is { } stop ? OrderRules.StopPrice(stop, OrderRules.TickOf(instrument.Spec, stop)) : (decimal?)null; + var key = ClientOrderIds.Key(StopKey(position.InstrumentId, position.Strategy)); + var next = await NextStopSequenceAsync(position.InstrumentId, position.Strategy, key, cancellationToken).ConfigureAwait(false); + positions.Add(new GuardPosition(position.InstrumentId, instrument.Symbol, position.Strategy, position.Quantity, level, OrderRules.LotOf(instrument.Spec), + instrument.IsCrypto, key, next)); + } + + var orders = new List(); + var open = await _db.ReadAsync(s => s.QueryAsync(OrderSelect + " WHERE account_id = @a AND status IN ('created', 'submitted', 'partially_filled')", ReadOrder, + ("a", _account.AccountId)), cancellationToken).ConfigureAwait(false); + foreach (var row in open) + { + var symbol = (await _instruments(row.InstrumentId, cancellationToken).ConfigureAwait(false))?.Symbol ?? ""; + orders.Add(new GuardOrder(row.ClientOrderId, row.InstrumentId, symbol, row.Strategy, row.Side, row.Purpose, row.Quantity, row.Stop)); + } + + return new GuardAccount(_account.AccountId, _account.Name, _account.Broker, _account.Environment, _broker.Capabilities.FractionalStopsAreDayOnly, positions, orders); + } + + /// + /// A stop the guard placed while the engine was stopped becomes one of the engine's stops: recorded, then read + /// again from the broker (it may have triggered meanwhile). False when it was already known. + /// + public async Task AdoptStopAsync(PlacedStop placed, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(placed); + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (await OrderByClientIdAsync(placed.ClientOrderId, cancellationToken).ConfigureAwait(false) is not null) + { + return false; + } + + var row = new OrderRow(Ids.NewId(_clock), null, placed.InstrumentId, placed.Strategy, placed.ClientOrderId, OrderSide.Sell, OrderKind.Stop, OrderPurpose.Stop, + placed.Quantity, null, placed.StopPrice, OrderStatus.Created, 0m, null, placed.StopPrice, null, placed.TimeInForce, false, _clock.UtcNow); + await _db.WriteAsync(async s => + { + await InsertOrderAsync(s, row, cancellationToken).ConfigureAwait(false); + await s.ExecuteAsync("UPDATE orders SET broker_order_id = @b, submitted_at = @now, reason = 'placed by the guard' WHERE id = @id", cancellationToken, + ("b", placed.BrokerOrderId), ("now", _clock.UtcNow), ("id", row.Id)).ConfigureAwait(false); + }, cancellationToken).ConfigureAwait(false); + if (await _broker.GetOrderAsync(placed.ClientOrderId, placed.Symbol, cancellationToken).ConfigureAwait(false) is { } remote) + { + await ApplyRemoteAsync(row, remote, "placed by the guard", cancellationToken).ConfigureAwait(false); + await SyncFillsCoreAsync(cancellationToken).ConfigureAwait(false); + } + + return true; + } + finally + { + _gate.Release(); + } + } +} diff --git a/src/Anubis/Trading/OrderManager.cs b/src/Anubis/Trading/OrderManager.cs index 8c68e48..c8dbb5d 100644 --- a/src/Anubis/Trading/OrderManager.cs +++ b/src/Anubis/Trading/OrderManager.cs @@ -375,14 +375,15 @@ public sealed partial class OrderManager : IDisposable /// /// Cancels the executions under way, except protective sales (SPEC 9.3, "Ferma tutto"): the open order of each is - /// canceled with the broker, and the stops it had removed come back at once for the quantity held. + /// canceled with the broker, and the stops it had removed come back at once for the quantity held. The answer is + /// the executions canceled. /// - public async Task CancelWorkingAsync(string reason, Func? strategy = null, CancellationToken cancellationToken = default) + public async Task> CancelWorkingAsync(string reason, Func? strategy = null, CancellationToken cancellationToken = default) { await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { - var count = 0; + var canceled = new List(); foreach (var exec in await WorkingAsync(cancellationToken).ConfigureAwait(false)) { if (exec.Purpose == OrderPurpose.Protective || (strategy is not null && !strategy(exec.Strategy))) @@ -400,10 +401,10 @@ public sealed partial class OrderManager : IDisposable await SyncFillsCoreAsync(cancellationToken).ConfigureAwait(false); await FinishAsync(exec, ExecutionStates.Canceled, reason, instrument, cancellationToken).ConfigureAwait(false); - count++; + canceled.Add(exec.Id); } - return count; + return canceled; } finally { @@ -626,6 +627,10 @@ public sealed partial class OrderManager : IDisposable } }, cancellationToken).ConfigureAwait(false); await SendAsync(exec, row, instrument, cancellationToken).ConfigureAwait(false); + if (exec.Side == OrderSide.Sell && exec.Purpose == OrderPurpose.Exit && AfterSale is { } afterSale) + { + await afterSale(cancellationToken).ConfigureAwait(false); + } } private async Task SendAsync(ExecRow exec, OrderRow row, Instrument instrument, CancellationToken cancellationToken) diff --git a/src/Anubis/Trading/ReasonCatalog.cs b/src/Anubis/Trading/ReasonCatalog.cs index 7571e11..951a1eb 100644 --- a/src/Anubis/Trading/ReasonCatalog.cs +++ b/src/Anubis/Trading/ReasonCatalog.cs @@ -83,6 +83,7 @@ public static class ReasonCatalog ["superseded"] = "Sostituito da una decisione più recente.", ["order_in_progress"] = "Un ordine sullo stesso strumento era ancora in corso.", ["nothing_held"] = "Niente da vendere: la posizione era già chiusa.", + ["restart"] = "Annullato alla ripartenza del bot: se era ancora dovuto, è stato riproposto.", ["recovered"] = "Recuperato dopo un'assenza.", }; diff --git a/src/Anubis/Trading/TradingService.cs b/src/Anubis/Trading/TradingService.cs index 14a45bc..bd44a26 100644 --- a/src/Anubis/Trading/TradingService.cs +++ b/src/Anubis/Trading/TradingService.cs @@ -4,6 +4,7 @@ using Anubis.Core; using Anubis.Data; using Anubis.Db; using Anubis.Risk; +using Anubis.Supervision; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -192,6 +193,9 @@ public sealed partial class TradingService : IDisposable private volatile bool _started; + /// The startup steps of SPEC 9.4 are done and the loop runs. + public bool IsStarted => _started; + /// The live engine (SPEC 5.5, issue #47): the decisions at their times, the planned orders, the recovery. public LiveEngine Engine { get; } @@ -216,6 +220,23 @@ public sealed partial class TradingService : IDisposable public TimeSpan TickInterval { get; init; } = TimeSpan.FromSeconds(5); + /// The channel to the guard (SPEC 9.3): startup steps and the beat of the loop. Null without a guard. + public GuardLink? Guard { get; init; } + + /// Where the state file for the guard is written and its notes are read. Null without a guard. + public GuardPaths? GuardFiles { get; init; } + + /// The guard drill's fault (ADR 0012): or nothing. + public string? Fault { get; init; } + + /// How often the loop writes the state file for the guard. + public TimeSpan GuardStateEvery { get; init; } = TimeSpan.FromSeconds(10); + + /// A broker clock further than this from the computer's is reported at the start (SPEC 9.4). + public TimeSpan ClockTolerance { get; init; } = TimeSpan.FromSeconds(5); + + private DateTimeOffset _guardStateAt; + public IReadOnlyCollection Managers => [.. _managers.Values]; public OrderManager? ManagerOf(Guid accountId) => _managers.GetValueOrDefault(accountId); @@ -233,14 +254,21 @@ public sealed partial class TradingService : IDisposable return accounts; } - /// Accounts, brokers and order managers; then the simulator catches up, keys are checked and every account reconciled. + /// + /// The startup of SPEC 9.4 after the migrations: brokers, clock, corporate actions, then for every account the + /// complete reconciliation, the purchases and sales left open canceled (exits restart, entries are proposed again) + /// and every position checked for its stop. Each step is told to the guard. + /// public async Task StartAsync(CancellationToken cancellationToken) { + Guard?.Step("conti e strategie"); var accounts = await PrepareAsync(cancellationToken).ConfigureAwait(false); var alpaca = await AlpacaKeysAsync(cancellationToken).ConfigureAwait(false); var okx = await OkxKeysAsync(cancellationToken).ConfigureAwait(false); await RefreshSpecsAsync(cancellationToken).ConfigureAwait(false); await RefreshLimitsAsync(cancellationToken).ConfigureAwait(false); + Guard?.Step("connessione ai broker"); + var hang = Fault == GuardChannel.HangMidSale; foreach (var account in accounts) { var broker = Build(account, alpaca, okx); @@ -259,10 +287,16 @@ public sealed partial class TradingService : IDisposable Gate = Risk is null ? null : Risk.GateAsync, LivePermit = Stages.LiveRefusalAsync, PurchasePriceCheck = (limit, quote) => RiskRules.TechnicalRefusal(_clock.UtcNow, quote.At, quote.Mid, limit, null, _limits), + + // The guard drill only: the main loop stops here for good, with a sale halfway. + AfterSale = hang ? _ => Task.Delay(Timeout.Infinite, CancellationToken.None) : null, }; } await GuardAsync("simulatore", () => FundSimulatorsAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + Guard?.Step("orologio"); + await GuardAsync("orologio", () => CheckClocksAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + Guard?.Step("azioni societarie"); // SPEC 5.6: corporate actions come first, so the stops the reconciliation puts back already know about a split. if (_market is not null) @@ -271,6 +305,7 @@ public sealed partial class TradingService : IDisposable } await GuardAsync("azioni societarie nei registri", () => CorporateActionsAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + var notes = GuardFiles is null ? [] : Supervision.GuardFiles.ReadNotes(GuardFiles); foreach (var manager in _managers.Values) { await GuardAsync("avvio " + manager.Account.Name, async () => @@ -284,12 +319,104 @@ public sealed partial class TradingService : IDisposable await Reconciler.CheckKeyAsync(manager, cancellationToken).ConfigureAwait(false); } + // The stops the guard put back while the engine was stopped become the engine's own, before anything compares. + foreach (var placed in notes.SelectMany(n => n.Note.Placed).Where(p => p.AccountId == manager.Account.AccountId)) + { + await manager.AdoptStopAsync(placed, cancellationToken).ConfigureAwait(false); + } + + Guard?.Step("riconciliazione " + manager.Account.Name); await Reconciler.ReconcileAsync(manager, cancellationToken).ConfigureAwait(false); + Guard?.Step("ordini rimasti aperti " + manager.Account.Name); + await Engine.ResumeAsync(manager, cancellationToken).ConfigureAwait(false); + Guard?.Step("stop " + manager.Account.Name); + await manager.AlignStopsAsync(cancellationToken).ConfigureAwait(false); }, cancellationToken).ConfigureAwait(false); } + await GuardAsync("note della guardia", () => TakeGuardNotesAsync(notes, cancellationToken), cancellationToken).ConfigureAwait(false); await GuardAsync("patrimonio", () => PhotographAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + await GuardAsync("stato per la guardia", () => WriteGuardStateAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + await GuardAsync("chiavi per la guardia", () => SendGuardKeysAsync(alpaca, okx, cancellationToken), cancellationToken).ConfigureAwait(false); _started = true; + Guard?.Set(GuardStages.Running); + } + + /// + /// What the guard needs if the engine stops answering (SPEC 9.3): positions, stop levels and open orders of every + /// account. Written to a temporary file first, so the guard never reads half of it. + /// + public async Task WriteGuardStateAsync(CancellationToken cancellationToken) + { + if (GuardFiles is null) + { + return; + } + + var accounts = new List(); + foreach (var manager in _managers.Values) + { + accounts.Add(await manager.GuardAccountAsync(cancellationToken).ConfigureAwait(false)); + } + + Supervision.GuardFiles.WriteState(GuardFiles, new GuardState(_clock.UtcNow, InstallationCode, accounts)); + _guardStateAt = _clock.UtcNow; + } + + // The guard's notes go in the diary once their stops are adopted; then they are gone. + private async Task TakeGuardNotesAsync(IReadOnlyList<(string Path, GuardNote Note)> notes, CancellationToken cancellationToken) + { + foreach (var (path, note) in notes) + { + await _db.WriteAsync(s => Diary.WriteAsync(s, _clock, DiaryKinds.System, "guard_intervention", string.Join(" ", note.Lines), + new { at = note.At, reason = note.Reason, stops = note.Placed.Count, stopFile = note.StopFileCreated }, cancellationToken), cancellationToken).ConfigureAwait(false); + File.Delete(path); + } + } + + // The keys reach the guard in memory only: it needs them to protect the positions while the engine is stopped. + private async Task SendGuardKeysAsync(AlpacaCredentials? alpaca, OkxCredentials? okx, CancellationToken cancellationToken) + { + if (Guard is null) + { + return; + } + + var keys = new List(); + if (alpaca is not null) + { + keys.Add(new GuardKeys(Venues.Alpaca, TradingEnvironments.Demo, [alpaca.KeyId, alpaca.Secret])); + } + + if (okx is not null) + { + keys.Add(new GuardKeys(Venues.Okx, TradingEnvironments.Demo, [okx.Key, okx.Secret, okx.Passphrase])); + } + + var token = await _secrets.RevealAsync("TELEGRAM_BOT_TOKEN", cancellationToken).ConfigureAwait(false); + var chat = await _secrets.RevealAsync("TELEGRAM_CHAT_ID", cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(token) && !string.IsNullOrWhiteSpace(chat)) + { + keys.Add(new GuardKeys(GuardKeys.Telegram, "", [token, chat])); + } + + Guard.SendKeys(keys); + } + + // SPEC 9.4: the clock of each broker against the computer's. A large gap is said, not corrected: the computer's clock is the system's. + private async Task CheckClocksAsync(CancellationToken cancellationToken) + { + foreach (var manager in _managers.Values.Where(m => m.Broker is not PaperBroker)) + { + var remote = await manager.Broker.ServerTimeAsync(cancellationToken).ConfigureAwait(false); + var gap = (remote - _clock.UtcNow).Duration(); + if (gap > ClockTolerance) + { + await _db.WriteAsync(s => Diary.NotifyAsync(s, _clock, "clock_gap", + $"L'orologio di questo computer differisce di {Math.Round(gap.TotalSeconds)} secondi da quello di {manager.Account.Name}: controlla la sincronizzazione dell'ora.", + cancellationToken), cancellationToken).ConfigureAwait(false); + } + } } /// @@ -377,6 +504,12 @@ public sealed partial class TradingService : IDisposable } await TickAllAsync(cancellationToken).ConfigureAwait(false); + Guard?.Beat(); + if (GuardFiles is not null && _clock.UtcNow - _guardStateAt >= GuardStateEvery) + { + await GuardAsync("stato per la guardia", () => WriteGuardStateAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + } + await Task.Delay(TickInterval, cancellationToken).ConfigureAwait(false); } } diff --git a/tests/Anubis.Tests/GuardTests.cs b/tests/Anubis.Tests/GuardTests.cs new file mode 100644 index 0000000..036cb29 --- /dev/null +++ b/tests/Anubis.Tests/GuardTests.cs @@ -0,0 +1,414 @@ +using Anubis.Core; +using Anubis.Data; +using Anubis.Supervision; +using Anubis.Trading; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Anubis.Tests; + +/// The guard (issue #48, SPEC 9.3, 9.4): when it intervenes, what it does on the broker, and the restart. +[TestClass] +public sealed class GuardTests +{ + private static readonly CancellationToken _none = CancellationToken.None; + private static readonly TimeSpan _minute = TimeSpan.FromMinutes(1); + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task TwoMinutesOfSilenceAfterAStartMakeTheGuardInterveneAndRestartWithoutTheFault() + { + using var g = new GuardRig(fault: true); + g.Engine.Beat(GuardStages.Starting, "database"); + await g.PassAsync(TimeSpan.FromMinutes(5)).ConfigureAwait(false); + g.Engine.Beat(GuardStages.Running); + await g.PassAsync(TimeSpan.FromMinutes(10), () => g.Engine.Beat(GuardStages.Running)).ConfigureAwait(false); + await g.PassAsync(TimeSpan.FromSeconds(110)).ConfigureAwait(false); + Assert.AreEqual(0, g.Guardian.Interventions, "Five minutes of startup, a loop that beats, then 1:50 of silence: nothing yet."); + + await g.PassAsync(TimeSpan.FromSeconds(20)).ConfigureAwait(false); + Assert.AreEqual(1, g.Guardian.Interventions); + Assert.IsTrue(g.Engines[0].Killed); + Assert.HasCount(2, g.Engines); + Assert.IsTrue(g.Engines[0].Start.Fault, "The first engine had the drill's fault…"); + Assert.IsFalse(g.Engines[1].Start.Fault, "…the one the guard restarts never has it."); + Assert.Contains("non dava segni di vita da 2 minuti", g.Alerts.Single()); + var note = GuardFiles.ReadNotes(g.Paths).Single().Note; + Assert.AreEqual(GuardNote.Silence, note.Reason); + Assert.Contains("Non ho trovato il file di stato", string.Join(" ", note.Lines)); + Assert.IsFalse(File.Exists(g.Paths.StopFile)); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task TheErrorPageIsNeverAReasonToIntervene() + { + using var g = new GuardRig(); + g.Engine.Beat(GuardStages.Waiting); + await g.PassAsync(TimeSpan.FromHours(1)).ConfigureAwait(false); + Assert.AreEqual(0, g.Guardian.Interventions); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task StartupStepsThatDoNotMoveForTenMinutesMakeTheGuardIntervene() + { + using var g = new GuardRig(); + g.Engine.Beat(GuardStages.Starting, "migrazioni e identità"); + await g.PassAsync(TimeSpan.FromMinutes(9)).ConfigureAwait(false); + g.Engine.Beat(GuardStages.Starting, "riconciliazione OKX demo"); + await g.PassAsync(TimeSpan.FromMinutes(9), () => g.Engine.Beat(GuardStages.Starting, "riconciliazione OKX demo")).ConfigureAwait(false); + Assert.AreEqual(0, g.Guardian.Interventions, "A step that moves resets the count; the same step repeated does not."); + + await g.PassAsync(TimeSpan.FromMinutes(2)).ConfigureAwait(false); + Assert.AreEqual(1, g.Guardian.Interventions); + Assert.Contains("al passo \"riconciliazione OKX demo\"", g.Alerts.Single()); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task AJumpOfTheGuardsOwnClockIsNotSilence() + { + using var g = new GuardRig(); + g.Engine.Beat(GuardStages.Running); + await g.PassAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); + + // The computer slept for a quarter of an hour: guard and engine both stopped. + g.Clock.Advance(TimeSpan.FromMinutes(15)); + await g.Guardian.RoundAsync(_none).ConfigureAwait(false); + await g.PassAsync(TimeSpan.FromSeconds(110)).ConfigureAwait(false); + Assert.AreEqual(0, g.Guardian.Interventions); + + await g.PassAsync(TimeSpan.FromSeconds(20)).ConfigureAwait(false); + Assert.AreEqual(1, g.Guardian.Interventions, "After the jump the silence counts again from the wake-up."); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task TheContainersStopIsPassedOnAndNeverRestarted() + { + using var g = new GuardRig(); + g.Engine.Beat(GuardStages.Running); + Assert.AreEqual(0, await g.Guardian.StopAsync().ConfigureAwait(false)); + Assert.IsTrue(g.Engine.StopAsked); + Assert.IsFalse(g.Engine.Killed); + Assert.HasCount(1, g.Engines); + + // An engine that does not close in time is ended. + using var slow = new GuardRig(options: new GuardOptions { GracefulStop = TimeSpan.FromMilliseconds(200) }); + slow.Engine.ExitOnStop = false; + Assert.AreEqual(137, await slow.Guardian.StopAsync().ConfigureAwait(false)); + Assert.IsTrue(slow.Engine.Killed); + Assert.AreEqual(0, slow.Guardian.Interventions); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task AStopTheEngineAnnouncesEndsTheGuardWithoutARestart() + { + using var g = new GuardRig(); + g.Engine.Beat(GuardStages.Running); + g.Engine.Start.OnMessage(new GuardMessage(GuardMessageTypes.Stopping) { Reason = "uscita" }); + g.Engine.Exit(0); + Assert.AreEqual(0, await g.Guardian.RoundAsync(_none).ConfigureAwait(false)); + Assert.HasCount(1, g.Engines); + Assert.AreEqual(0, g.Guardian.Interventions); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task TheThirdForcedRestartWithinAnHourRestartsInStopEverything() + { + using var g = new GuardRig(); + for (var i = 1; i <= 3; i++) + { + g.Engine.Beat(GuardStages.Running); + await g.PassAsync(TimeSpan.FromMinutes(3)).ConfigureAwait(false); + Assert.AreEqual(i, g.Guardian.Interventions); + Assert.AreEqual(i == 3, File.Exists(g.Paths.StopFile), $"After intervention {i}."); + } + + var last = GuardFiles.ReadNotes(g.Paths)[^1].Note; + Assert.IsTrue(last.StopFileCreated); + Assert.Contains("\"Ferma tutto\"", g.Alerts[^1]); + + // Spread over more than an hour, three are not three within an hour. + using var spread = new GuardRig(); + for (var i = 0; i < 3; i++) + { + spread.Engine.Beat(GuardStages.Running); + await spread.PassAsync(TimeSpan.FromMinutes(3)).ConfigureAwait(false); + spread.Engine.Beat(GuardStages.Running); + await spread.PassAsync(TimeSpan.FromMinutes(30), () => spread.Engine.Beat(GuardStages.Running)).ConfigureAwait(false); + } + + Assert.AreEqual(3, spread.Guardian.Interventions); + Assert.IsFalse(File.Exists(spread.Paths.StopFile)); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task AnEngineThatKeepsClosingIsRestartedWithPausesAndTheUserIsRemindedEveryHour() + { + using var g = new GuardRig(options: new GuardOptions { StalledStart = TimeSpan.FromHours(5) }); + g.Engine.Exit(1); + await g.Guardian.RoundAsync(_none).ConfigureAwait(false); + Assert.HasCount(2, g.Engines, "The first restart is immediate."); + Assert.Contains("si è chiuso senza avvisare", g.Alerts[0]); + + g.Engine.Exit(1); + await g.Guardian.RoundAsync(_none).ConfigureAwait(false); + Assert.HasCount(2, g.Engines, "The second waits ten seconds."); + await g.PassAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); + Assert.HasCount(3, g.Engines); + + // It starts, but never gets to run: every hour a reminder about the stops on fractions of shares. + await g.PassAsync(TimeSpan.FromMinutes(61)).ConfigureAwait(false); + var reminder = g.Alerts.Single(a => a.Contains("non riparte", StringComparison.Ordinal)); + Assert.Contains("frazioni di azione scadono a fine seduta", reminder); + g.Engine.Beat(GuardStages.Running); + await g.PassAsync(TimeSpan.FromMinutes(70), () => g.Engine.Beat(GuardStages.Running)).ConfigureAwait(false); + Assert.AreEqual(1, g.Alerts.Count(a => a.Contains("non riparte", StringComparison.Ordinal)), "Once it runs, the reminders stop."); + } + + /// + /// The acceptance test of issue #48 on a broker account: the main loop blocks right after a sale leaves, with the + /// stop already removed. The guard stops the engine, cancels the sale, puts the stop back for the quantity held and + /// restarts the engine, which adopts the guard's stop and finds the account in order. + /// + [TestMethod] + [TestCategory(Category.Simulation)] + public async Task ALoopBlockedWithASaleHalfwayGetsItsStopBackAndTheEngineRestarts() + { + await using var rig = await TradingRig.CreateAsync().ConfigureAwait(false); + var account = await rig.AccountAsync(Venues.Okx, TradingEnvironments.Demo).ConfigureAwait(false); + var btc = await rig.InstrumentAsync(TradingRig.Btc).ConfigureAwait(false); + var paper = rig.Paper(PaperOptions.Okx(), account.AccountId); + await paper.DepositAsync(100_000m, "EUR", _none).ConfigureAwait(false); + await rig.Ledger.RecordCapitalAsync(account.AccountId, StrategyNames.CryptoTrend, 100_000m, "EUR", "test", _none).ConfigureAwait(false); + rig.Quote(btc.Symbol, 60000m, 60010m); + + // The demo account at the broker, which the guard reaches on its own. + var remote = new FaultyBroker(paper, TradingEnvironments.Demo); + using var hang = new CancellationTokenSource(); + var sold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + OrderManager Engine(Func? afterSale) => new(rig.Db, rig.Clock, rig.Ledger, remote, account, TradingRig.Installation, rig.FindAsync, rig.QuoteAsync) + { + ConfirmPrice = rig.QuoteAsync, + MarketOpen = (_, _) => Task.FromResult(true), + AfterSale = afterSale, + }; + + using var first = Engine(async _ => + { + sold.TrySetResult(); + await Task.Delay(Timeout.Infinite, hang.Token).ConfigureAwait(false); + }); + await first.SubmitAsync(new TradeIntent(btc.Id, StrategyNames.CryptoTrend, OrderSide.Buy, 0.01m, OrderPurpose.Entry, Ids.NewId(rig.Clock)) + { + StopLevel = 57000m, + StartAggressive = true, + }, _none).ConfigureAwait(false); + await rig.TurnsAsync(first, 3, _minute).ConfigureAwait(false); + var held = (await rig.Ledger.PositionAsync(account.AccountId, btc.Id, StrategyNames.CryptoTrend, _none).ConfigureAwait(false))!.Quantity; + Assert.AreEqual(held, (await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false)).Sum(o => o.Quantity)); + + var folder = Directory.CreateTempSubdirectory("anubis-guard-").FullName; + try + { + var paths = new GuardPaths(folder); + GuardFiles.WriteState(paths, new GuardState(rig.Clock.UtcNow, TradingRig.Installation, [await first.GuardAccountAsync(_none).ConfigureAwait(false)])); + + // The sale: the stop goes, the order leaves, and the loop never comes back. + await first.SubmitAsync(new TradeIntent(btc.Id, StrategyNames.CryptoTrend, OrderSide.Sell, held, OrderPurpose.Exit, Ids.NewId(rig.Clock)), _none).ConfigureAwait(false); + var blocked = Task.Run(() => first.TickAsync(hang.Token)); + await sold.Task.WaitAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); + Assert.IsEmpty(await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false), "Halfway: the stop is gone…"); + Assert.HasCount(1, await OpenAsync(remote, OrderPurpose.Exit).ConfigureAwait(false), "…and the sale waits on the book."); + + // The guard: two minutes of silence, then it works on the account with the keys it was given. + var engines = new List(); + var guardian = new Guardian(paths, rig.Clock, new GuardOptions(), s => + { + var engine = new FakeEngine(s); + engines.Add(engine); + return engine; + }, (a, keys) => a.AccountId == account.AccountId && keys.Count > 0 ? remote : null, (_, _, _) => Task.CompletedTask, NullLogger.Instance); + guardian.Start(); + engines[0].Start.OnMessage(new GuardMessage(GuardMessageTypes.Keys) { Keys = [new GuardKeys(Venues.Okx, TradingEnvironments.Demo, ["k", "s", "p"])] }); + engines[0].Beat(GuardStages.Running); + for (var i = 0; i < 13 && guardian.Interventions == 0; i++) + { + rig.Clock.Advance(TimeSpan.FromSeconds(10)); + await guardian.RoundAsync(_none).ConfigureAwait(false); + } + + Assert.AreEqual(1, guardian.Interventions); + Assert.IsTrue(engines[0].Killed); + Assert.HasCount(2, engines, "The guard restarted the engine."); + Assert.IsEmpty(await OpenAsync(remote, OrderPurpose.Exit).ConfigureAwait(false), "The sale left waiting is canceled."); + var stops = await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false); + Assert.AreEqual((held, 57000m), (stops.Sum(o => o.Quantity), stops.Single().StopPrice), "The stop is back for the whole quantity."); + var note = GuardFiles.ReadNotes(paths).Single().Note; + Assert.AreEqual(stops.Single().ClientOrderId, note.Placed.Single().ClientOrderId); + Assert.IsTrue(note.Lines.Any(l => l.Contains("Ho annullato la vendita di BTC", StringComparison.Ordinal)), string.Join(" | ", note.Lines)); + + // The engine starts again (SPEC 9.4): it adopts the guard's stop, reconciles, cancels what it had left open. + using var second = Engine(null); + using var live = new LiveEngine(rig.Db, rig.Clock, rig.Ledger, new StageBook(rig.Db, rig.Clock, TradingConsent.None, (_, _, _) => Task.FromResult(false)), + _ => second, rig.QuoteAsync); + Assert.IsTrue(await second.AdoptStopAsync(note.Placed.Single(), _none).ConfigureAwait(false)); + var report = await new Reconciler(rig.Db, rig.Clock, rig.Ledger).ReconcileAsync(second, _none).ConfigureAwait(false); + await live.ResumeAsync(second, _none).ConfigureAwait(false); + await second.AlignStopsAsync(_none).ConfigureAwait(false); + engines[1].Beat(GuardStages.Running); + await rig.TurnsAsync(second, 2, _minute).ConfigureAwait(false); + + // The only thing the engine finds changed is the sale the guard canceled; nothing is left different. + Assert.IsEmpty(report.Differences, string.Join(" | ", report.Differences.Select(d => d.Message))); + Assert.AreEqual(OrderPurpose.Exit, ClientOrderIds.Parse(report.Healed.Single().Split(':')[1])?.Purpose, report.Healed.Single()); + Assert.AreEqual(GuardStages.Running, guardian.Stage); + var after = await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false); + Assert.AreEqual(stops.Single().ClientOrderId, after.Single().ClientOrderId, "The guard's stop stays: no second stop on the same quantity."); + var sale = await rig.QueryAsync("SELECT status, reason FROM executions WHERE side = 'sell'", r => r.Text("status") + "/" + r.TextOrNull("reason")).ConfigureAwait(false); + Assert.AreEqual("canceled/" + LiveReasons.Restart, sale.Single()); + var adopted = await rig.QueryAsync("SELECT status, reason FROM orders WHERE client_order_id = @c", r => r.Text("status") + "/" + r.TextOrNull("reason"), + ("c", after.Single().ClientOrderId)).ConfigureAwait(false); + Assert.AreEqual("submitted/placed by the guard", adopted.Single()); + Assert.AreEqual((held, 57000m), await rig.Ledger.PositionAsync(account.AccountId, btc.Id, StrategyNames.CryptoTrend, _none).ConfigureAwait(false) is { } p + ? (p.Quantity, p.StopPrice!.Value) : default); + + await hang.CancelAsync().ConfigureAwait(false); + await Assert.ThrowsAsync(() => blocked).ConfigureAwait(false); + } + finally + { + Directory.Delete(folder, recursive: true); + } + } + + /// + /// The same proof with real processes (the drill that also runs in the Docker image): the real guard, the real + /// engine as its child, a sale blocked halfway on the internal simulator, then the restart by SPEC 9.4. + /// + [TestMethod] + [TestCategory(Category.Integration)] + [Timeout(600_000, CooperativeCancellation = true)] + public async Task TheGuardDrillPassesWithTheRealEngineAsChildProcess() + { + if (TestDatabase.Engine == "postgres") + { + Assert.Inconclusive("The drill has its own SQLite database: it runs in the SQLite pass."); + } + + var folder = Directory.CreateTempSubdirectory("anubis-drill-").FullName; + try + { + var result = await GuardDrill.RunInAsync(folder, Path.Combine(AppContext.BaseDirectory, "Anubis.Server.dll"), GuardDrill.Options, + line => Console.WriteLine(line), _none).ConfigureAwait(false); + Assert.IsTrue(result.Passed, string.Join(Environment.NewLine, result.Lines)); + } + finally + { + try + { + Directory.Delete(folder, recursive: true); + } + catch (IOException) + { + } + } + } + + private static async Task> OpenAsync(FaultyBroker broker, OrderPurpose purpose) => + [.. (await broker.GetOpenOrdersAsync(_none).ConfigureAwait(false)).Where(o => ClientOrderIds.Parse(o.ClientOrderId)?.Purpose == purpose)]; + + /// The engine process as the guard sees it, driven by the test. + private sealed class FakeEngine(EngineStart start) : IEngineProcess + { + private readonly TaskCompletionSource _exit = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public EngineStart Start { get; } = start; + + public Task Exited => _exit.Task; + + public bool StopAsked { get; private set; } + + public bool Killed { get; private set; } + + public bool ExitOnStop { get; set; } = true; + + public Task RequestStopAsync(CancellationToken cancellationToken) + { + StopAsked = true; + if (ExitOnStop) + { + Start.OnMessage(new GuardMessage(GuardMessageTypes.Stopping) { Reason = "uscita" }); + _exit.TrySetResult(0); + } + + return Task.CompletedTask; + } + + public void Kill() + { + Killed = true; + _exit.TrySetResult(137); + } + + public void Exit(int code) => _exit.TrySetResult(code); + + public void Beat(string stage, string? step = null) => Start.OnMessage(GuardMessage.Beat(stage, step)); + + public void Dispose() + { + } + } + + /// A guard on a manual clock with fake engines, a data folder of its own and the alerts it sends. + private sealed class GuardRig : IDisposable + { + public GuardRig(bool fault = false, GuardOptions? options = null) + { + Folder = Directory.CreateTempSubdirectory("anubis-guard-").FullName; + Paths = new GuardPaths(Folder); + Guardian = new Guardian(Paths, Clock, options ?? new GuardOptions(), s => + { + var engine = new FakeEngine(s); + Engines.Add(engine); + return engine; + }, (_, _) => null, (text, _, _) => + { + Alerts.Add(text); + return Task.CompletedTask; + }, NullLogger.Instance); + Guardian.Start(fault); + } + + public ManualClock Clock { get; } = new(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero)); + + public string Folder { get; } + + public GuardPaths Paths { get; } + + public Guardian Guardian { get; } + + public List Engines { get; } = []; + + public List Alerts { get; } = []; + + public FakeEngine Engine => Engines[^1]; + + /// Time passes in rounds of ten seconds; runs before every round. + public async Task PassAsync(TimeSpan span, Action? each = null) + { + for (var t = TimeSpan.Zero; t < span; t += TimeSpan.FromSeconds(10)) + { + Clock.Advance(TimeSpan.FromSeconds(10)); + each?.Invoke(); + await Guardian.RoundAsync(_none).ConfigureAwait(false); + } + } + + public void Dispose() => Directory.Delete(Folder, recursive: true); + } +} diff --git a/tools/Anubis.Tasks/Program.cs b/tools/Anubis.Tasks/Program.cs index 2393bb6..e981ce2 100644 --- a/tools/Anubis.Tasks/Program.cs +++ b/tools/Anubis.Tasks/Program.cs @@ -39,7 +39,7 @@ int Test(string engine, string[] extra) ["test", "Anubis.slnx", "--filter", "TestCategory!=LiveDemo", "--logger", "console;verbosity=normal", .. extra], env) ?? 0; } -// What the remote CI does, run locally: check on SQLite, tests on PostgreSQL, image build. +// What the remote CI does, run locally: check on SQLite, tests on PostgreSQL, image build, the guard drill in the image. int Ci() { var failed = Check(); @@ -70,9 +70,14 @@ int Ci() } } - return failed != 0 ? failed : BuildImage("ci"); + failed = failed != 0 ? failed : BuildImage("ci"); + return failed != 0 ? failed : GuardDrill("ci"); } +// The guard drill inside the image (issue #48): the real guard and the real engine, a sale blocked halfway, the restart. +int GuardDrill(string tag) => + Step("Esercitazione della guardia nell'immagine", "docker", ["run", "--rm", $"gitea.encke-hake.ts.net/balbosoft/anubis:{tag}", "drill"]) ?? 0; + int BuildImage(string tag) => Step("Costruzione dell'immagine Docker", "docker", ["build", "-f", "deploy/docker/Dockerfile", "-t", $"gitea.encke-hake.ts.net/balbosoft/anubis:{tag}", "."]) ?? 0;