diff --git a/src/Anubis/Backtest/Engine.cs b/src/Anubis/Backtest/Engine.cs index 31be079..4c24d5f 100644 --- a/src/Anubis/Backtest/Engine.cs +++ b/src/Anubis/Backtest/Engine.cs @@ -104,6 +104,9 @@ public static class BacktestReasons /// public static class BacktestEngine { + /// Changes when the engine computes differently: plans record it, so a report says which engine made it. + public const string Version = "1"; + public static BacktestRun Run(IStrategy strategy, BacktestDataset data, BacktestOptions? options = null, PortfolioState? start = null) { ArgumentNullException.ThrowIfNull(strategy); diff --git a/src/Anubis/Backtest/Laboratory.cs b/src/Anubis/Backtest/Laboratory.cs new file mode 100644 index 0000000..17e818e --- /dev/null +++ b/src/Anubis/Backtest/Laboratory.cs @@ -0,0 +1,323 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Threading.Channels; +using Anubis.Core; +using Anubis.Data; +using Anubis.Db; +using Anubis.Strategies; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Anubis.Backtest; + +/// What the user asks for in the new-trial form of the laboratory. +public sealed record TrialRequest( + string Strategy, + string Source, + string Venue, + IReadOnlyList Symbols, + DateOnly From, + DateOnly To, + decimal InitialCapital, + IReadOnlyDictionary>? Grid = null); + +/// A request the laboratory refuses, with the reason in Italian for the form. +public sealed class TrialRequestException(string message) : Exception(message); + +/// +/// The laboratory (SPEC 12.2, 13.4): it writes the plan of a trial before running it, runs one trial at a time on a +/// dedicated thread with low priority, so trading and the interface never wait for a backtest, and writes the report +/// back. Only the asynchronous side touches the database. After a restart, trials left halfway start again. +/// +public sealed partial class Laboratory : IDisposable +{ + public const int MaxSymbols = 20; + + public const int MaxVariants = 200; + + private readonly Database _db; + private readonly IClock _clock; + private readonly string _researchRoot; + private readonly ILogger _logger; + private readonly Channel _queue = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true }); + private readonly ConcurrentDictionary _progress = new(); + private readonly LowPriorityWorker _worker = new("anubis-laboratory"); + + public Laboratory(Database db, IClock clock, string researchRoot, ILogger? logger = null) + { + _db = db; + _clock = clock; + _researchRoot = researchRoot; + _logger = logger ?? NullLogger.Instance; + Trials = new TrialStore(db, clock); + } + + public TrialStore Trials { get; } + + /// What the running trial is doing, in words; null when it is not running here. + public string? Progress(Guid id) => _progress.GetValueOrDefault(id); + + /// Checks the request, writes the plan and queues the trial. The plan is final from here on. + public async Task PlanAsync(TrialRequest request, CancellationToken cancellationToken) + { + var (plan, variants) = BuildPlan(request, _clock.UtcNow); + var trial = await Trials.PlanAsync(plan, variants, cancellationToken).ConfigureAwait(false); + await _queue.Writer.WriteAsync(trial.Id, cancellationToken).ConfigureAwait(false); + return trial; + } + + /// The plan of a request: the family's own grid where the request gives none, the default costs, checks and thresholds. + public static (TrialPlan Plan, int Variants) BuildPlan(TrialRequest request, DateTimeOffset now) + { + ArgumentNullException.ThrowIfNull(request); + var family = StrategyCatalog.Find(request.Strategy) ?? throw new TrialRequestException("Strategia sconosciuta."); + if (request.Source is not (TrialSources.Database or TrialSources.Research)) + { + throw new TrialRequestException("Origine dei dati sconosciuta."); + } + + var symbols = request.Symbols.Select(s => s.Trim()).Where(s => s.Length > 0).Distinct(StringComparer.Ordinal).ToList(); + if (symbols.Count == 0) + { + throw new TrialRequestException("Indica almeno uno strumento."); + } + + if (symbols.Count > MaxSymbols) + { + throw new TrialRequestException($"Al massimo {MaxSymbols} strumenti per prova."); + } + + if (request.From >= request.To) + { + throw new TrialRequestException("La data di inizio deve venire prima di quella di fine."); + } + + if (request.InitialCapital is <= 0m or > 10_000_000m) + { + throw new TrialRequestException("Il capitale iniziale deve essere tra 1 e 10.000.000."); + } + + var grid = new SortedDictionary>(StringComparer.Ordinal); + foreach (var p in family.Parameters) + { + var values = request.Grid?.GetValueOrDefault(p.Name) is { Count: > 0 } given ? given : p.Grid; + if (values.Any(x => x < p.Min || x > p.Max)) + { + throw new TrialRequestException(string.Create(CultureInfo.GetCultureInfo("it-IT"), $"{p.Label}: i valori vanno da {p.Min} a {p.Max}.")); + } + + grid[p.Name] = [.. values.Distinct().Order()]; + } + + var plan = new TrialPlan( + family.Id, + family.Version, + grid, + new TrialData(request.Source, request.Venue, symbols, request.From, request.To), + request.InitialCapital, + [1m, 2m, 3m], + TrialValidation.Default, + TrialThresholds.Default, + BacktestEngine.Version, + now); + var variants = ParameterGrid.Of(family, plan.Grid).Count; + if (variants == 0) + { + throw new TrialRequestException("Nessuna combinazione dei parametri è valida."); + } + + if (variants > MaxVariants) + { + throw new TrialRequestException($"Troppe combinazioni ({variants}): al massimo {MaxVariants}."); + } + + return (plan, variants); + } + + /// Runs the queue until stopped: first the trials left planned or halfway by the last run, then the new ones. + public async Task RunAsync(CancellationToken cancellationToken) + { + foreach (var id in await Trials.ResumeAsync(cancellationToken).ConfigureAwait(false)) + { + await _queue.Writer.WriteAsync(id, cancellationToken).ConfigureAwait(false); + } + + await foreach (var id in _queue.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + await ProcessAsync(id, cancellationToken).ConfigureAwait(false); + } + } + + /// One trial: load its data, compute on the worker thread, write the report. A failure is recorded, not thrown. + public async Task ProcessAsync(Guid id, CancellationToken cancellationToken) + { + if (!await Trials.StartAsync(id, cancellationToken).ConfigureAwait(false)) + { + return; + } + + var trial = (await Trials.GetAsync(id, cancellationToken).ConfigureAwait(false))!; + _progress[id] = "lettura dei dati"; + try + { + var family = StrategyCatalog.Find(trial.Strategy) ?? throw new InvalidOperationException("Strategia non più disponibile: " + trial.Strategy); + if (family.Version != trial.StrategyVersion) + { + throw new InvalidOperationException($"La strategia è cambiata dalla versione {trial.StrategyVersion} alla {family.Version}: pianifica una prova nuova."); + } + + var data = await LoadAsync(_db, _clock, _researchRoot, trial.Plan.Data, cancellationToken).ConfigureAwait(false); + var prior = await Trials.PriorAsync(trial.Strategy, id, cancellationToken).ConfigureAwait(false); + var now = _clock.UtcNow; + var (report, summary) = await _worker.RunAsync( + () => TrialRunner.Run(id, trial.Plan, trial.PlanHash, family, data, prior, now, text => _progress[id] = text, cancellationToken)).ConfigureAwait(false); + await Trials.FinishAsync(id, report.DataFingerprint, summary, report.ToJson(), report.Passed, cancellationToken).ConfigureAwait(false); + LogFinished(_logger, id, trial.Strategy, report.Passed); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; // Left running: the next start plans it again. + } + catch (Exception ex) + { + LogFailed(_logger, id, ex); + await Trials.FailAsync(id, Redactor.Redact(ex.Message), CancellationToken.None).ConfigureAwait(false); + } + finally + { + _progress.TryRemove(id, out _); + } + } + + /// + /// The bars of a trial. From the database: daily bars and, for stocks, corporate actions. From the research history: + /// Binance daily archives, with estimated order rules (the archive does not carry them). + /// + public static async Task LoadAsync(Database db, IClock clock, string researchRoot, TrialData data, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(data); + var instruments = new List(); + if (data.Source == TrialSources.Database) + { + var catalog = new InstrumentCatalog(db, clock); + var bars = new BarStore(db, clock); + var actions = new CorporateActionStore(db, clock); + foreach (var symbol in data.Symbols) + { + var instrument = await catalog.FindAsync(data.Venue, symbol, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Strumento sconosciuto: {symbol} su {data.Venue}."); + var daily = await bars.DailyAsync(instrument.Id, data.From, data.To, cancellationToken).ConfigureAwait(false); + var events = instrument.IsCrypto ? [] : await actions.ForSymbolAsync(symbol, cancellationToken).ConfigureAwait(false); + instruments.Add(new BacktestInstrument(instrument.Spec, daily, [.. events.Where(a => a.ExDate > data.From && a.ExDate <= data.To)], "database")); + } + } + else + { + var store = new ResearchStore(researchRoot); + var from = new DateTimeOffset(data.From.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); + var to = new DateTimeOffset(data.To.AddDays(1).ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); + foreach (var symbol in data.Symbols) + { + var daily = new List(); + await foreach (var b in store.ReadAsync(data.Venue, symbol, Timeframes.Day, from, to, cancellationToken).ConfigureAwait(false)) + { + daily.Add(new DailyBar(DateOnly.FromDateTime(b.OpensAt.UtcDateTime), (double)b.Open, (double)b.High, (double)b.Low, (double)b.Close, (double)b.Volume)); + } + + instruments.Add(new BacktestInstrument(ResearchSpec(data.Venue, symbol), daily, null, "storico di ricerca " + data.Venue)); + } + } + + var empty = instruments.FirstOrDefault(i => i.Bars.Count == 0); + if (empty is not null) + { + throw new InvalidOperationException($"Nessuna barra giornaliera per {empty.Symbol} tra il {data.From:dd/MM/yyyy} e il {data.To:dd/MM/yyyy}."); + } + + var currencies = instruments.Select(i => i.Spec.QuoteCurrency).Distinct(StringComparer.Ordinal).ToList(); + if (currencies.Count != 1) + { + throw new InvalidOperationException("Gli strumenti di una prova devono essere quotati nella stessa valuta: " + string.Join(", ", currencies) + "."); + } + + return new BacktestDataset(instruments, currencies[0]); + } + + /// Order rules for a research series: the archive has none, so small steps that never bind are assumed. + public static InstrumentSpec ResearchSpec(string venue, string symbol) + { + ArgumentNullException.ThrowIfNull(symbol); + string[] quotes = ["USDT", "USDC", "EUR", "USD"]; + var quote = quotes.FirstOrDefault(q => symbol.EndsWith(q, StringComparison.Ordinal) && symbol.Length > q.Length) + ?? throw new InvalidOperationException("Valuta di quotazione non riconosciuta per " + symbol + "."); + return new InstrumentSpec(venue, symbol, symbol, AssetClasses.Crypto, quote) + { + BaseCurrency = symbol[..^quote.Length], + TickSize = 0.00000001m, + LotSize = 0.00000001m, + MinSize = 0.00000001m, + MinNotional = 5m, + Status = "live", + }; + } + + public void Dispose() + { + _queue.Writer.TryComplete(); + _worker.Dispose(); + } + + [LoggerMessage(Level = LogLevel.Information, Message = "Backtest trial {Id} of {Strategy} finished, passed: {Passed}")] + private static partial void LogFinished(ILogger logger, Guid id, string strategy, bool passed); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Backtest trial {Id} failed")] + private static partial void LogFailed(ILogger logger, Guid id, Exception exception); +} + +/// +/// One thread with low priority that runs heavy work in order. Its callers await the result on their own side: the +/// thread never touches the database. +/// +internal sealed class LowPriorityWorker : IDisposable +{ + private readonly BlockingCollection _work = []; + private readonly Thread _thread; + + public LowPriorityWorker(string name) + { + _thread = new Thread(Loop) { IsBackground = true, Name = name, Priority = ThreadPriority.BelowNormal }; + _thread.Start(); + } + + public Task RunAsync(Func work) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _work.Add(() => + { + try + { + done.SetResult(work()); + } + catch (Exception ex) + { + done.SetException(ex); + } + }); + return done.Task; + } + + private void Loop() + { + foreach (var item in _work.GetConsumingEnumerable()) + { + item(); + } + } + + public void Dispose() + { + _work.CompleteAdding(); + _thread.Join(TimeSpan.FromSeconds(10)); + _work.Dispose(); + } +} diff --git a/src/Anubis/Backtest/Report.cs b/src/Anubis/Backtest/Report.cs new file mode 100644 index 0000000..a89a15c --- /dev/null +++ b/src/Anubis/Backtest/Report.cs @@ -0,0 +1,265 @@ +using System.Diagnostics; +using System.Text.Json; +using Anubis.Data; +using Anubis.Strategies; + +namespace Anubis.Backtest; + +/// An instrument of the report: where its bars come from and what period they cover. +public sealed record ReportInstrument(string Symbol, string Venue, DateOnly? First, DateOnly? Last, int Bars, int Actions, string Source); + +/// A point of the equity curve of the chosen variant and of buy and hold, on the same day. +public sealed record ReportPoint(DateOnly Day, decimal Equity, decimal Benchmark); + +/// A variant of the grid with its full-period figures. +public sealed record ReportVariant(string Parameters, double Sharpe, double PlateauScore, decimal NetResult, double MaxDrawdown, int Trades, bool Chosen); + +/// The chosen variant with costs multiplied (SPEC 12.2 point 3). +public sealed record ReportCosts(decimal Multiplier, decimal NetResult, double TotalReturn, double Sharpe, double MaxDrawdown, decimal Fees); + +public sealed record ReportWindow(int Window, DateOnly TrainFrom, DateOnly TrainTo, DateOnly TestFrom, DateOnly TestTo, string Chosen, double Return, double Sharpe); + +public sealed record ReportWalkForward(IReadOnlyList Windows, int Purge, double Return, double Sharpe); + +public sealed record ReportCombinatorial(int Groups, int TestGroups, int Splits, int Purge, int Embargo, IReadOnlyList PathSharpes, double MeanSharpe, double WorstSharpe, double PositiveShare); + +public sealed record ReportOverfitting(int Groups, int Combinations, int Variants, double Probability); + +public sealed record ReportLookAhead(IReadOnlyList Cutoffs, int Decisions, int Changed, IReadOnlyList Examples) +{ + public bool Passed => Changed == 0; +} + +/// +/// A threshold of SPEC 12.2 checked on the result. is null when the check does not apply (PBO with +/// a single variant). +/// +public sealed record ReportCheck(string Key, string Name, double? Value, string Comparison, double Threshold, string Unit, bool? Passed); + +public static class ReportCheckKeys +{ + public const string DoubleCosts = "double_costs"; + public const string DeflatedSharpe = "deflated_sharpe"; + public const string Overfitting = "overfitting"; + public const string Drawdown = "drawdown"; + public const string LookAhead = "look_ahead"; +} + +/// +/// The final report of a trial (SPEC 12.2 point 6): figures of the chosen variant, buy and hold, cost sensitivity, the +/// grid, the validations, the checks and what the backtest does not model. +/// +public sealed record BacktestReport( + Guid TrialId, + string Strategy, + string StrategyName, + string StrategyVersion, + string PlanHash, + string DataFingerprint, + string Currency, + int PeriodsPerYear, + int Sessions, + IReadOnlyList Instruments, + string Chosen, + BacktestMetrics Metrics, + BacktestMetrics Benchmark, + IReadOnlyList Costs, + IReadOnlyList Variants, + ReportWalkForward? WalkForward, + ReportCombinatorial? Combinatorial, + ReportOverfitting? Overfitting, + DeflatedSharpe DeflatedSharpe, + int TrialsCounted, + ReportLookAhead LookAhead, + IReadOnlyList Checks, + bool Passed, + IReadOnlyList Limitations, + IReadOnlyList Curve, + IReadOnlyList Trades, + IReadOnlyDictionary Notes, + DateTimeOffset ComputedAt, + double Seconds) +{ + public string ToJson() => JsonSerializer.Serialize(this, TrialPlan.Json); + + public static BacktestReport FromJson(string text) => JsonSerializer.Deserialize(text, TrialPlan.Json) ?? throw new JsonException("Empty report"); +} + +/// +/// Runs a trial from its plan, on data already loaded (pure computation, no database: the laboratory runs it on its own +/// low-priority thread). Every variant of the grid runs once on the whole period; the validations work on their daily +/// returns; the chosen variant (plateau rule on the whole period) is then run again with doubled and tripled costs. +/// +public static class TrialRunner +{ + public const int MaxCurvePoints = 500; + + public const int MaxTrades = 300; + + public static (BacktestReport Report, TrialSummary Summary) Run( + Guid trialId, TrialPlan plan, string planHash, IStrategyFamily family, BacktestDataset data, PriorTrials prior, DateTimeOffset now, + Action? progress = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(plan); + ArgumentNullException.ThrowIfNull(family); + ArgumentNullException.ThrowIfNull(data); + ArgumentNullException.ThrowIfNull(prior); + var watch = Stopwatch.StartNew(); + if (data.Sessions.Count < 30) + { + throw new InvalidOperationException($"Dati insufficienti: {data.Sessions.Count} sedute, ne servono almeno 30."); + } + + var variants = ParameterGrid.Of(family, plan.Grid); + if (variants.Count == 0) + { + throw new InvalidOperationException("Nessuna combinazione valida nella griglia dei parametri."); + } + + var options = new BacktestOptions { InitialCapital = plan.InitialCapital }; + var runs = new BacktestRun[variants.Count]; + for (var i = 0; i < variants.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + progress?.Invoke($"variante {i + 1} di {variants.Count}"); + runs[i] = BacktestEngine.Run(family.Create(variants[i].Parameters), data, options); + } + + var returns = runs.Select(r => r.DailyReturns()).ToArray(); + var sharpes = returns.Select(r => Moments.Of(r).Sharpe).ToArray(); + var scores = Validation.PlateauScores(variants, sharpes); + var chosenIndex = Validation.ChoosePlateau(variants, sharpes); + var chosen = variants[chosenIndex]; + var run = runs[chosenIndex]; + var metrics = BacktestMetrics.Of(run, data.PeriodsPerYear); + + progress?.Invoke("costi doppi e tripli, comprare e tenere"); + var costs = new List(); + foreach (var multiplier in plan.CostMultipliers.Distinct().Order()) + { + var costed = multiplier == 1m ? run : BacktestEngine.Run(family.Create(chosen.Parameters), data, options with { Costs = new CostModel(multiplier) }); + var m = multiplier == 1m ? metrics : BacktestMetrics.Of(costed, data.PeriodsPerYear); + costs.Add(new ReportCosts(multiplier, costed.NetResult, m.TotalReturn, m.Sharpe, m.MaxDrawdown, costed.Fees)); + } + + var benchmarkRun = BacktestEngine.Run(new BuyAndHold(), data, options); + var benchmark = BacktestMetrics.Of(benchmarkRun, data.PeriodsPerYear); + + cancellationToken.ThrowIfCancellationRequested(); + progress?.Invoke("convalide"); + var days = run.Equity.Select(p => p.Day).ToArray(); + DateOnly DayOf(int returnIndex) => days[Math.Clamp(returnIndex + 1, 0, days.Length - 1)]; + var v = plan.Validation; + var wf = Validation.WalkForward(variants, returns, v.WalkForwardWindows); + var walkForward = wf is null ? null : new ReportWalkForward( + [.. wf.Windows.Select(w => new ReportWindow(w.Window, DayOf(w.TrainFrom), DayOf(w.TrainTo - 1), DayOf(w.TestFrom), DayOf(w.TestTo - 1), + variants[w.Chosen].ToString(), w.TestReturn, w.TestSharpe))], + wf.Purge, wf.Return, wf.Sharpe); + var cv = Validation.Combinatorial(variants, returns, v.CombinatorialGroups, v.CombinatorialTestGroups); + var combinatorial = cv is null ? null : new ReportCombinatorial(cv.Groups, cv.TestGroups, cv.Splits, cv.Purge, cv.Embargo, cv.PathSharpes, + cv.MeanSharpe, cv.WorstSharpe, cv.PositiveShare); + var pbo = Validation.Overfitting(returns, v.OverfittingGroups); + var overfitting = pbo is null ? null : new ReportOverfitting(pbo.Groups, pbo.Combinations, pbo.Variants, pbo.Probability); + + // Every variant of every finished trial of this strategy counts as a try (SPEC 12.2 point 1). + var trials = prior.Variants + variants.Count; + var spread = Moments.Of([.. prior.Sharpes, .. sharpes]).StandardDeviation; + var dsr = SharpeTests.Deflated(returns[chosenIndex], trials, spread * spread); + + progress?.Invoke("test anti sguardo in avanti"); + var look = LookAheadCheck.Run(_ => family.Create(chosen.Parameters), data, options, v.LookAheadCuts); + var lookAhead = new ReportLookAhead(look.Cutoffs, look.Decisions, look.Mismatches.Count, + [.. look.Mismatches.Take(5).Select(m => $"{m.Day:yyyy-MM-dd}: {m.Full} / {m.Cut}")]); + + var t = plan.Thresholds; + var doubled = costs.FirstOrDefault(c => c.Multiplier == 2m); + List checks = + [ + new(ReportCheckKeys.DoubleCosts, "Risultato netto con costi doppi", doubled is null ? null : (double)doubled.NetResult, ">", (double)t.MinNetAtDoubleCosts, "money", + doubled is null ? false : doubled.NetResult > t.MinNetAtDoubleCosts), + new(ReportCheckKeys.DeflatedSharpe, "Deflated Sharpe Ratio", dsr.Value, "≥", t.MinDeflatedSharpe, "ratio", dsr.Value >= t.MinDeflatedSharpe), + new(ReportCheckKeys.Overfitting, "Probabilità di sovra-adattamento (PBO)", pbo?.Probability, "≤", t.MaxOverfitting, "ratio", + pbo is null ? null : pbo.Probability <= t.MaxOverfitting), + new(ReportCheckKeys.Drawdown, "Calo massimo", metrics.MaxDrawdown, "≤", t.MaxDrawdown, "percent", metrics.MaxDrawdown <= t.MaxDrawdown), + new(ReportCheckKeys.LookAhead, "Decisioni cambiate aggiungendo dati successivi", look.Mismatches.Count, "=", 0, "count", look.Passed), + ]; + var passed = checks.All(c => c.Passed != false); + + var benchmarkByDay = benchmarkRun.Equity.ToDictionary(p => p.Day, p => p.Equity); + var step = Math.Max(1, (int)Math.Ceiling(run.Equity.Count / (double)MaxCurvePoints)); + var curve = run.Equity + .Where((_, i) => i % step == 0 || i == run.Equity.Count - 1) + .Select(p => new ReportPoint(p.Day, p.Equity, benchmarkByDay.GetValueOrDefault(p.Day, plan.InitialCapital))) + .ToList(); + + var report = new BacktestReport( + trialId, + family.Id, + family.Name, + family.Version, + planHash, + data.Fingerprint(), + data.Currency, + data.PeriodsPerYear, + data.Sessions.Count, + [.. data.Instruments.Select(i => new ReportInstrument(i.Symbol, i.Spec.Venue, i.Bars.Count > 0 ? i.Bars[0].Day : null, + i.Bars.Count > 0 ? i.Bars[^1].Day : null, i.Bars.Count, i.Actions.Count, i.Source))], + chosen.ToString(), + metrics, + benchmark, + costs, + [.. variants.Select((x, i) => new ReportVariant(x.ToString(), sharpes[i], scores[i], runs[i].NetResult, + Metrics(runs[i], data).MaxDrawdown, runs[i].Trades.Count(tr => tr.IsClosed), i == chosenIndex))], + walkForward, + combinatorial, + overfitting, + dsr, + trials, + lookAhead, + checks, + passed, + Limitations(plan, family, data), + curve, + [.. run.Trades.TakeLast(MaxTrades)], + run.Notes.GroupBy(n => n.Reason).ToDictionary(g => g.Key, g => g.Count(), StringComparer.Ordinal), + now, + watch.Elapsed.TotalSeconds); + + var summary = new TrialSummary(data.Currency, chosen.ToString(), run.NetResult, metrics.TotalReturn, metrics.Sharpe, metrics.MaxDrawdown, metrics.Trades, + dsr.Value, pbo?.Probability, sharpes); + return (report, summary); + } + + private static BacktestMetrics Metrics(BacktestRun run, BacktestDataset data) => BacktestMetrics.Of(run, data.PeriodsPerYear); + + /// What this backtest does not model, in plain Italian (SPEC 12.1: the report declares it). + public static IReadOnlyList Limitations(TrialPlan plan, IStrategyFamily family, BacktestDataset data) + { + ArgumentNullException.ThrowIfNull(plan); + ArgumentNullException.ThrowIfNull(family); + ArgumentNullException.ThrowIfNull(data); + var result = new List + { + "I correttivi senza una storia ricostruibile (notizie, calendario economico, trimestrali, modello) non ricevono dati e non intervengono.", + "Il regime di mercato e i limiti di rischio arrivano con la Fase 5: in questo backtest ogni intenzione della strategia passa così com'è.", + "Le decisioni si prendono sulla chiusura e si eseguono all'apertura della seduta dopo, con lo scostamento stimato: nessun inseguimento con ordini limite.", + "Sharpe e volatilità sono calcolati senza tasso privo di rischio.", + }; + if (plan.Data.Source == TrialSources.Research) + { + result.Add("Prezzi dello storico Binance in USDT: passo di prezzo e quantità minima sono stime, i costi sono quelli di OKX."); + } + + if (data.Instruments.Any(i => i.Spec.AssetClass == AssetClasses.Equity && i.Actions.Count == 0)) + { + result.Add("Per alcuni titoli non risultano frazionamenti né dividendi nel periodo: se ce ne sono stati, mancano dal database."); + } + + if (family.Id == "sma_cross") + { + result.Add("L'incrocio di medie è una strategia d'esempio per provare il laboratorio, non una delle strategie di base."); + } + + return result; + } +} diff --git a/src/Anubis/Backtest/Trials.cs b/src/Anubis/Backtest/Trials.cs new file mode 100644 index 0000000..49b9e6c --- /dev/null +++ b/src/Anubis/Backtest/Trials.cs @@ -0,0 +1,215 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Backtest; + +/// Where the bars of a trial come from. +public static class TrialSources +{ + /// Daily bars and corporate actions collected in the database (SPEC 6.2). + public const string Database = "database"; + + /// The research history on disk (SPEC 6.6): Binance daily archives from 2017, in USDT. + public const string Research = "research"; +} + +/// The data a trial reads: one venue, a few instruments, a period (both days included). +public sealed record TrialData(string Source, string Venue, IReadOnlyList Symbols, DateOnly From, DateOnly To); + +/// +/// The thresholds of SPEC 12.2, fixed in the plan before the trial runs: changing them after seeing the results needs +/// an ADR. +/// +public sealed record TrialThresholds(decimal MinNetAtDoubleCosts, double MinDeflatedSharpe, double MaxOverfitting, double MaxDrawdown) +{ + public static TrialThresholds Default { get; } = new(0m, 0.90, 0.40, 0.25); +} + +/// How the trial validates: walk-forward windows, purged cross-validation, PBO blocks, look-ahead cuts. +public sealed record TrialValidation(int WalkForwardWindows, int CombinatorialGroups, int CombinatorialTestGroups, int OverfittingGroups, int LookAheadCuts) +{ + public static TrialValidation Default { get; } = new(4, 6, 2, 16, 6); +} + +/// +/// The plan of a trial (SPEC 12.2 point 1): rules, grid, data, costs, validations and thresholds, written before any +/// result and never changed. Its hash goes with the report. +/// +public sealed record TrialPlan( + string Strategy, + string StrategyVersion, + IReadOnlyDictionary> Grid, + TrialData Data, + decimal InitialCapital, + IReadOnlyList CostMultipliers, + TrialValidation Validation, + TrialThresholds Thresholds, + string Engine, + DateTimeOffset WrittenAt) +{ + public static readonly JsonSerializerOptions Json = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals, + WriteIndented = false, + }; + + public string ToJson() => JsonSerializer.Serialize(this, Json); + + public static TrialPlan FromJson(string text) => JsonSerializer.Deserialize(text, Json) ?? throw new JsonException("Empty plan"); + + /// SHA-256 of the plan's text as stored. + public static string Hash(string json) => Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(json))); +} + +public static class TrialStatuses +{ + public const string Planned = "planned"; + public const string Running = "running"; + public const string Done = "done"; + public const string Failed = "failed"; +} + +public static class TrialVerdicts +{ + public const string Passed = "passed"; + public const string Rejected = "rejected"; +} + +/// The few numbers of a finished trial the list shows, and the Sharpe ratios the next trials' DSR counts. +public sealed record TrialSummary( + string Currency, + string Chosen, + decimal NetResult, + double TotalReturn, + double Sharpe, + double MaxDrawdown, + int Trades, + double? DeflatedSharpe, + double? Overfitting, + IReadOnlyList VariantSharpes); + +/// A row of the trial register. +public sealed record Trial( + Guid Id, + string Strategy, + string StrategyVersion, + TrialPlan Plan, + string PlanHash, + int Variants, + string Status, + string? Verdict, + string? DataFingerprint, + TrialSummary? Summary, + string? Error, + DateTimeOffset CreatedAt, + DateTimeOffset? StartedAt, + DateTimeOffset? FinishedAt) +{ + public bool IsFinished => Status is TrialStatuses.Done or TrialStatuses.Failed; +} + +/// The trials before this one of the same strategy: how many variants they tried and their Sharpe ratios. +public sealed record PriorTrials(int Trials, int Variants, IReadOnlyList Sharpes) +{ + public static PriorTrials None { get; } = new(0, 0, []); +} + +/// +/// The trial register (SPEC 12.2 point 1). A plan is inserted once and never updated: the store has no method that +/// changes it, and every later write touches only status, results and times. +/// +public sealed class TrialStore(Database db, IClock clock) +{ + private const string Columns = "id, strategy, strategy_version, plan, plan_hash, variants, status, verdict, data_fingerprint, summary, error, created_at, started_at, finished_at"; + + public async Task PlanAsync(TrialPlan plan, int variants, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(plan); + var id = Ids.NewId(clock); + var json = plan.ToJson(); + var now = clock.UtcNow; + await db.WriteAsync(s => s.ExecuteAsync( + "INSERT INTO backtest_trials (id, strategy, strategy_version, plan, plan_hash, variants, status, created_at) " + + "VALUES (@id, @s, @v, @p, @h, @n, @st, @now)", + cancellationToken, ("id", id), ("s", plan.Strategy), ("v", plan.StrategyVersion), ("p", new DbJson(json)), ("h", TrialPlan.Hash(json)), + ("n", variants), ("st", TrialStatuses.Planned), ("now", now)), cancellationToken).ConfigureAwait(false); + return (await GetAsync(id, cancellationToken).ConfigureAwait(false))!; + } + + public Task GetAsync(Guid id, CancellationToken cancellationToken) => + db.ReadAsync(s => s.FirstOrDefaultAsync($"SELECT {Columns} FROM backtest_trials WHERE id = @id", Read, ("id", id)), cancellationToken); + + public async Task> ListAsync(int limit, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync($"SELECT {Columns} FROM backtest_trials ORDER BY created_at DESC, id DESC LIMIT @n", Read, ("n", limit)), + cancellationToken).ConfigureAwait(false); + + public Task ReportJsonAsync(Guid id, CancellationToken cancellationToken) => + db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT report FROM backtest_trials WHERE id = @id AND report IS NOT NULL", r => r.Json("report").Text, ("id", id)), + cancellationToken); + + /// Marks a planned trial as running. False when it was not planned (another worker took it, or it is finished). + public async Task StartAsync(Guid id, CancellationToken cancellationToken) => + await db.WriteAsync(s => s.ExecuteAsync("UPDATE backtest_trials SET status = @r, started_at = @now WHERE id = @id AND status = @p", + cancellationToken, ("r", TrialStatuses.Running), ("now", clock.UtcNow), ("id", id), ("p", TrialStatuses.Planned)), cancellationToken).ConfigureAwait(false) == 1; + + public Task FinishAsync(Guid id, string fingerprint, TrialSummary summary, string reportJson, bool passed, CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync( + "UPDATE backtest_trials SET status = @d, verdict = @v, data_fingerprint = @f, summary = @s, report = @rep, finished_at = @now WHERE id = @id AND status = @r", + cancellationToken, ("d", TrialStatuses.Done), ("v", passed ? TrialVerdicts.Passed : TrialVerdicts.Rejected), ("f", fingerprint), + ("s", new DbJson(JsonSerializer.Serialize(summary, TrialPlan.Json))), ("rep", new DbJson(reportJson)), ("now", clock.UtcNow), ("id", id), + ("r", TrialStatuses.Running)), cancellationToken); + + public Task FailAsync(Guid id, string error, CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync("UPDATE backtest_trials SET status = @f, error = @e, finished_at = @now WHERE id = @id AND status IN (@p, @r)", + cancellationToken, ("f", TrialStatuses.Failed), ("e", error), ("now", clock.UtcNow), ("id", id), ("p", TrialStatuses.Planned), + ("r", TrialStatuses.Running)), cancellationToken); + + /// + /// After a restart: trials left running go back to planned (their work was lost with the process), and every planned + /// trial is returned oldest first, to run again from the start with the same plan. + /// + public async Task> ResumeAsync(CancellationToken cancellationToken) => + await db.WriteAsync(async s => + { + await s.ExecuteAsync("UPDATE backtest_trials SET status = @p, started_at = NULL WHERE status = @r", cancellationToken, + ("p", TrialStatuses.Planned), ("r", TrialStatuses.Running)).ConfigureAwait(false); + return await s.QueryAsync("SELECT id FROM backtest_trials WHERE status = @p ORDER BY created_at, id", r => r.Id("id"), ("p", TrialStatuses.Planned)) + .ConfigureAwait(false); + }, cancellationToken).ConfigureAwait(false); + + /// The finished trials of other than : what the DSR deflates for. + public async Task PriorAsync(string strategy, Guid except, CancellationToken cancellationToken) + { + var rows = await db.ReadAsync(s => s.QueryAsync( + "SELECT variants, summary FROM backtest_trials WHERE strategy = @s AND id <> @id AND status = @d", + r => (Variants: (int)r.Number("variants"), Summary: r.IsNull("summary") ? null : JsonSerializer.Deserialize(r.Json("summary").Text, TrialPlan.Json)), + ("s", strategy), ("id", except), ("d", TrialStatuses.Done)), cancellationToken).ConfigureAwait(false); + return new PriorTrials(rows.Count, rows.Sum(r => r.Variants), [.. rows.SelectMany(r => r.Summary?.VariantSharpes ?? [])]); + } + + private static Trial Read(DbRow r) + { + var plan = r.Json("plan").Text; + return new Trial( + r.Id("id"), + r.Text("strategy"), + r.Text("strategy_version"), + TrialPlan.FromJson(plan), + r.Text("plan_hash"), + (int)r.Number("variants"), + r.Text("status"), + r.TextOrNull("verdict"), + r.TextOrNull("data_fingerprint"), + r.IsNull("summary") ? null : JsonSerializer.Deserialize(r.Json("summary").Text, TrialPlan.Json), + r.TextOrNull("error"), + r.Moment("created_at"), + r.MomentOrNull("started_at"), + r.MomentOrNull("finished_at")); + } +} diff --git a/src/Anubis/Db/Migrations/0004_lab.sql b/src/Anubis/Db/Migrations/0004_lab.sql new file mode 100644 index 0000000..95b3780 --- /dev/null +++ b/src/Anubis/Db/Migrations/0004_lab.sql @@ -0,0 +1,25 @@ +-- Phase 4 schema (SPEC 12.2, ADR 0010): the register of backtest trials of the laboratory. + +-- One row per trial. The plan (strategy, grid, data, costs, validations, thresholds) is written before any result +-- and never changed afterwards: plan_hash is the SHA-256 of its text, so a report always shows the plan it answers. +-- Every trial stays, failed and passed alike: the Deflated Sharpe Ratio counts them all (SPEC 12.2 point 1). +CREATE TABLE backtest_trials ( + id {uuid} PRIMARY KEY, + strategy TEXT NOT NULL, + strategy_version TEXT NOT NULL, + plan {json} NOT NULL, + plan_hash TEXT NOT NULL, + variants INTEGER NOT NULL, + status TEXT NOT NULL, + verdict TEXT, + data_fingerprint TEXT, + summary {json}, + report {json}, + error TEXT, + created_at {timestamp} NOT NULL, + started_at {timestamp}, + finished_at {timestamp} +){strict}; + +CREATE INDEX backtest_trials_strategy ON backtest_trials (strategy, created_at); +CREATE INDEX backtest_trials_status ON backtest_trials (status); diff --git a/tests/Anubis.Tests/LaboratoryTests.cs b/tests/Anubis.Tests/LaboratoryTests.cs new file mode 100644 index 0000000..4fc6a85 --- /dev/null +++ b/tests/Anubis.Tests/LaboratoryTests.cs @@ -0,0 +1,175 @@ +using Anubis.Backtest; +using Anubis.Core; +using Anubis.Data; + +namespace Anubis.Tests; + +/// The trial register and the laboratory (issue #35): plan first, one trial at a time, report back. +[TestClass] +public sealed class LaboratoryTests +{ + private static readonly DateOnly From = new(2024, 1, 1); + + private static readonly Dictionary> SmallGrid = new() + { + ["fast"] = [5m, 10m], + ["slow"] = [20m, 40m], + }; + + internal static async Task SeedBitcoinAsync(TestDb test, IClock clock, int days = 400) + { + var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false); + await new BarStore(test.Db, clock).UpsertDailyAsync(btc.Id, BacktestData.Walk(days, 3, start: 40_000, drift: 0.001, volatility: 0.03, from: From), "test", + CancellationToken.None).ConfigureAwait(false); + return btc; + } + + internal static TrialRequest Request(params string[] symbols) => + new("sma_cross", TrialSources.Database, Venues.Okx, symbols.Length > 0 ? symbols : ["BTC-EUR"], From, From.AddDays(399), 10_000m, SmallGrid); + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task APlanIsWrittenBeforeTheResultsAndNeverChanges() + { + var clock = new ManualClock(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero)); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + await SeedBitcoinAsync(test, clock).ConfigureAwait(false); + using var lab = new Laboratory(test.Db, clock, Path.GetTempPath()); + + var trial = await lab.PlanAsync(Request(), CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(TrialStatuses.Planned, trial.Status); + Assert.AreEqual(4, trial.Variants); + var before = await RawPlanAsync(test, trial.Id).ConfigureAwait(false); + Assert.AreEqual(TrialPlan.Hash(before.Plan), before.Hash); + Assert.AreEqual(TrialThresholds.Default, trial.Plan.Thresholds, "Thresholds are fixed in the plan before the run."); + + clock.Advance(TimeSpan.FromMinutes(1)); + await lab.ProcessAsync(trial.Id, CancellationToken.None).ConfigureAwait(false); + + var done = (await lab.Trials.GetAsync(trial.Id, CancellationToken.None).ConfigureAwait(false))!; + Assert.AreEqual(TrialStatuses.Done, done.Status, done.Error); + Assert.IsNotNull(done.Verdict); + Assert.IsNotNull(done.Summary); + Assert.HasCount(4, done.Summary.VariantSharpes); + Assert.AreEqual(before, await RawPlanAsync(test, trial.Id).ConfigureAwait(false), "The plan text and its hash are never updated."); + + var report = BacktestReport.FromJson((await lab.Trials.ReportJsonAsync(trial.Id, CancellationToken.None).ConfigureAwait(false))!); + Assert.AreEqual(before.Hash, report.PlanHash); + Assert.AreEqual(done.DataFingerprint, report.DataFingerprint); + Assert.HasCount(4, report.Variants); + Assert.AreEqual(1, report.Variants.Count(v => v.Chosen)); + CollectionAssert.AreEqual(new[] { 1m, 2m, 3m }, report.Costs.Select(c => c.Multiplier).ToArray()); + Assert.IsTrue(report.Costs[0].Fees < report.Costs[1].Fees && report.Costs[1].Fees < report.Costs[2].Fees, "Costs grow with the multiplier."); + Assert.IsTrue(report.LookAhead.Passed); + Assert.HasCount(5, report.Checks); + Assert.AreEqual(report.Passed, done.Verdict == TrialVerdicts.Passed); + Assert.AreEqual(4, report.TrialsCounted); + Assert.IsNotNull(report.WalkForward); + Assert.IsNotNull(report.Combinatorial); + Assert.IsNotNull(report.Overfitting); + Assert.IsTrue(report.Curve.Count is > 100 and <= TrialRunner.MaxCurvePoints + 1); + Assert.IsTrue(report.Limitations.Any(l => l.Contains("Fase 5", StringComparison.Ordinal))); + + // A second trial of the same strategy deflates for the variants of both. + var second = await lab.PlanAsync(Request() with { Grid = new Dictionary> { ["fast"] = [10m], ["slow"] = [30m] } }, + CancellationToken.None).ConfigureAwait(false); + await lab.ProcessAsync(second.Id, CancellationToken.None).ConfigureAwait(false); + var again = BacktestReport.FromJson((await lab.Trials.ReportJsonAsync(second.Id, CancellationToken.None).ConfigureAwait(false))!); + Assert.AreEqual(5, again.TrialsCounted); + Assert.IsNull(again.Overfitting, "One variant: no PBO."); + Assert.IsNull(again.Checks.Single(c => c.Key == ReportCheckKeys.Overfitting).Passed); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task TrialsLeftHalfwayStartAgainAndFailuresKeepTheirReason() + { + var clock = new ManualClock(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero)); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + await SeedBitcoinAsync(test, clock, 60).ConfigureAwait(false); + using var lab = new Laboratory(test.Db, clock, Path.GetTempPath()); + + var halfway = await lab.PlanAsync(Request(), CancellationToken.None).ConfigureAwait(false); + Assert.IsTrue(await lab.Trials.StartAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false)); + Assert.IsFalse(await lab.Trials.StartAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false), "A running trial is not taken twice."); + var unknown = await lab.PlanAsync(Request("XYZ-EUR"), CancellationToken.None).ConfigureAwait(false); + + // The process stopped: on the next start both are planned again, oldest first. + var resumed = await lab.Trials.ResumeAsync(CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEqual(new[] { halfway.Id, unknown.Id }, resumed.ToArray()); + Assert.AreEqual(TrialStatuses.Planned, (await lab.Trials.GetAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false))!.Status); + + await lab.ProcessAsync(unknown.Id, CancellationToken.None).ConfigureAwait(false); + var failed = (await lab.Trials.GetAsync(unknown.Id, CancellationToken.None).ConfigureAwait(false))!; + Assert.AreEqual(TrialStatuses.Failed, failed.Status); + StringAssert.Contains(failed.Error, "Strumento sconosciuto"); + Assert.IsNull(await lab.Trials.ReportJsonAsync(unknown.Id, CancellationToken.None).ConfigureAwait(false)); + + // The queue runs in the background and drains the resumed trial too. + using var stop = new CancellationTokenSource(); + var running = lab.RunAsync(stop.Token); + for (var i = 0; i < 600 && !(await lab.Trials.GetAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false))!.IsFinished; i++) + { + await Task.Delay(50).ConfigureAwait(false); + } + + await stop.CancelAsync().ConfigureAwait(false); + await Assert.ThrowsAsync(() => running).ConfigureAwait(false); + Assert.AreEqual(TrialStatuses.Done, (await lab.Trials.GetAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false))!.Status); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void RequestsOutsideTheRulesAreRefusedBeforeAnythingIsWritten() + { + var now = DateTimeOffset.UnixEpoch; + Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { Strategy = "nope" }, now)); + Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { To = From }, now)); + Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { Symbols = [" "] }, now)); + Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { InitialCapital = 0m }, now)); + var outOfRange = Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with + { + Grid = new Dictionary> { ["fast"] = [1m] }, + }, now)); + StringAssert.Contains(outOfRange.Message, "vanno da 2 a 200"); + var none = Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with + { + Grid = new Dictionary> { ["fast"] = [60m], ["slow"] = [50m] }, + }, now)); + StringAssert.Contains(none.Message, "Nessuna combinazione"); + + // Without a grid, the family's own; the plan reads back exactly as written. + var (plan, variants) = Laboratory.BuildPlan(Request() with { Grid = null }, now); + Assert.AreEqual(9, variants); + var json = plan.ToJson(); + Assert.AreEqual(json, TrialPlan.FromJson(json).ToJson()); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task TheResearchHistoryLoadsWithEstimatedOrderRules() + { + var root = Directory.CreateTempSubdirectory("anubis-lab-").FullName; + try + { + var store = new ResearchStore(root); + var bars = BacktestData.Walk(50, 9, from: From).Select(b => new ResearchBar(new DateTimeOffset(b.Day.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero), + (decimal)b.Open, (decimal)b.High, (decimal)b.Low, (decimal)b.Close, (decimal)b.Volume)); + await store.WriteAsync(Venues.Binance, "ETHUSDT", Timeframes.Day, bars, CancellationToken.None).ConfigureAwait(false); + + var data = await Laboratory.LoadAsync(null!, SystemClock.Instance, root, + new TrialData(TrialSources.Research, Venues.Binance, ["ETHUSDT"], From.AddDays(10), From.AddDays(29)), CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual("USDT", data.Currency); + Assert.HasCount(20, data.Sessions); + Assert.AreEqual("ETH", data["ETHUSDT"]!.Spec.BaseCurrency); + Assert.IsTrue(data.IsCrypto); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + private static Task<(string Plan, string Hash)> RawPlanAsync(TestDb test, Guid id) => + test.Db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT plan, plan_hash FROM backtest_trials WHERE id = @id", r => (r.Json("plan").Text, r.Text("plan_hash")), ("id", id))); +}