359 lines
10 KiB
C#
359 lines
10 KiB
C#
using System.Globalization;
|
|
using System.Text;
|
|
using Encelado.Bot.Configuration;
|
|
using Encelado.Bot.Logging;
|
|
using Encelado.Core.Market;
|
|
using Encelado.Core.Portfolio;
|
|
using Encelado.Core.Strategies;
|
|
|
|
namespace Encelado.Bot.Diagnostics;
|
|
|
|
/// <summary>
|
|
/// Structured, machine-readable record of everything the engine decided and why.
|
|
/// <para>
|
|
/// Two CSV files, joined on <c>decisionId</c>:
|
|
/// </para>
|
|
/// <list type="bullet">
|
|
/// <item><b>decisions</b> — one row per evaluated bar per symbol: the bar itself
|
|
/// including the aggressor breakdown, every indicator the strategy publishes, the
|
|
/// position at the time, and the signal that came out. This is the dataset to load
|
|
/// into pandas when asking "why did it do that" or "would a different threshold have
|
|
/// helped".</item>
|
|
/// <item><b>executions</b> — one row per signal that reached the order path: the risk
|
|
/// verdict, the size that survived it, and the broker's answer.</item>
|
|
/// </list>
|
|
/// <para>
|
|
/// CSV rather than JSON on purpose: it opens in Excel, loads in one line of pandas, and
|
|
/// stays readable when a run produces tens of thousands of rows. Writes are buffered and
|
|
/// flushed on a timer, so the decision path never waits on the disk.
|
|
/// </para>
|
|
/// </summary>
|
|
public sealed class AnalyticsLog : IDisposable
|
|
{
|
|
private readonly StreamWriter? _decisions;
|
|
private readonly StreamWriter? _executions;
|
|
private readonly Lock _gate = new();
|
|
private readonly StringBuilder _row = new(512);
|
|
|
|
private string[] _metricNames = [];
|
|
private bool _decisionHeaderWritten;
|
|
private long _nextId;
|
|
private DateTime _lastFlush = DateTime.UtcNow;
|
|
|
|
public AnalyticsLog(LoggingOptions options)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(options);
|
|
|
|
_decisions = Open(options.ResolvePath(options.DecisionLog));
|
|
_executions = Open(options.ResolvePath(options.ExecutionLog));
|
|
|
|
if (_executions is not null && _executions.BaseStream.Length == 0)
|
|
{
|
|
_executions.WriteLine(
|
|
"timestampUtc,decisionId,symbol,side,phase,approved,riskReason,riskDetail," +
|
|
"quantity,referencePrice,stopPrice,targetPrice,notional,equity,buyingPower," +
|
|
"grossExposure,openPositions,orderId,error,latencyMs");
|
|
}
|
|
}
|
|
|
|
public bool IsEnabled => _decisions is not null || _executions is not null;
|
|
|
|
public string? DecisionPath { get; private init; }
|
|
|
|
/// <summary>Allocates the id that ties a decision row to its execution row.</summary>
|
|
public long NextDecisionId() => Interlocked.Increment(ref _nextId);
|
|
|
|
/// <summary>
|
|
/// Records one bar evaluation. Called on the market-data thread once per closed
|
|
/// bar per symbol — a handful of times a day on this configuration, so the cost is
|
|
/// irrelevant, but it stays buffered anyway.
|
|
/// </summary>
|
|
public void Decision(
|
|
long decisionId,
|
|
string symbol,
|
|
in Bar bar,
|
|
IStrategy strategy,
|
|
in PositionView position,
|
|
in Signal signal,
|
|
in Quote quote,
|
|
double quoteAgeSeconds,
|
|
double equity,
|
|
bool sessionOpen,
|
|
bool halted)
|
|
{
|
|
if (_decisions is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
IReadOnlyList<StrategyMetric> metrics = strategy.Diagnostics;
|
|
|
|
lock (_gate)
|
|
{
|
|
if (!_decisionHeaderWritten)
|
|
{
|
|
WriteDecisionHeader(metrics);
|
|
}
|
|
|
|
_row.Clear();
|
|
|
|
Add(bar.TimeUtc.ToString("O", CultureInfo.InvariantCulture));
|
|
Add(decisionId);
|
|
Add(symbol);
|
|
Add(strategy.Name);
|
|
Add(strategy.IsReady ? 1 : 0);
|
|
|
|
Add(bar.Open);
|
|
Add(bar.High);
|
|
Add(bar.Low);
|
|
Add(bar.Close);
|
|
Add(bar.Volume);
|
|
Add(bar.TakerBuyVolume);
|
|
Add(bar.Delta);
|
|
Add(bar.TradeCount);
|
|
|
|
// Indicator values, in the same order the header declared.
|
|
foreach (string name in _metricNames)
|
|
{
|
|
double value = 0;
|
|
foreach (StrategyMetric m in metrics)
|
|
{
|
|
if (m.Name == name)
|
|
{
|
|
value = double.IsFinite(m.Value) ? m.Value : 0;
|
|
break;
|
|
}
|
|
}
|
|
|
|
Add(value);
|
|
}
|
|
|
|
Add(position.Quantity);
|
|
Add(position.AverageEntryPrice);
|
|
Add(position.UnrealizedPnl);
|
|
Add(position.BarsHeld);
|
|
Add(position.StopPrice);
|
|
Add(position.TargetPrice);
|
|
|
|
Add(signal.Kind.ToString());
|
|
Add(signal.Strength);
|
|
Add(signal.StopPrice);
|
|
Add(signal.TargetPrice);
|
|
|
|
Add(quote.IsValid ? quote.BidPrice : 0);
|
|
Add(quote.IsValid ? quote.AskPrice : 0);
|
|
Add(quote.IsValid ? quote.RelativeSpread : 0);
|
|
Add(quoteAgeSeconds);
|
|
|
|
Add(equity);
|
|
Add(sessionOpen ? 1 : 0);
|
|
Add(halted ? 1 : 0);
|
|
Add(signal.Reason, last: true);
|
|
|
|
_decisions.WriteLine(_row.ToString());
|
|
MaybeFlush();
|
|
}
|
|
}
|
|
|
|
/// <summary>Records what the order path did with a signal.</summary>
|
|
public void Execution(
|
|
long decisionId,
|
|
string symbol,
|
|
Side side,
|
|
string phase,
|
|
bool approved,
|
|
string riskReason,
|
|
string riskDetail,
|
|
double quantity,
|
|
double referencePrice,
|
|
double stopPrice,
|
|
double targetPrice,
|
|
double equity,
|
|
double buyingPower,
|
|
double grossExposure,
|
|
int openPositions,
|
|
string? orderId,
|
|
string? error,
|
|
double latencyMs)
|
|
{
|
|
if (_executions is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
lock (_gate)
|
|
{
|
|
_row.Clear();
|
|
|
|
Add(DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture));
|
|
Add(decisionId);
|
|
Add(symbol);
|
|
Add(side.ToString());
|
|
Add(phase);
|
|
Add(approved ? 1 : 0);
|
|
Add(riskReason);
|
|
Add(riskDetail);
|
|
Add(quantity);
|
|
Add(referencePrice);
|
|
Add(stopPrice);
|
|
Add(targetPrice);
|
|
Add(quantity * referencePrice);
|
|
Add(equity);
|
|
Add(buyingPower);
|
|
Add(grossExposure);
|
|
Add(openPositions);
|
|
Add(orderId ?? string.Empty);
|
|
Add(error ?? string.Empty);
|
|
Add(latencyMs, last: true);
|
|
|
|
_executions.WriteLine(_row.ToString());
|
|
MaybeFlush();
|
|
}
|
|
}
|
|
|
|
private void WriteDecisionHeader(IReadOnlyList<StrategyMetric> metrics)
|
|
{
|
|
string[] names = new string[metrics.Count];
|
|
for (int i = 0; i < metrics.Count; i++)
|
|
{
|
|
names[i] = metrics[i].Name;
|
|
}
|
|
|
|
_metricNames = names;
|
|
_decisionHeaderWritten = true;
|
|
|
|
if (_decisions!.BaseStream.Length > 0)
|
|
{
|
|
// Appending to an existing file: keep its header rather than writing a
|
|
// second one in the middle.
|
|
return;
|
|
}
|
|
|
|
StringBuilder header = new(400);
|
|
header.Append("barTimeUtc,decisionId,symbol,strategy,ready,")
|
|
.Append("open,high,low,close,volume,takerBuyVolume,delta,trades,");
|
|
|
|
foreach (string name in names)
|
|
{
|
|
header.Append(name).Append(',');
|
|
}
|
|
|
|
header.Append("positionQty,positionEntry,positionPnl,barsHeld,positionStop,positionTarget,")
|
|
.Append("signal,signalStrength,signalStop,signalTarget,")
|
|
.Append("bid,ask,spreadPct,quoteAgeSec,equity,sessionOpen,halted,reason");
|
|
|
|
_decisions.WriteLine(header.ToString());
|
|
}
|
|
|
|
private void Add(double value, bool last = false)
|
|
{
|
|
if (double.IsFinite(value))
|
|
{
|
|
_row.Append(value.ToString("G10", CultureInfo.InvariantCulture));
|
|
}
|
|
|
|
if (!last)
|
|
{
|
|
_row.Append(',');
|
|
}
|
|
}
|
|
|
|
private void Add(long value, bool last = false)
|
|
{
|
|
_row.Append(value.ToString(CultureInfo.InvariantCulture));
|
|
if (!last)
|
|
{
|
|
_row.Append(',');
|
|
}
|
|
}
|
|
|
|
private void Add(string? value, bool last = false)
|
|
{
|
|
if (!string.IsNullOrEmpty(value))
|
|
{
|
|
// Quote only when necessary; a reason string routinely contains commas.
|
|
if (value.AsSpan().IndexOfAny(',', '"', '\n') >= 0)
|
|
{
|
|
_row.Append('"').Append(value.Replace("\"", "\"\"", StringComparison.Ordinal)).Append('"');
|
|
}
|
|
else
|
|
{
|
|
_row.Append(value);
|
|
}
|
|
}
|
|
|
|
if (!last)
|
|
{
|
|
_row.Append(',');
|
|
}
|
|
}
|
|
|
|
private void MaybeFlush()
|
|
{
|
|
if (DateTime.UtcNow - _lastFlush < TimeSpan.FromSeconds(5))
|
|
{
|
|
return;
|
|
}
|
|
|
|
_lastFlush = DateTime.UtcNow;
|
|
Flush();
|
|
}
|
|
|
|
public void Flush()
|
|
{
|
|
lock (_gate)
|
|
{
|
|
try
|
|
{
|
|
_decisions?.Flush();
|
|
_executions?.Flush();
|
|
}
|
|
catch (IOException ex)
|
|
{
|
|
Log.Warn($"analytics flush failed: {ex.Message}");
|
|
}
|
|
}
|
|
}
|
|
|
|
private static StreamWriter? Open(string? path)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(path))
|
|
{
|
|
return null;
|
|
}
|
|
|
|
try
|
|
{
|
|
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!);
|
|
return new StreamWriter(
|
|
new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite, 16384),
|
|
Encoding.UTF8)
|
|
{ AutoFlush = false };
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
Log.Warn($"cannot open analytics file {path}: {ex.Message}");
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
lock (_gate)
|
|
{
|
|
try
|
|
{
|
|
_decisions?.Flush();
|
|
_executions?.Flush();
|
|
}
|
|
catch (IOException)
|
|
{
|
|
// Best effort on shutdown.
|
|
}
|
|
|
|
_decisions?.Dispose();
|
|
_executions?.Dispose();
|
|
}
|
|
}
|
|
}
|