Recover missed work after an absence by the rules of SPEC 5.6 (refs #49)
At the start the host measures how long the bot was away from the last job that finished, records the absence and writes it in the diary; one longer than a day is kept for the Panoramica until the user has seen it. A job can now say how a run missed while the bot was off is treated: the nightly backup, the exchange rates, the market calendar and the cleanup run once at the restart, queued, however late; a job with a deadline is skipped past it and the diary says so. Corporate actions are read before the reconciliation puts any stop back. A decision the engine makes late because of an absence is written in the diary as recovered, as are the orders it releases late; entries that moved too far or expired are dropped with their reason. Tests switch the bot off for 3 hours, 20 hours and 3 days with a manual clock and check each of those rows. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
41e067e7d1
commit
680cab4bb6
9 files changed
+454
-18
No files matched your search
@@ -312,7 +312,28 @@ internal sealed partial class AnubisHost(
|
||||
// The laboratory computes on its own thread and reads only local data; the demo shows appendix F and runs none.
|
||||
_lab = options.Demo ? null : new Laboratory(_db, clock, Config.Data.Research, logger);
|
||||
_services = new AppServices(_db, settings, secrets, access, backups, scheduler, _market, _trading, _lab);
|
||||
StartScheduler(scheduler, backups);
|
||||
|
||||
// Before the scheduler writes its first run: how long the bot was away, and which runs it missed (SPEC 5.6).
|
||||
var lastDues = new Dictionary<string, DateTimeOffset>(StringComparer.Ordinal);
|
||||
try
|
||||
{
|
||||
if (!options.Demo)
|
||||
{
|
||||
await new AbsenceBook(_db, clock) { Zone = Config.TimeZone }.RecordAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
foreach (var (name, due) in await _db.ReadAsync(s => s.QueryAsync("SELECT name, last_due_at FROM jobs WHERE last_due_at IS NOT NULL",
|
||||
r => (r.Text("name"), r.Moment("last_due_at"))), cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
lastDues[name] = due;
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
LogStepFailed(logger, ex);
|
||||
}
|
||||
|
||||
StartScheduler(scheduler, backups, lastDues);
|
||||
SetStatus(HostMode.Ready, null);
|
||||
}
|
||||
|
||||
@@ -431,14 +452,15 @@ internal sealed partial class AnubisHost(
|
||||
!options.Demo && (overrides is null ? true : overrides.Trading == true) &&
|
||||
environment.GetValueOrDefault("ANUBIS_TRADING") is not ("0" or "off" or "false");
|
||||
|
||||
private void StartScheduler(Scheduler scheduler, Backups backups)
|
||||
private void StartScheduler(Scheduler scheduler, Backups backups, Dictionary<string, DateTimeOffset> lastDues)
|
||||
{
|
||||
scheduler.Add(new ScheduledJob("backup", "Salvataggio del database", Schedule.Daily(new TimeOnly(3, 30), Config.TimeZone),
|
||||
// A backup missed while the bot was off is taken once at the restart, whenever it comes (SPEC 5.6).
|
||||
var backup = new ScheduledJob("backup", "Salvataggio del database", Schedule.Daily(new TimeOnly(3, 30), Config.TimeZone),
|
||||
async ct => await backups.TakeAsync(_db!, "notturno", ct).ConfigureAwait(false))
|
||||
{ MaxLateness = TimeSpan.FromHours(2) });
|
||||
foreach (var job in (_market?.Jobs() ?? []).Concat(_trading?.Jobs() ?? []))
|
||||
{ MaxLateness = TimeSpan.FromHours(2), CatchUpWithin = TimeSpan.FromDays(3650) };
|
||||
foreach (var job in new[] { backup }.Concat(_market?.Jobs() ?? []).Concat(_trading?.Jobs() ?? []))
|
||||
{
|
||||
scheduler.Add(job);
|
||||
scheduler.Add(job, lastDues.TryGetValue(job.Name, out var due) ? due : null);
|
||||
}
|
||||
|
||||
_schedulerStop = new CancellationTokenSource();
|
||||
@@ -525,7 +547,13 @@ internal sealed partial class AnubisHost(
|
||||
{
|
||||
await _db!.WriteAsync(async s =>
|
||||
{
|
||||
var description = _services?.Scheduler.Upcoming(100).FirstOrDefault(u => u.Name == run.Name)?.Description ?? run.Name;
|
||||
var description = run.Description ?? _services?.Scheduler.Upcoming(100).FirstOrDefault(u => u.Name == run.Name)?.Description ?? run.Name;
|
||||
if (AbsenceBook.MissedLine(run with { Description = description }, Config.TimeZone) is { } line)
|
||||
{
|
||||
await Diary.WriteAsync(s, clock, DiaryKinds.System, run.Outcome == JobOutcome.Completed ? "job_recovered" : "job_skipped", line,
|
||||
new { job = run.Name, due = run.DueUtc }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var updated = await s.ExecuteAsync(
|
||||
"UPDATE jobs SET last_due_at = @d, last_started_at = @s, last_finished_at = @f, last_outcome = @o, last_error = @e, updated_at = @f WHERE name = @n",
|
||||
("d", run.DueUtc), ("s", run.StartedUtc), ("f", run.FinishedUtc), ("o", run.Outcome.ToString().ToLowerInvariant()), ("e", run.Error), ("n", run.Name)).ConfigureAwait(false);
|
||||
|
||||
@@ -121,6 +121,12 @@ public sealed record ScheduledJob(string Name, string Description, Schedule Sche
|
||||
|
||||
/// <summary>A move of the bot the user reads in "Prossime mosse" (SPEC 13.6), not a housekeeping job.</summary>
|
||||
public bool IsMove { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// A run missed while the bot was off is made once at the restart, queued, when the restart comes within this
|
||||
/// time of the missed run; later it is skipped and reported (SPEC 5.6). Null: a missed run is simply not made.
|
||||
/// </summary>
|
||||
public TimeSpan? CatchUpWithin { get; init; }
|
||||
}
|
||||
|
||||
public enum JobOutcome
|
||||
@@ -131,7 +137,13 @@ public enum JobOutcome
|
||||
}
|
||||
|
||||
/// <summary>Published on the bus after every run, skip or failure.</summary>
|
||||
public sealed record JobRun(string Name, DateTimeOffset DueUtc, DateTimeOffset StartedUtc, DateTimeOffset FinishedUtc, JobOutcome Outcome, string? Error);
|
||||
public sealed record JobRun(string Name, DateTimeOffset DueUtc, DateTimeOffset StartedUtc, DateTimeOffset FinishedUtc, JobOutcome Outcome, string? Error)
|
||||
{
|
||||
/// <summary>The run was missed while the bot was off and made, or skipped, at the restart (SPEC 5.6).</summary>
|
||||
public bool Missed { get; init; }
|
||||
|
||||
public string? Description { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>A run coming up, for "Prossime mosse" and the jobs list.</summary>
|
||||
public sealed record UpcomingRun(string Name, string Description, DateTimeOffset DueUtc)
|
||||
@@ -148,21 +160,60 @@ public sealed partial class Scheduler(IClock clock, EventBus bus, ILogger<Schedu
|
||||
private readonly ILogger _logger = logger ?? NullLogger<Scheduler>.Instance;
|
||||
private readonly Lock _gate = new();
|
||||
private readonly Dictionary<string, (ScheduledJob Job, DateTimeOffset? Due)> _jobs = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, DateTimeOffset> _recovering = new(StringComparer.Ordinal);
|
||||
private readonly List<JobRun> _missed = [];
|
||||
private TaskCompletionSource _changed = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
/// <summary>Moment of the last loop iteration; the health check uses it to see the loop is alive.</summary>
|
||||
public DateTimeOffset? LastBeatUtc { get; private set; }
|
||||
|
||||
public void Add(ScheduledJob job)
|
||||
public void Add(ScheduledJob job) => Add(job, null);
|
||||
|
||||
/// <summary>
|
||||
/// Adds a job whose last run, from an earlier life of the process, was due at <paramref name="lastDue"/>. A job with
|
||||
/// <see cref="ScheduledJob.CatchUpWithin"/> that missed a run since then makes it once at the start, or reports it
|
||||
/// skipped when the restart is too late (SPEC 5.6).
|
||||
/// </summary>
|
||||
public void Add(ScheduledJob job, DateTimeOffset? lastDue)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(job);
|
||||
var now = clock.UtcNow;
|
||||
var missed = job.CatchUpWithin is not null && lastDue is { } last ? LastMissed(job.Schedule, last, now) : null;
|
||||
lock (_gate)
|
||||
{
|
||||
_jobs[job.Name] = (job, job.Schedule.NextAfter(clock.UtcNow));
|
||||
if (missed is { } due && now - due <= job.CatchUpWithin)
|
||||
{
|
||||
_jobs[job.Name] = (job, now);
|
||||
_recovering[job.Name] = due;
|
||||
}
|
||||
else
|
||||
{
|
||||
_jobs[job.Name] = (job, job.Schedule.NextAfter(now));
|
||||
if (missed is { } skipped)
|
||||
{
|
||||
_missed.Add(new JobRun(job.Name, skipped, now, now, JobOutcome.Skipped, "Saltato al riavvio: oltre l'orario utile.")
|
||||
{ Missed = true, Description = job.Description });
|
||||
}
|
||||
}
|
||||
|
||||
_changed.TrySetResult();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The last due time after <paramref name="last"/> and not after <paramref name="now"/>: the run the absence took.</summary>
|
||||
private static DateTimeOffset? LastMissed(Schedule schedule, DateTimeOffset last, DateTimeOffset now)
|
||||
{
|
||||
DateTimeOffset? missed = null;
|
||||
var at = schedule.NextAfter(last);
|
||||
for (var i = 0; i < 100_000 && at is { } due && due <= now; i++)
|
||||
{
|
||||
missed = due;
|
||||
at = schedule.NextAfter(due);
|
||||
}
|
||||
|
||||
return missed;
|
||||
}
|
||||
|
||||
public IReadOnlyList<UpcomingRun> Upcoming(int count = 10)
|
||||
{
|
||||
lock (_gate)
|
||||
@@ -224,8 +275,13 @@ public sealed partial class Scheduler(IClock clock, EventBus bus, ILogger<Schedu
|
||||
{
|
||||
var now = clock.UtcNow;
|
||||
List<(ScheduledJob Job, DateTimeOffset Due)> due;
|
||||
List<JobRun> missed;
|
||||
lock (_gate)
|
||||
{
|
||||
// The runs an absence took and the restart came too late for: reported once, at the first round.
|
||||
missed = [.. _missed];
|
||||
_missed.Clear();
|
||||
|
||||
due = _jobs.Values.Where(j => j.Due is { } d && d <= now).Select(j => (j.Job, j.Due!.Value)).OrderBy(j => j.Value).ToList();
|
||||
foreach (var (job, _) in due)
|
||||
{
|
||||
@@ -233,9 +289,28 @@ public sealed partial class Scheduler(IClock clock, EventBus bus, ILogger<Schedu
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var (job, dueAt) in due)
|
||||
foreach (var run in missed)
|
||||
{
|
||||
bus.Publish(run);
|
||||
}
|
||||
|
||||
foreach (var (job, queuedAt) in due)
|
||||
{
|
||||
var started = clock.UtcNow;
|
||||
bool recovering;
|
||||
DateTimeOffset dueAt;
|
||||
lock (_gate)
|
||||
{
|
||||
recovering = _recovering.Remove(job.Name, out var missedDue);
|
||||
dueAt = recovering ? missedDue : queuedAt;
|
||||
}
|
||||
|
||||
if (recovering)
|
||||
{
|
||||
await RecoverAsync(job, dueAt, started, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (started - dueAt > job.MaxLateness)
|
||||
{
|
||||
LogSkipped(_logger, job.Name, dueAt);
|
||||
@@ -256,6 +331,21 @@ public sealed partial class Scheduler(IClock clock, EventBus bus, ILogger<Schedu
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A run missed while the bot was off, made once now: never skipped for lateness, reported as recovered.</summary>
|
||||
private async Task RecoverAsync(ScheduledJob job, DateTimeOffset dueAt, DateTimeOffset started, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await job.Run(cancellationToken).ConfigureAwait(false);
|
||||
bus.Publish(new JobRun(job.Name, dueAt, started, clock.UtcNow, JobOutcome.Completed, null) { Missed = true, Description = job.Description });
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
LogFailed(_logger, ex, job.Name);
|
||||
bus.Publish(new JobRun(job.Name, dueAt, started, clock.UtcNow, JobOutcome.Failed, ex.Message) { Missed = true, Description = job.Description });
|
||||
}
|
||||
}
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Job {Job} skipped: it was due at {Due} and is too late to run")]
|
||||
private static partial void LogSkipped(ILogger logger, string job, DateTimeOffset due);
|
||||
|
||||
|
||||
@@ -105,21 +105,22 @@ public sealed partial class MarketDataService : IDisposable
|
||||
{ MaxLateness = TimeSpan.FromHours(6) },
|
||||
new("dati-azioni-chiusura", "Barre giornaliere delle azioni dopo la chiusura", Schedule.Market(MarketAnchor.Close, TimeSpan.FromMinutes(30), UsCalendar),
|
||||
_ => WakeWorker()) { MaxLateness = TimeSpan.FromHours(6) },
|
||||
// SPEC 5.6: after an absence the last rate available is downloaded at once.
|
||||
new("dati-cambi-mattina", "Cambi di riferimento della BCE", Schedule.Daily(new TimeOnly(8, 0), Zones.Rome), References.RefreshFxAsync)
|
||||
{ MaxLateness = TimeSpan.FromHours(4) },
|
||||
{ MaxLateness = TimeSpan.FromHours(4), CatchUpWithin = TimeSpan.FromDays(3650) },
|
||||
new("dati-cambi-pomeriggio", "Cambi di riferimento della BCE", Schedule.Daily(new TimeOnly(16, 30), Zones.Rome), References.RefreshFxAsync)
|
||||
{ MaxLateness = TimeSpan.FromHours(4) },
|
||||
{ MaxLateness = TimeSpan.FromHours(4), CatchUpWithin = TimeSpan.FromDays(3650) },
|
||||
new("dati-calendario", "Calendario economico, riunioni e trimestrali", Schedule.Every(TimeSpan.FromHours(1)), RefreshCalendarAsync)
|
||||
{ MaxLateness = TimeSpan.FromMinutes(50) },
|
||||
new("dati-umore", "Umore del mercato", Schedule.Every(TimeSpan.FromHours(1)), References.RefreshMoodAsync) { MaxLateness = TimeSpan.FromMinutes(50) },
|
||||
new("dati-spread", "Campione orario degli spread delle crypto", Schedule.Every(TimeSpan.FromHours(1)), SampleSpreadsAsync)
|
||||
{ MaxLateness = TimeSpan.FromMinutes(50) },
|
||||
new("dati-calendario-borsa", "Calendario di borsa di Alpaca", Schedule.Daily(new TimeOnly(4, 0), Zones.Rome), RefreshUsCalendarAsync)
|
||||
{ MaxLateness = TimeSpan.FromHours(12) },
|
||||
{ MaxLateness = TimeSpan.FromHours(12), CatchUpWithin = TimeSpan.FromDays(3650) },
|
||||
new("dati-azioni-societarie", "Frazionamenti e dividendi prima dell'apertura", Schedule.Market(MarketAnchor.Open, TimeSpan.FromHours(-1), UsCalendar),
|
||||
RefreshCorporateActionsAsync) { MaxLateness = TimeSpan.FromMinutes(50) },
|
||||
new("dati-pulizia", "Pulizia delle barre al minuto e degli spread vecchi", Schedule.Daily(new TimeOnly(3, 45), Zones.Rome), PruneAsync)
|
||||
{ MaxLateness = TimeSpan.FromHours(12) },
|
||||
{ MaxLateness = TimeSpan.FromHours(12), CatchUpWithin = TimeSpan.FromDays(3650) },
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
@@ -420,7 +421,8 @@ public sealed partial class MarketDataService : IDisposable
|
||||
await UsCalendar.SaveAsync(sessions, today.AddDays(-30), today.AddDays(90), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task RefreshCorporateActionsAsync(CancellationToken cancellationToken)
|
||||
/// <summary>Splits and dividends of the stock universe from Alpaca; read first at a restart, before any stop is touched (SPEC 5.6, 6.10).</summary>
|
||||
public async Task RefreshCorporateActionsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (await AlpacaAsync(cancellationToken).ConfigureAwait(false) is not { } alpaca)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
using Anubis.Core;
|
||||
using Anubis.Data;
|
||||
using Anubis.Db;
|
||||
|
||||
namespace Anubis.Trading;
|
||||
|
||||
/// <summary>A time the bot was off: from its last sign of life to the restart (SPEC 5.6).</summary>
|
||||
public sealed record Absence(Guid Id, DateTimeOffset StartedAt, DateTimeOffset EndedAt, DateTimeOffset? AcknowledgedAt)
|
||||
{
|
||||
public TimeSpan Length => EndedAt - StartedAt;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The absences of the bot (SPEC 5.6): measured at the restart from the last job that ran, written in the diary, and
|
||||
/// declared at the top of the Panoramica when longer than a day, until the user has seen it.
|
||||
/// </summary>
|
||||
public sealed class AbsenceBook(Database db, IClock clock)
|
||||
{
|
||||
/// <summary>Shorter than this is a restart, not an absence: the minute pulse runs every minute.</summary>
|
||||
public static readonly TimeSpan Threshold = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>An absence longer than this is declared on the Panoramica.</summary>
|
||||
public static readonly TimeSpan Declared = TimeSpan.FromHours(24);
|
||||
|
||||
public TimeZoneInfo Zone { get; init; } = Zones.Rome;
|
||||
|
||||
/// <summary>
|
||||
/// At the start, before the scheduler: the time since the last job finished. An absence is recorded and written in
|
||||
/// the diary; null when the bot was not away (or never ran before).
|
||||
/// </summary>
|
||||
public async Task<Absence?> RecordAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var now = clock.UtcNow;
|
||||
return await db.WriteAsync(async s =>
|
||||
{
|
||||
var last = await s.FirstOrDefaultAsync("SELECT MAX(last_finished_at) AS last FROM jobs", r => r.IsNull("last") ? (DateTimeOffset?)null : r.Moment("last"))
|
||||
.ConfigureAwait(false);
|
||||
if (last is not { } since || now - since < Threshold)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var absence = new Absence(Ids.NewId(clock), since, now, null);
|
||||
await s.ExecuteAsync("INSERT INTO absences (id, started_at, ended_at, acknowledged_at, created_at) VALUES (@id, @s, @e, NULL, @now)", cancellationToken,
|
||||
("id", absence.Id), ("s", since), ("e", now), ("now", now)).ConfigureAwait(false);
|
||||
await Diary.WriteAsync(s, clock, DiaryKinds.System, "absence", DiaryText.Absence(absence.Length), new { started = since, ended = now }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return absence;
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>The absence the Panoramica declares: the last one longer than a day the user has not acknowledged.</summary>
|
||||
public async Task<Absence?> DeclaredAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var rows = await db.ReadAsync(s => s.QueryAsync(
|
||||
"SELECT id, started_at, ended_at, acknowledged_at FROM absences WHERE acknowledged_at IS NULL ORDER BY ended_at DESC",
|
||||
r => new Absence(r.Id("id"), r.Moment("started_at"), r.Moment("ended_at"), r.MomentOrNull("acknowledged_at"))), cancellationToken).ConfigureAwait(false);
|
||||
return rows.FirstOrDefault(a => a.Length > Declared);
|
||||
}
|
||||
|
||||
/// <summary>The user has seen the notice: every absence up to this one stops being declared.</summary>
|
||||
public Task AcknowledgeAsync(Guid id, CancellationToken cancellationToken) =>
|
||||
db.WriteAsync(async s =>
|
||||
{
|
||||
var ended = await s.FirstOrDefaultAsync("SELECT ended_at FROM absences WHERE id = @id", r => (DateTimeOffset?)r.Moment("ended_at"), ("id", id)).ConfigureAwait(false);
|
||||
if (ended is { } until)
|
||||
{
|
||||
await s.ExecuteAsync("UPDATE absences SET acknowledged_at = @now WHERE acknowledged_at IS NULL AND ended_at <= @e", cancellationToken, ("now", clock.UtcNow), ("e", until))
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>The diary line of a run missed while the bot was off: made at the restart, or skipped with its reason.</summary>
|
||||
public static string? MissedLine(JobRun run, TimeZoneInfo zone)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(run);
|
||||
if (!run.Missed)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var what = Lower(run.Description ?? run.Name);
|
||||
return run.Outcome switch
|
||||
{
|
||||
JobOutcome.Completed => DiaryText.Recovered(what, run.DueUtc, zone),
|
||||
JobOutcome.Skipped => DiaryText.Skipped($"{what} (previsto alle {TimeZoneInfo.ConvertTime(run.DueUtc, zone):HH:mm})", "Il riavvio è arrivato oltre l'orario utile."),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static string Lower(string text) => text.Length == 0 ? text : char.ToLowerInvariant(text[0]) + text[1..];
|
||||
}
|
||||
@@ -749,6 +749,14 @@ public sealed partial class LiveEngine : IDisposable
|
||||
summary += " " + ReasonCatalog.Text(LiveReasons.LateExitsOnly);
|
||||
}
|
||||
|
||||
if (m.Now - m.Due > Options.RecoveredAfter)
|
||||
{
|
||||
// Made late because the bot was off (SPEC 5.6): said before what was decided.
|
||||
var what = m.Strategy == StrategyNames.CryptoTrend ? "la chiusura giornaliera crypto" : "il calcolo dei segnali sulle azioni";
|
||||
await Diary.WriteAsync(s, _clock, DiaryKinds.Decisions, LiveReasons.Recovered, DiaryText.Recovered(what, m.Due, Zone), new { decision = decisionId },
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await Diary.WriteAsync(s, _clock, DiaryKinds.Decisions, "decision", summary, new { decision = decisionId }, cancellationToken).ConfigureAwait(false);
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -263,6 +263,14 @@ public sealed partial class TradingService : IDisposable
|
||||
}
|
||||
|
||||
await GuardAsync("simulatore", () => FundSimulatorsAsync(cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// SPEC 5.6: corporate actions come first, so the stops the reconciliation puts back already know about a split.
|
||||
if (_market is not null)
|
||||
{
|
||||
await GuardAsync("azioni societarie", () => _market.RefreshCorporateActionsAsync(cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await GuardAsync("azioni societarie nei registri", () => CorporateActionsAsync(cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
foreach (var manager in _managers.Values)
|
||||
{
|
||||
await GuardAsync("avvio " + manager.Account.Name, async () =>
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
using Anubis.Core;
|
||||
using Anubis.Data;
|
||||
using Anubis.Trading;
|
||||
|
||||
namespace Anubis.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Recovery after an absence (issue #49, SPEC 5.6): the bot is switched off for 3 hours, 20 hours and 3 days with a
|
||||
/// simulated clock, and the rows of 5.6 whose jobs exist in this phase are checked at the restart.
|
||||
/// </summary>
|
||||
[TestClass]
|
||||
public sealed class AbsenceTests
|
||||
{
|
||||
private static readonly CancellationToken _none = CancellationToken.None;
|
||||
private static readonly DateOnly LiveFrom = new(2026, 9, 1);
|
||||
|
||||
// The bot is switched off the evening before the first live day, at 23:00 UTC.
|
||||
private static readonly DateTimeOffset Off = new(2026, 8, 31, 23, 0, 0, TimeSpan.Zero);
|
||||
|
||||
/// <summary>
|
||||
/// Off from 23:00 to 02:00 UTC: the crypto decision of 00:05 is made at the restart and its purchases go if the
|
||||
/// price is within 2% of the one at 00:05; one that moved more is dropped. Both end in the diary with the reason.
|
||||
/// The missed backup is taken once; the morning summary is not due yet; no notice on the Panoramica.
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
public async Task ThreeHoursOffRecoverTheCryptoDecisionAndDropAnEntryThatMovedTooFar()
|
||||
{
|
||||
await using var live = await LiveRig.CreateAsync(LiveFrom, LiveFrom.AddDays(39)).ConfigureAwait(false);
|
||||
var jobs = await RestartAsync(live.Rig, Off.AddHours(3)).ConfigureAwait(false);
|
||||
Assert.AreEqual("Sono stato fermo per 3 ore. Ho ripreso i lavori saltati con le regole del recupero.", await LastDiaryAsync(live.Rig, "absence").ConfigureAwait(false));
|
||||
Assert.IsNull(await new AbsenceBook(live.Rig.Db, live.Rig.Clock).DeclaredAsync(_none).ConfigureAwait(false));
|
||||
Assert.AreEqual(1, jobs.Backups);
|
||||
Assert.AreEqual(0, jobs.Summaries);
|
||||
Assert.AreEqual(0, jobs.Plain, "A job without recovery rules is not run for the time it missed.");
|
||||
|
||||
// The data step brings yesterday's bar; SOL has moved 3% from the price of 00:05, ETH has not.
|
||||
await live.StoreBarAsync(LiveFrom.AddDays(-1)).ConfigureAwait(false);
|
||||
live.Quotes(LiveFrom, b => b.Open);
|
||||
var sol = (decimal)live.Bar("SOL-EUR", LiveFrom).Open * 1.03m;
|
||||
live.Rig.Quote("SOL-EUR", sol * 0.9999m, sol * 1.0001m, 1000m);
|
||||
await live.MinutesAsync(10).ConfigureAwait(false);
|
||||
|
||||
var runs = await live.Rig.QueryAsync("SELECT run_key, mode FROM engine_runs", r => (r.Text("run_key"), r.Text("mode"))).ConfigureAwait(false);
|
||||
CollectionAssert.AreEqual(new[] { ("crypto_trend:2026-08-31", DecisionModes.Full) }, runs.ToList());
|
||||
var fills = await live.LiveFillsAsync().ConfigureAwait(false);
|
||||
Assert.IsTrue(fills.Any(f => f.Symbol == "ETH-EUR" && f.Side == OrderSide.Buy), "The recovered purchase goes.");
|
||||
Assert.IsFalse(fills.Any(f => f.Symbol == "SOL-EUR"), "The purchase whose price moved 3% is dropped.");
|
||||
var diary = await DiaryAsync(live.Rig).ConfigureAwait(false);
|
||||
Assert.Contains("Recuperato un lavoro saltato: la chiusura giornaliera crypto (previsto alle 02:05).", diary);
|
||||
Assert.Contains("Recuperato un lavoro saltato: l'acquisto di ETH/EUR (previsto alle 02:05).", diary);
|
||||
Assert.IsTrue(diary.Any(d => d.StartsWith("Scartato l'acquisto di SOL/EUR: il prezzo si è mosso del 3", StringComparison.Ordinal)), string.Join("\n", diary));
|
||||
var planned = await live.Rig.QueryAsync("SELECT outcome FROM planned_orders WHERE status = @s", r => r.Text("outcome"), ("s", PlannedStates.Dropped)).ConfigureAwait(false);
|
||||
CollectionAssert.AreEqual(new[] { LiveReasons.PriceMoved }, planned.ToList());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Off for 20 hours, back at 19:00 UTC: the crypto decision is past 12:00 and only exits and stops are updated;
|
||||
/// the meanrev purchase decided at the last close expired an hour after the open; the morning summary is
|
||||
/// skipped (past 12:00 in Italy) and said so in the diary; the backup is taken once; no notice on the Panoramica.
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
public async Task TwentyHoursOffLetTheLateDecisionsUpdateOnlyExitsAndDropExpiredEntries()
|
||||
{
|
||||
await using (var live = await LiveRig.CreateAsync(LiveFrom, LiveFrom.AddDays(39)).ConfigureAwait(false))
|
||||
{
|
||||
var jobs = await RestartAsync(live.Rig, Off.AddHours(20)).ConfigureAwait(false);
|
||||
Assert.AreEqual(1, jobs.Backups);
|
||||
Assert.AreEqual(0, jobs.Summaries);
|
||||
Assert.HasCount(1, jobs.Skipped);
|
||||
Assert.AreEqual("Saltato: riepilogo del mattino (previsto alle 08:00). Il riavvio è arrivato oltre l'orario utile.",
|
||||
AbsenceBook.MissedLine(jobs.Skipped[0], Zones.Rome));
|
||||
Assert.IsNull(await new AbsenceBook(live.Rig.Db, live.Rig.Clock).DeclaredAsync(_none).ConfigureAwait(false));
|
||||
|
||||
await live.StoreBarAsync(LiveFrom.AddDays(-1)).ConfigureAwait(false);
|
||||
live.Quotes(LiveFrom, b => b.Close);
|
||||
await live.MinutesAsync(10).ConfigureAwait(false);
|
||||
var runs = await live.Rig.QueryAsync("SELECT mode FROM engine_runs", r => r.Text("mode")).ConfigureAwait(false);
|
||||
CollectionAssert.AreEqual(new[] { DecisionModes.ExitsOnly }, runs.ToList());
|
||||
Assert.IsEmpty(await live.LiveFillsAsync().ConfigureAwait(false), "No purchase after 12:00 UTC: the next close decides.");
|
||||
var decision = await LastDiaryAsync(live.Rig, "decision").ConfigureAwait(false);
|
||||
Assert.Contains(ReasonCatalog.Text(LiveReasons.LateExitsOnly), decision!);
|
||||
}
|
||||
|
||||
// The stock side: the bot goes off after the signals of the close and comes back the next day at 12:30 in New
|
||||
// York, past the hour after the open in which a meanrev purchase is valid.
|
||||
await using var stocks = await StockRig.CreateAsync(LiveFrom, LiveFrom.AddDays(41)).ConfigureAwait(false);
|
||||
await stocks.SessionAsync(LiveFrom).ConfigureAwait(false);
|
||||
var pending = await stocks.Rig.QueryAsync("SELECT id FROM planned_orders WHERE status = @s", r => r.Id("id"), ("s", PlannedStates.Pending)).ConfigureAwait(false);
|
||||
Assert.HasCount(1, pending, "The close of the first day plans a purchase for the next open.");
|
||||
stocks.Rig.Clock.Set(new DateTimeOffset(2026, 9, 2, 16, 30, 0, TimeSpan.Zero));
|
||||
stocks.Quotes(LiveFrom.AddDays(1), b => b.Open);
|
||||
await stocks.MinutesAsync(3).ConfigureAwait(false);
|
||||
var expired = await stocks.Rig.QueryAsync("SELECT outcome FROM planned_orders WHERE status = @s", r => r.Text("outcome"), ("s", PlannedStates.Expired)).ConfigureAwait(false);
|
||||
CollectionAssert.AreEqual(new[] { LiveReasons.EntryExpired }, expired.ToList());
|
||||
Assert.IsEmpty(await stocks.LiveFillsAsync().ConfigureAwait(false));
|
||||
Assert.IsTrue((await DiaryAsync(stocks.Rig).ConfigureAwait(false)).Any(d => d.StartsWith("Saltato: l'acquisto di BBB. Ingresso scaduto", StringComparison.Ordinal)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Off for 3 days: only the decision of the restart day is made (the missed ones are not replayed), the backup is
|
||||
/// taken once and not three times, and the Panoramica declares the absence until the user has seen it.
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
public async Task ThreeDaysOffAreDeclaredUntilSeenAndMissedRunsAreMadeOnce()
|
||||
{
|
||||
await using var live = await LiveRig.CreateAsync(LiveFrom, LiveFrom.AddDays(39)).ConfigureAwait(false);
|
||||
var back = Off.AddDays(3);
|
||||
var jobs = await RestartAsync(live.Rig, back).ConfigureAwait(false);
|
||||
Assert.AreEqual(1, jobs.Backups, "Three nights missed, one backup at the restart.");
|
||||
Assert.AreEqual("Sono stato fermo per 3 giorni. Ho ripreso i lavori saltati con le regole del recupero.", await LastDiaryAsync(live.Rig, "absence").ConfigureAwait(false));
|
||||
|
||||
var book = new AbsenceBook(live.Rig.Db, live.Rig.Clock);
|
||||
var declared = await book.DeclaredAsync(_none).ConfigureAwait(false);
|
||||
Assert.IsNotNull(declared);
|
||||
Assert.AreEqual(TimeSpan.FromDays(3), declared.Length);
|
||||
await book.AcknowledgeAsync(declared.Id, _none).ConfigureAwait(false);
|
||||
Assert.IsNull(await book.DeclaredAsync(_none).ConfigureAwait(false));
|
||||
|
||||
for (var day = LiveFrom.AddDays(-1); day <= LiveFrom.AddDays(2); day = day.AddDays(1))
|
||||
{
|
||||
await live.StoreBarAsync(day).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
live.Quotes(LiveFrom.AddDays(2), b => b.Close);
|
||||
await live.MinutesAsync(5).ConfigureAwait(false);
|
||||
var runs = await live.Rig.QueryAsync("SELECT run_key FROM engine_runs", r => r.Text("run_key")).ConfigureAwait(false);
|
||||
CollectionAssert.AreEqual(new[] { "crypto_trend:2026-09-02" }, runs.ToList());
|
||||
}
|
||||
|
||||
private sealed class Counts
|
||||
{
|
||||
public int Backups { get; set; }
|
||||
|
||||
public int Summaries { get; set; }
|
||||
|
||||
public int Plain { get; set; }
|
||||
|
||||
public List<JobRun> Skipped { get; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The restart: the last sign of life was at <see cref="Off"/>; the clock is now <paramref name="back"/>. The
|
||||
/// absence is recorded, then the scheduler starts with three jobs that ran last at <see cref="Off"/>: the nightly
|
||||
/// backup (made once whenever the restart comes), the morning summary (only until 12:00 in Italy) and a job without
|
||||
/// recovery rules.
|
||||
/// </summary>
|
||||
private static async Task<Counts> RestartAsync(TradingRig rig, DateTimeOffset back)
|
||||
{
|
||||
await rig.Db.WriteAsync(s => s.ExecuteAsync(
|
||||
"INSERT INTO jobs (id, name, description, last_due_at, last_started_at, last_finished_at, last_outcome, created_at, updated_at) VALUES (@id, 'motore', 'Motore', @t, @t, @t, 'completed', @t, @t)",
|
||||
_none, ("id", Ids.NewId(rig.Clock)), ("t", Off)), _none).ConfigureAwait(false);
|
||||
rig.Clock.Set(back);
|
||||
await new AbsenceBook(rig.Db, rig.Clock).RecordAsync(_none).ConfigureAwait(false);
|
||||
|
||||
var counts = new Counts();
|
||||
var bus = new EventBus();
|
||||
using var runs = bus.Subscribe<JobRun>();
|
||||
var scheduler = new Scheduler(rig.Clock, bus);
|
||||
scheduler.Add(new ScheduledJob("backup", "Salvataggio del database", Schedule.Daily(new TimeOnly(3, 30), Zones.Rome), _ =>
|
||||
{
|
||||
counts.Backups++;
|
||||
return Task.CompletedTask;
|
||||
})
|
||||
{ MaxLateness = TimeSpan.FromHours(2), CatchUpWithin = TimeSpan.FromDays(3650) }, Off);
|
||||
scheduler.Add(new ScheduledJob("riepilogo", "Riepilogo del mattino", Schedule.Daily(new TimeOnly(8, 0), Zones.Rome), _ =>
|
||||
{
|
||||
counts.Summaries++;
|
||||
return Task.CompletedTask;
|
||||
})
|
||||
{ CatchUpWithin = TimeSpan.FromHours(4) }, Off);
|
||||
scheduler.Add(new ScheduledJob("pulizia", "Pulizia", Schedule.Daily(new TimeOnly(1, 0), Zones.Rome), _ =>
|
||||
{
|
||||
counts.Plain++;
|
||||
return Task.CompletedTask;
|
||||
}), Off);
|
||||
await scheduler.RunDueAsync(_none).ConfigureAwait(false);
|
||||
await scheduler.RunDueAsync(_none).ConfigureAwait(false);
|
||||
while (runs.TryRead(out var run))
|
||||
{
|
||||
if (run!.Missed && run.Outcome == JobOutcome.Skipped)
|
||||
{
|
||||
counts.Skipped.Add(run);
|
||||
}
|
||||
else if (run.Missed)
|
||||
{
|
||||
Assert.AreEqual(JobOutcome.Completed, run.Outcome);
|
||||
StringAssert.StartsWith(AbsenceBook.MissedLine(run, Zones.Rome), "Recuperato un lavoro saltato: salvataggio del database (previsto alle 03:30)");
|
||||
}
|
||||
}
|
||||
|
||||
return counts;
|
||||
}
|
||||
|
||||
private static async Task<List<string>> DiaryAsync(TradingRig rig) =>
|
||||
[.. await rig.QueryAsync("SELECT message FROM journal ORDER BY occurred_at, id", r => r.Text("message")).ConfigureAwait(false)];
|
||||
|
||||
private static async Task<string?> LastDiaryAsync(TradingRig rig, string code) =>
|
||||
(await rig.QueryAsync("SELECT message FROM journal WHERE code = @c ORDER BY occurred_at DESC, id DESC", r => r.Text("message"), ("c", code)).ConfigureAwait(false)) is { Count: > 0 } rows ? rows[0] : null;
|
||||
}
|
||||
@@ -134,7 +134,7 @@ internal sealed class StockRig : IAsyncDisposable
|
||||
|
||||
private DailyBar? Bar(string symbol, DateOnly day) => Bars[symbol].SingleOrDefault(b => b.Day == day);
|
||||
|
||||
private void Quotes(DateOnly day, Func<DailyBar, double> pick)
|
||||
public void Quotes(DateOnly day, Func<DailyBar, double> pick)
|
||||
{
|
||||
foreach (var symbol in Bars.Keys)
|
||||
{
|
||||
@@ -146,7 +146,7 @@ internal sealed class StockRig : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private async Task MinutesAsync(int count)
|
||||
public async Task MinutesAsync(int count)
|
||||
{
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
|
||||
@@ -213,6 +213,15 @@ internal sealed class LiveRig : IAsyncDisposable
|
||||
await Equity.TakeAsync(none).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Several minutes of the process from the clock's time.</summary>
|
||||
public async Task MinutesAsync(int count)
|
||||
{
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
await MinuteAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>One minute of the process: the engine's pulse, then a turn of the simulator and of the order manager.</summary>
|
||||
private async Task MinuteAsync()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user