using System.Threading.Channels;
using Chronos.Core.Diagnostics;
using Chronos.Core.Model;
using Chronos.Data.Jobs;
using Chronos.Protocol;
namespace Chronos.Data.Scheduling;
/// How a keeps time.
public sealed record SchedulerOptions
{
/// Clock of the runs (tests pass their own); its timers pace the loop.
public TimeProvider Time { get; init; } = TimeProvider.System;
/// How often the loop looks for due schedules (and for schedules other processes changed).
public TimeSpan Tick { get; init; } = TimeSpan.FromSeconds(30);
///
/// How late a run may be and still count as on time. A schedule further behind (the host was off, the PC slept) missed
/// its run: with catch-up it fires once, without it moves to its next occurrence without running.
///
public TimeSpan CatchUpGrace { get; init; } = TimeSpan.FromMinutes(5);
}
/// What a schedule's run did: the outcome () and the first job it queued.
internal sealed record ScheduleFire(Guid ScheduleId, string Outcome, Guid? JobId);
/// A schedule created or edited, or the first invalid field by its API name (name, recurrence, parameters.steps…).
public sealed record ScheduleEditResult(ScheduleDto? Schedule, string? Problem)
{
/// True when the schedule to edit does not exist.
public bool NotFound => Schedule is null && Problem is null;
}
/// What did.
public enum ScheduleDeleteOutcome
{
Deleted,
NotFound,
/// A system schedule: disable it instead.
System,
}
///
/// Runs the schedules of one host (owner review 2, R10), in the style of the it queues jobs on.
/// seeds the host's system schedules (edits kept), gives enabled schedules without a next run one,
/// moves past the runs that schedules without catch-up missed, fires those with catch-up once, then looks every
/// (or at ). A due run is claimed by moving its next run with a
/// compare-and-set, so several processes on one database fire it once; the claimed run then queues the action's jobs (at the
/// schedule's priority, with its id) unless the schedule's previous job is still pending, and records the outcome. A crash
/// between the claim and the queueing loses that one run (it is logged as claimed first). A next run more than two periods
/// ahead means the clock went back: it is computed again. Edits go through this class so the next run follows them.
///
public sealed class Scheduler : IAsyncDisposable
{
/// Longest schedule name.
public const int MaxNameLength = 100;
/// The only target so far: the host's archive.
public const string CurrentTarget = "current";
private readonly ChronosDatabase _db;
private readonly JobEngine _engine;
private readonly Channel _wake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
private readonly CancellationTokenSource _shutdown = new();
private readonly Lock _gate = new();
private IScheduleHost? _host;
private Task _loop = Task.CompletedTask;
public Scheduler(ChronosDatabase db, JobEngine engine, SchedulerOptions? options = null)
{
_db = db ?? throw new ArgumentNullException(nameof(db));
_engine = engine ?? throw new ArgumentNullException(nameof(engine));
Options = options ?? new SchedulerOptions();
}
public SchedulerOptions Options { get; }
/// The host given to ; null before.
public IScheduleHost? Host => Volatile.Read(ref _host);
/// A schedule was created, edited, reset or ran (also by the loop). Raised on the caller's or the loop's thread.
public event Action? ScheduleChanged;
/// A schedule was deleted.
public event Action? ScheduleDeleted;
///
/// Seeds 's system schedules, settles the runs missed while the host was off (see the type's
/// remarks) and starts the loop. The first pass runs before this returns.
///
public async Task StartAsync(IScheduleHost host, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(host);
if (Host is not null) throw new InvalidOperationException("The scheduler has started already.");
var added = await _db.Schedules.EnsureSystemAsync(host.SystemSchedules, Now(), cancellationToken).ConfigureAwait(false);
if (added > 0) Log.Info("schedules", $"Added {added} system schedule(s)");
if (Interlocked.CompareExchange(ref _host, host, null) is not null) throw new InvalidOperationException("The scheduler has started already.");
try
{
await RunDueAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
Log.Warning("schedules", "The schedules could not be run at start; the loop tries again", ex);
}
var shutdown = _shutdown.Token;
lock (_gate) _loop = Task.Run(() => LoopAsync(shutdown), CancellationToken.None);
}
/// Makes the loop look for due schedules now (another process changed them, the archive came back).
public void Wake() => _wake.Writer.TryWrite(true);
///
/// One pass of the loop: corrects next runs (missing, clock gone back, missed without catch-up), then claims and fires
/// every due schedule. Returns how many fired here.
///
public async Task RunDueAsync(CancellationToken cancellationToken = default)
{
var host = RequireHost();
var now = Now();
foreach (var schedule in await _db.Schedules.ListAsync(cancellationToken).ConfigureAwait(false))
{
if (schedule.Enabled) await CorrectAsync(host, schedule, now, cancellationToken).ConfigureAwait(false);
}
var fired = 0;
foreach (var schedule in await _db.Schedules.ListDueAsync(now, cancellationToken).ConfigureAwait(false))
{
DateTimeOffset? next = ScheduleRecurrence.TryParse(schedule.Recurrence, out var recurrence)
? recurrence.Next(now, host.TimeZone, schedule.NextRun)
: null;
// The claim: one process wins the run; the others see the moved next run and leave it.
if (!await _db.Schedules.TryAdvanceAsync(schedule.Id, schedule.NextRun, next, cancellationToken).ConfigureAwait(false)) continue;
if (next is null)
{
Log.Warning("schedules", $"Schedule {schedule.Id} has an invalid recurrence '{schedule.Recurrence}'; it stops until edited");
await RecordAsync(schedule.Id, null, ScheduleOutcome.Failed("recurrence"), cancellationToken).ConfigureAwait(false);
continue;
}
Log.Debug("schedules", $"{schedule.Name} ({schedule.Action}) due {schedule.NextRun:O} claimed; next {next:O}");
await FireAsync(host, schedule, schedule.CreatedBy, manual: false, cancellationToken).ConfigureAwait(false);
fired++;
}
return fired;
}
///
/// Runs the schedule now, as its next run would (also a disabled one), without moving its next run; null when it does
/// not exist. is stamped on the jobs it queues.
///
public async Task RunNowAsync(Guid id, Guid? userId, CancellationToken cancellationToken = default)
{
var host = RequireHost();
if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } schedule) return null;
var fire = await FireAsync(host, schedule, userId ?? schedule.CreatedBy, manual: true, cancellationToken).ConfigureAwait(false);
var dto = await GetAsync(id, cancellationToken).ConfigureAwait(false) ?? ScheduleMapping.ToDto(schedule);
return new ScheduleRunDto(dto, fire.Outcome, fire.JobId);
}
///
/// Runs the system schedule now (, which skips while its work is
/// pending) when it is enabled and has not run since ; null when it did not run. The hosts' archive
/// scan once at start, as their polling did, and on the desktop when its archive is opened again (owner review 2, R10).
///
public async Task RunSystemNowAsync(string systemKey, DateTimeOffset? since = null, CancellationToken cancellationToken = default)
{
_ = RequireHost();
if (await _db.Schedules.GetBySystemKeyAsync(systemKey, cancellationToken).ConfigureAwait(false) is not { Enabled: true } schedule
|| schedule.LastRun >= since) return null;
return await RunNowAsync(schedule.Id, null, cancellationToken).ConfigureAwait(false);
}
/// The schedules (system ones first), the actions with their availability here, the host's time zone and clean-up steps.
public async Task ListAsync(CancellationToken cancellationToken = default)
{
var host = RequireHost();
var items = await _db.Schedules.ListAsync(cancellationToken).ConfigureAwait(false);
var actions = await ScheduleActions.AvailabilityAsync(Context(host), cancellationToken).ConfigureAwait(false);
return new SchedulesDto([.. items.Select(ScheduleMapping.ToDto)], [.. actions.Select(ScheduleMapping.ToDto)], host.TimeZone.Id, host.CleanupSteps);
}
public async Task GetAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } schedule ? ScheduleMapping.ToDto(schedule) : null;
/// Creates a schedule (enabled ones get their next run at once); the result names the first invalid field instead when there is one.
public async Task CreateAsync(CreateScheduleRequest request, Guid? createdBy, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
var host = RequireHost();
if (Problem(host, request.Name, request.Action, request.Target, request.Recurrence, request.Parameters, request.Priority) is { } problem)
return new ScheduleEditResult(null, problem);
var recurrence = ScheduleMapping.FromDto(request.Recurrence)!;
var now = Now();
var schedule = new Schedule
{
Id = Guid.CreateVersion7(),
Name = request.Name.Trim(),
Action = request.Action,
Target = CurrentTarget,
ParametersJson = ScheduleMapping.ParametersJson(request.Action, request.Parameters),
Recurrence = recurrence.Format(),
CatchUp = request.CatchUp,
Priority = Priority(request.Priority, JobPriority.Background),
Enabled = request.Enabled,
NextRun = request.Enabled ? recurrence.Next(now, host.TimeZone) : null,
CreatedBy = createdBy,
Created = now,
Updated = now,
};
await _db.Schedules.InsertAsync(schedule, cancellationToken).ConfigureAwait(false);
Log.Info("schedules", $"Schedule {schedule.Name} ({schedule.Action}, {schedule.Recurrence}) created");
return new ScheduleEditResult(Changed(schedule), null);
}
///
/// Changes the fields gives. The next run is computed again when the recurrence changes or
/// the schedule is enabled; a disabled schedule has none. A system schedule keeps its name and action (asking to change
/// them is a problem of field name or action).
///
public async Task UpdateAsync(Guid id, UpdateScheduleRequest request, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
var host = RequireHost();
if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } current) return new ScheduleEditResult(null, null);
if (current.IsSystem && request.Name is { } name && name.Trim() != current.Name) return new ScheduleEditResult(null, "name");
if (current.IsSystem && request.Action is { } changed && changed != current.Action) return new ScheduleEditResult(null, "action");
var action = request.Action ?? current.Action;
// Another action starts from its own defaults; the same one keeps the stored values the request leaves out.
var parameters = request.Parameters ?? (action == current.Action ? ScheduleMapping.Parameters(current) : null);
var recurrenceDto = request.Recurrence ?? (ScheduleRecurrence.TryParse(current.Recurrence, out var stored) ? ScheduleMapping.ToDto(stored) : null);
if (Problem(host, request.Name ?? current.Name, action, null, recurrenceDto, parameters, request.Priority) is { } problem)
return new ScheduleEditResult(null, problem);
var recurrence = ScheduleMapping.FromDto(recurrenceDto)!;
var now = Now();
var enabled = request.Enabled ?? current.Enabled;
var reschedule = !enabled || !current.Enabled || current.NextRun is null || recurrence.Format() != current.Recurrence;
var schedule = current with
{
Name = (request.Name ?? current.Name).Trim(),
Action = action,
ParametersJson = ScheduleMapping.ParametersJson(action, parameters),
Recurrence = recurrence.Format(),
CatchUp = request.CatchUp ?? current.CatchUp,
Priority = Priority(request.Priority, current.Priority),
Enabled = enabled,
NextRun = !enabled ? null : reschedule ? recurrence.Next(now, host.TimeZone) : current.NextRun,
Updated = now,
};
if (!await _db.Schedules.UpdateAsync(schedule, reschedule, cancellationToken).ConfigureAwait(false)) return new ScheduleEditResult(null, null);
Log.Info("schedules", $"Schedule {schedule.Name} ({schedule.Action}, {schedule.Recurrence}, {(enabled ? "enabled" : "disabled")}) changed");
var saved = await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) ?? schedule;
return new ScheduleEditResult(Changed(saved), null);
}
/// Deletes a schedule a user created; a system schedule is refused ().
public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default)
{
if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } schedule) return ScheduleDeleteOutcome.NotFound;
if (schedule.IsSystem) return ScheduleDeleteOutcome.System;
if (!await _db.Schedules.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) return ScheduleDeleteOutcome.NotFound;
Log.Info("schedules", $"Schedule {schedule.Name} ({schedule.Action}) deleted");
ScheduleDeleted?.Invoke(id);
return ScheduleDeleteOutcome.Deleted;
}
///
/// Puts a system schedule back to the host's defaults () with a new next
/// run. A schedule a user created, or one the host no longer defines, is a problem of field systemKey.
///
public async Task ResetAsync(Guid id, CancellationToken cancellationToken = default)
{
var host = RequireHost();
if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } current) return new ScheduleEditResult(null, null);
if (current.SystemKey is not { } key || host.SystemSchedules.FirstOrDefault(s => s.SystemKey == key) is not { } defaults)
return new ScheduleEditResult(null, "systemKey");
var now = Now();
await _db.Schedules.ResetSystemAsync(key, defaults, now, cancellationToken).ConfigureAwait(false);
if (defaults.Enabled) await _db.Schedules.TryAdvanceAsync(id, null, defaults.Recurrence.Next(now, host.TimeZone), cancellationToken).ConfigureAwait(false);
Log.Info("schedules", $"System schedule {key} reset");
var saved = await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false);
return saved is null ? new ScheduleEditResult(null, null) : new ScheduleEditResult(Changed(saved), null);
}
public async ValueTask DisposeAsync()
{
Task loop;
lock (_gate) loop = _loop;
await _shutdown.CancelAsync().ConfigureAwait(false);
try
{
await loop.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
_shutdown.Dispose();
}
///
/// Fills in a missing next run, computes it again after the clock went back, and moves a schedule without catch-up past
/// a run it missed. Compare-and-set like a claim, so concurrent passes agree.
///
private async Task CorrectAsync(IScheduleHost host, Schedule schedule, DateTimeOffset now, CancellationToken cancellationToken)
{
if (!ScheduleRecurrence.TryParse(schedule.Recurrence, out var recurrence)) return;
DateTimeOffset next;
string why;
if (schedule.NextRun is not { } due)
{
next = recurrence.Next(now, host.TimeZone);
why = "first run";
}
else if (due - now > recurrence.Period * 2)
{
next = recurrence.Next(now, host.TimeZone);
why = "the clock went back";
}
else if (!schedule.CatchUp && now - due > Options.CatchUpGrace)
{
next = recurrence.Next(now, host.TimeZone, due);
why = $"missed the run of {due:O} (no catch-up)";
}
else
{
return;
}
if (!await _db.Schedules.TryAdvanceAsync(schedule.Id, schedule.NextRun, next, cancellationToken).ConfigureAwait(false)) return;
Log.Info("schedules", $"{schedule.Name} ({schedule.Action}): next run {next:O} ({why})");
Changed(schedule with { NextRun = next });
}
/// Queues the schedule's action unless its previous job is pending, records the outcome and logs one line.
private async Task FireAsync(IScheduleHost host, Schedule schedule, Guid? createdBy, bool manual, CancellationToken cancellationToken)
{
string outcome;
Guid? jobId = null;
try
{
if (schedule.LastJobId is { } last
&& await _db.Jobs.GetAsync(last, cancellationToken).ConfigureAwait(false) is { State: JobState.Queued or JobState.Running or JobState.Paused })
{
outcome = ScheduleOutcome.SkippedPending;
}
else
{
var resolution = await ScheduleActions.ResolveAsync(Context(host), schedule.Action, ScheduleMapping.Parameters(schedule), cancellationToken)
.ConfigureAwait(false);
outcome = resolution.Outcome;
foreach (var request in resolution.Requests)
{
var job = await _engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, schedule.Priority, createdBy,
schedule.Id, cancellationToken).ConfigureAwait(false);
jobId ??= job.Id;
}
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
Log.Warning("schedules", $"{schedule.Name} ({schedule.Action}) could not run", ex);
outcome = ScheduleOutcome.Failed(ex.Message);
}
await RecordAsync(schedule.Id, jobId, outcome, CancellationToken.None).ConfigureAwait(false);
Log.Info("schedules", $"{schedule.Name} ({schedule.Action}{(manual ? ", run now" : string.Empty)}): {outcome}{(jobId is { } id ? $", job {id}" : string.Empty)}");
return new ScheduleFire(schedule.Id, outcome, jobId);
}
private async Task RecordAsync(Guid id, Guid? jobId, string outcome, CancellationToken cancellationToken)
{
await _db.Schedules.RecordOutcomeAsync(id, jobId, outcome, Now(), cancellationToken).ConfigureAwait(false);
if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } saved) Changed(saved);
}
private async Task LoopAsync(CancellationToken shutdown)
{
while (!shutdown.IsCancellationRequested)
{
await WaitAsync(shutdown).ConfigureAwait(false);
if (shutdown.IsCancellationRequested) return;
try
{
await RunDueAsync(shutdown).ConfigureAwait(false);
}
catch (OperationCanceledException) when (shutdown.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
// Nothing a pass throws ends the loop (a database briefly away, a host hook that failed).
Log.Warning("schedules", "The schedules could not be run; the next tick tries again", ex);
}
}
}
/// Waits for , the tick or shutdown, whichever comes first.
private async Task WaitAsync(CancellationToken shutdown)
{
using var tick = new CancellationTokenSource(Options.Tick, Options.Time);
using var either = CancellationTokenSource.CreateLinkedTokenSource(shutdown, tick.Token);
try
{
await _wake.Reader.ReadAsync(either.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
// Tick elapsed or shutdown.
}
}
private ScheduleDto Changed(Schedule schedule)
{
var dto = ScheduleMapping.ToDto(schedule);
ScheduleChanged?.Invoke(dto);
return dto;
}
private ScheduleActionContext Context(IScheduleHost host) => new(_db, host, _engine.CanRun, Now());
private DateTimeOffset Now() => Options.Time.GetUtcNow();
private IScheduleHost RequireHost() => Host ?? throw new InvalidOperationException("Start the scheduler with its host first.");
private static JobPriority Priority(string? wire, JobPriority fallback) => wire is not null && JobWire.TryParsePriority(wire, out var priority) ? priority : fallback;
/// The first invalid field of a schedule as given, by its API name; null when it is valid.
private static string? Problem(IScheduleHost host, string? name, string? action, string? target, RecurrenceDto? recurrence,
ScheduleParametersDto? parameters, string? priority)
{
if (string.IsNullOrWhiteSpace(name) || name.Trim().Length > MaxNameLength) return "name";
if (action is null) return "action";
if (ScheduleActions.Problem(action, parameters, host.CleanupSteps) is { } invalid) return invalid;
if (target is not null && target != CurrentTarget) return "target";
if (ScheduleMapping.FromDto(recurrence) is null) return "recurrence";
if (priority is not null && !JobWire.TryParsePriority(priority, out _)) return "priority";
return null;
}
}