using System.Threading.Channels; using Chronos.Core.Diagnostics; using Chronos.Core.Model; using Chronos.Data.Jobs; using Chronos.Protocol; namespace Chronos.Data.Scheduling; /// How a keeps time. public sealed record SchedulerOptions { /// Clock of the runs (tests pass their own); its timers pace the loop. public TimeProvider Time { get; init; } = TimeProvider.System; /// How often the loop looks for due schedules (and for schedules other processes changed). public TimeSpan Tick { get; init; } = TimeSpan.FromSeconds(30); /// /// How late a run may be and still count as on time. A schedule further behind (the host was off, the PC slept) missed /// its run: with catch-up it fires once, without it moves to its next occurrence without running. /// public TimeSpan CatchUpGrace { get; init; } = TimeSpan.FromMinutes(5); } /// What a schedule's run did: the outcome () and the first job it queued. internal sealed record ScheduleFire(Guid ScheduleId, string Outcome, Guid? JobId); /// A schedule created or edited, or the first invalid field by its API name (name, recurrence, parameters.steps…). public sealed record ScheduleEditResult(ScheduleDto? Schedule, string? Problem) { /// True when the schedule to edit does not exist. public bool NotFound => Schedule is null && Problem is null; } /// What did. public enum ScheduleDeleteOutcome { Deleted, NotFound, /// A system schedule: disable it instead. System, } /// /// Runs the schedules of one host (owner review 2, R10), in the style of the it queues jobs on. /// seeds the host's system schedules (edits kept), gives enabled schedules without a next run one, /// moves past the runs that schedules without catch-up missed, fires those with catch-up once, then looks every /// (or at ). A due run is claimed by moving its next run with a /// compare-and-set, so several processes on one database fire it once; the claimed run then queues the action's jobs (at the /// schedule's priority, with its id) unless the schedule's previous job is still pending, and records the outcome. A crash /// between the claim and the queueing loses that one run (it is logged as claimed first). A next run more than two periods /// ahead means the clock went back: it is computed again. Edits go through this class so the next run follows them. /// public sealed class Scheduler : IAsyncDisposable { /// Longest schedule name. public const int MaxNameLength = 100; /// The only target so far: the host's archive. public const string CurrentTarget = "current"; private readonly ChronosDatabase _db; private readonly JobEngine _engine; private readonly Channel _wake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite }); private readonly CancellationTokenSource _shutdown = new(); private readonly Lock _gate = new(); private IScheduleHost? _host; private Task _loop = Task.CompletedTask; public Scheduler(ChronosDatabase db, JobEngine engine, SchedulerOptions? options = null) { _db = db ?? throw new ArgumentNullException(nameof(db)); _engine = engine ?? throw new ArgumentNullException(nameof(engine)); Options = options ?? new SchedulerOptions(); } public SchedulerOptions Options { get; } /// The host given to ; null before. public IScheduleHost? Host => Volatile.Read(ref _host); /// A schedule was created, edited, reset or ran (also by the loop). Raised on the caller's or the loop's thread. public event Action? ScheduleChanged; /// A schedule was deleted. public event Action? ScheduleDeleted; /// /// Seeds 's system schedules, settles the runs missed while the host was off (see the type's /// remarks) and starts the loop. The first pass runs before this returns. /// public async Task StartAsync(IScheduleHost host, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(host); if (Host is not null) throw new InvalidOperationException("The scheduler has started already."); var added = await _db.Schedules.EnsureSystemAsync(host.SystemSchedules, Now(), cancellationToken).ConfigureAwait(false); if (added > 0) Log.Info("schedules", $"Added {added} system schedule(s)"); if (Interlocked.CompareExchange(ref _host, host, null) is not null) throw new InvalidOperationException("The scheduler has started already."); try { await RunDueAsync(cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { Log.Warning("schedules", "The schedules could not be run at start; the loop tries again", ex); } var shutdown = _shutdown.Token; lock (_gate) _loop = Task.Run(() => LoopAsync(shutdown), CancellationToken.None); } /// Makes the loop look for due schedules now (another process changed them, the archive came back). public void Wake() => _wake.Writer.TryWrite(true); /// /// One pass of the loop: corrects next runs (missing, clock gone back, missed without catch-up), then claims and fires /// every due schedule. Returns how many fired here. /// public async Task RunDueAsync(CancellationToken cancellationToken = default) { var host = RequireHost(); var now = Now(); foreach (var schedule in await _db.Schedules.ListAsync(cancellationToken).ConfigureAwait(false)) { if (schedule.Enabled) await CorrectAsync(host, schedule, now, cancellationToken).ConfigureAwait(false); } var fired = 0; foreach (var schedule in await _db.Schedules.ListDueAsync(now, cancellationToken).ConfigureAwait(false)) { DateTimeOffset? next = ScheduleRecurrence.TryParse(schedule.Recurrence, out var recurrence) ? recurrence.Next(now, host.TimeZone, schedule.NextRun) : null; // The claim: one process wins the run; the others see the moved next run and leave it. if (!await _db.Schedules.TryAdvanceAsync(schedule.Id, schedule.NextRun, next, cancellationToken).ConfigureAwait(false)) continue; if (next is null) { Log.Warning("schedules", $"Schedule {schedule.Id} has an invalid recurrence '{schedule.Recurrence}'; it stops until edited"); await RecordAsync(schedule.Id, null, ScheduleOutcome.Failed("recurrence"), cancellationToken).ConfigureAwait(false); continue; } Log.Debug("schedules", $"{schedule.Name} ({schedule.Action}) due {schedule.NextRun:O} claimed; next {next:O}"); await FireAsync(host, schedule, schedule.CreatedBy, manual: false, cancellationToken).ConfigureAwait(false); fired++; } return fired; } /// /// Runs the schedule now, as its next run would (also a disabled one), without moving its next run; null when it does /// not exist. is stamped on the jobs it queues. /// public async Task RunNowAsync(Guid id, Guid? userId, CancellationToken cancellationToken = default) { var host = RequireHost(); if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } schedule) return null; var fire = await FireAsync(host, schedule, userId ?? schedule.CreatedBy, manual: true, cancellationToken).ConfigureAwait(false); var dto = await GetAsync(id, cancellationToken).ConfigureAwait(false) ?? ScheduleMapping.ToDto(schedule); return new ScheduleRunDto(dto, fire.Outcome, fire.JobId); } /// /// Runs the system schedule now (, which skips while its work is /// pending) when it is enabled and has not run since ; null when it did not run. The hosts' archive /// scan once at start, as their polling did, and on the desktop when its archive is opened again (owner review 2, R10). /// public async Task RunSystemNowAsync(string systemKey, DateTimeOffset? since = null, CancellationToken cancellationToken = default) { _ = RequireHost(); if (await _db.Schedules.GetBySystemKeyAsync(systemKey, cancellationToken).ConfigureAwait(false) is not { Enabled: true } schedule || schedule.LastRun >= since) return null; return await RunNowAsync(schedule.Id, null, cancellationToken).ConfigureAwait(false); } /// The schedules (system ones first), the actions with their availability here, the host's time zone and clean-up steps. public async Task ListAsync(CancellationToken cancellationToken = default) { var host = RequireHost(); var items = await _db.Schedules.ListAsync(cancellationToken).ConfigureAwait(false); var actions = await ScheduleActions.AvailabilityAsync(Context(host), cancellationToken).ConfigureAwait(false); return new SchedulesDto([.. items.Select(ScheduleMapping.ToDto)], [.. actions.Select(ScheduleMapping.ToDto)], host.TimeZone.Id, host.CleanupSteps); } public async Task GetAsync(Guid id, CancellationToken cancellationToken = default) => await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } schedule ? ScheduleMapping.ToDto(schedule) : null; /// Creates a schedule (enabled ones get their next run at once); the result names the first invalid field instead when there is one. public async Task CreateAsync(CreateScheduleRequest request, Guid? createdBy, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(request); var host = RequireHost(); if (Problem(host, request.Name, request.Action, request.Target, request.Recurrence, request.Parameters, request.Priority) is { } problem) return new ScheduleEditResult(null, problem); var recurrence = ScheduleMapping.FromDto(request.Recurrence)!; var now = Now(); var schedule = new Schedule { Id = Guid.CreateVersion7(), Name = request.Name.Trim(), Action = request.Action, Target = CurrentTarget, ParametersJson = ScheduleMapping.ParametersJson(request.Action, request.Parameters), Recurrence = recurrence.Format(), CatchUp = request.CatchUp, Priority = Priority(request.Priority, JobPriority.Background), Enabled = request.Enabled, NextRun = request.Enabled ? recurrence.Next(now, host.TimeZone) : null, CreatedBy = createdBy, Created = now, Updated = now, }; await _db.Schedules.InsertAsync(schedule, cancellationToken).ConfigureAwait(false); Log.Info("schedules", $"Schedule {schedule.Name} ({schedule.Action}, {schedule.Recurrence}) created"); return new ScheduleEditResult(Changed(schedule), null); } /// /// Changes the fields gives. The next run is computed again when the recurrence changes or /// the schedule is enabled; a disabled schedule has none. A system schedule keeps its name and action (asking to change /// them is a problem of field name or action). /// public async Task UpdateAsync(Guid id, UpdateScheduleRequest request, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(request); var host = RequireHost(); if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } current) return new ScheduleEditResult(null, null); if (current.IsSystem && request.Name is { } name && name.Trim() != current.Name) return new ScheduleEditResult(null, "name"); if (current.IsSystem && request.Action is { } changed && changed != current.Action) return new ScheduleEditResult(null, "action"); var action = request.Action ?? current.Action; // Another action starts from its own defaults; the same one keeps the stored values the request leaves out. var parameters = request.Parameters ?? (action == current.Action ? ScheduleMapping.Parameters(current) : null); var recurrenceDto = request.Recurrence ?? (ScheduleRecurrence.TryParse(current.Recurrence, out var stored) ? ScheduleMapping.ToDto(stored) : null); if (Problem(host, request.Name ?? current.Name, action, null, recurrenceDto, parameters, request.Priority) is { } problem) return new ScheduleEditResult(null, problem); var recurrence = ScheduleMapping.FromDto(recurrenceDto)!; var now = Now(); var enabled = request.Enabled ?? current.Enabled; var reschedule = !enabled || !current.Enabled || current.NextRun is null || recurrence.Format() != current.Recurrence; var schedule = current with { Name = (request.Name ?? current.Name).Trim(), Action = action, ParametersJson = ScheduleMapping.ParametersJson(action, parameters), Recurrence = recurrence.Format(), CatchUp = request.CatchUp ?? current.CatchUp, Priority = Priority(request.Priority, current.Priority), Enabled = enabled, NextRun = !enabled ? null : reschedule ? recurrence.Next(now, host.TimeZone) : current.NextRun, Updated = now, }; if (!await _db.Schedules.UpdateAsync(schedule, reschedule, cancellationToken).ConfigureAwait(false)) return new ScheduleEditResult(null, null); Log.Info("schedules", $"Schedule {schedule.Name} ({schedule.Action}, {schedule.Recurrence}, {(enabled ? "enabled" : "disabled")}) changed"); var saved = await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) ?? schedule; return new ScheduleEditResult(Changed(saved), null); } /// Deletes a schedule a user created; a system schedule is refused (). public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) { if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } schedule) return ScheduleDeleteOutcome.NotFound; if (schedule.IsSystem) return ScheduleDeleteOutcome.System; if (!await _db.Schedules.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) return ScheduleDeleteOutcome.NotFound; Log.Info("schedules", $"Schedule {schedule.Name} ({schedule.Action}) deleted"); ScheduleDeleted?.Invoke(id); return ScheduleDeleteOutcome.Deleted; } /// /// Puts a system schedule back to the host's defaults () with a new next /// run. A schedule a user created, or one the host no longer defines, is a problem of field systemKey. /// public async Task ResetAsync(Guid id, CancellationToken cancellationToken = default) { var host = RequireHost(); if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } current) return new ScheduleEditResult(null, null); if (current.SystemKey is not { } key || host.SystemSchedules.FirstOrDefault(s => s.SystemKey == key) is not { } defaults) return new ScheduleEditResult(null, "systemKey"); var now = Now(); await _db.Schedules.ResetSystemAsync(key, defaults, now, cancellationToken).ConfigureAwait(false); if (defaults.Enabled) await _db.Schedules.TryAdvanceAsync(id, null, defaults.Recurrence.Next(now, host.TimeZone), cancellationToken).ConfigureAwait(false); Log.Info("schedules", $"System schedule {key} reset"); var saved = await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false); return saved is null ? new ScheduleEditResult(null, null) : new ScheduleEditResult(Changed(saved), null); } public async ValueTask DisposeAsync() { Task loop; lock (_gate) loop = _loop; await _shutdown.CancelAsync().ConfigureAwait(false); try { await loop.ConfigureAwait(false); } catch (OperationCanceledException) { } _shutdown.Dispose(); } /// /// Fills in a missing next run, computes it again after the clock went back, and moves a schedule without catch-up past /// a run it missed. Compare-and-set like a claim, so concurrent passes agree. /// private async Task CorrectAsync(IScheduleHost host, Schedule schedule, DateTimeOffset now, CancellationToken cancellationToken) { if (!ScheduleRecurrence.TryParse(schedule.Recurrence, out var recurrence)) return; DateTimeOffset next; string why; if (schedule.NextRun is not { } due) { next = recurrence.Next(now, host.TimeZone); why = "first run"; } else if (due - now > recurrence.Period * 2) { next = recurrence.Next(now, host.TimeZone); why = "the clock went back"; } else if (!schedule.CatchUp && now - due > Options.CatchUpGrace) { next = recurrence.Next(now, host.TimeZone, due); why = $"missed the run of {due:O} (no catch-up)"; } else { return; } if (!await _db.Schedules.TryAdvanceAsync(schedule.Id, schedule.NextRun, next, cancellationToken).ConfigureAwait(false)) return; Log.Info("schedules", $"{schedule.Name} ({schedule.Action}): next run {next:O} ({why})"); Changed(schedule with { NextRun = next }); } /// Queues the schedule's action unless its previous job is pending, records the outcome and logs one line. private async Task FireAsync(IScheduleHost host, Schedule schedule, Guid? createdBy, bool manual, CancellationToken cancellationToken) { string outcome; Guid? jobId = null; try { if (schedule.LastJobId is { } last && await _db.Jobs.GetAsync(last, cancellationToken).ConfigureAwait(false) is { State: JobState.Queued or JobState.Running or JobState.Paused }) { outcome = ScheduleOutcome.SkippedPending; } else { var resolution = await ScheduleActions.ResolveAsync(Context(host), schedule.Action, ScheduleMapping.Parameters(schedule), cancellationToken) .ConfigureAwait(false); outcome = resolution.Outcome; foreach (var request in resolution.Requests) { var job = await _engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, schedule.Priority, createdBy, schedule.Id, cancellationToken).ConfigureAwait(false); jobId ??= job.Id; } } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } catch (Exception ex) { Log.Warning("schedules", $"{schedule.Name} ({schedule.Action}) could not run", ex); outcome = ScheduleOutcome.Failed(ex.Message); } await RecordAsync(schedule.Id, jobId, outcome, CancellationToken.None).ConfigureAwait(false); Log.Info("schedules", $"{schedule.Name} ({schedule.Action}{(manual ? ", run now" : string.Empty)}): {outcome}{(jobId is { } id ? $", job {id}" : string.Empty)}"); return new ScheduleFire(schedule.Id, outcome, jobId); } private async Task RecordAsync(Guid id, Guid? jobId, string outcome, CancellationToken cancellationToken) { await _db.Schedules.RecordOutcomeAsync(id, jobId, outcome, Now(), cancellationToken).ConfigureAwait(false); if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } saved) Changed(saved); } private async Task LoopAsync(CancellationToken shutdown) { while (!shutdown.IsCancellationRequested) { await WaitAsync(shutdown).ConfigureAwait(false); if (shutdown.IsCancellationRequested) return; try { await RunDueAsync(shutdown).ConfigureAwait(false); } catch (OperationCanceledException) when (shutdown.IsCancellationRequested) { return; } catch (Exception ex) { // Nothing a pass throws ends the loop (a database briefly away, a host hook that failed). Log.Warning("schedules", "The schedules could not be run; the next tick tries again", ex); } } } /// Waits for , the tick or shutdown, whichever comes first. private async Task WaitAsync(CancellationToken shutdown) { using var tick = new CancellationTokenSource(Options.Tick, Options.Time); using var either = CancellationTokenSource.CreateLinkedTokenSource(shutdown, tick.Token); try { await _wake.Reader.ReadAsync(either.Token).ConfigureAwait(false); } catch (OperationCanceledException) { // Tick elapsed or shutdown. } } private ScheduleDto Changed(Schedule schedule) { var dto = ScheduleMapping.ToDto(schedule); ScheduleChanged?.Invoke(dto); return dto; } private ScheduleActionContext Context(IScheduleHost host) => new(_db, host, _engine.CanRun, Now()); private DateTimeOffset Now() => Options.Time.GetUtcNow(); private IScheduleHost RequireHost() => Host ?? throw new InvalidOperationException("Start the scheduler with its host first."); private static JobPriority Priority(string? wire, JobPriority fallback) => wire is not null && JobWire.TryParsePriority(wire, out var priority) ? priority : fallback; /// The first invalid field of a schedule as given, by its API name; null when it is valid. private static string? Problem(IScheduleHost host, string? name, string? action, string? target, RecurrenceDto? recurrence, ScheduleParametersDto? parameters, string? priority) { if (string.IsNullOrWhiteSpace(name) || name.Trim().Length > MaxNameLength) return "name"; if (action is null) return "action"; if (ScheduleActions.Problem(action, parameters, host.CleanupSteps) is { } invalid) return invalid; if (target is not null && target != CurrentTarget) return "target"; if (ScheduleMapping.FromDto(recurrence) is null) return "recurrence"; if (priority is not null && !JobWire.TryParsePriority(priority, out _)) return "priority"; return null; } }