Files
Chronos/src/Titano.Data/Jobs/JobEngine.cs
T
Claude 11a38f745a feat(data): persistent job engine with checkpoints, pause/resume and crash recovery
JobEngine claims queued jobs atomically, persists progress at most every
checkpoint interval, stops a handler when its job is cancelled or paused
elsewhere and requeues jobs a crash left running.

Also installs the SQLite import resolver under a lock so a concurrent
first caller cannot reach a P/Invoke before the resolver is in place.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-02 10:40:01 +00:00

247 lines
10 KiB
C#

using System.Collections.Concurrent;
using System.Threading.Channels;
using Titano.Core.Diagnostics;
using Titano.Core.Model;
namespace Titano.Data.Jobs;
public sealed record JobEngineOptions
{
/// <summary>Concurrent workers (BRIEF §14.6: default cores / 2).</summary>
public int Workers { get; init; } = Math.Max(1, Environment.ProcessorCount / 2);
/// <summary>Identity stamped on claimed jobs (node id); also used for crash recovery.</summary>
public string WorkerId { get; init; } = Environment.MachineName;
/// <summary>Kinds this engine runs; null = every kind with a registered handler.</summary>
public IReadOnlyCollection<JobKind>? Kinds { get; init; }
public TimeSpan CheckpointInterval { get; init; } = TimeSpan.FromSeconds(5);
/// <summary>Fallback poll for jobs enqueued by other processes.</summary>
public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(3);
/// <summary>Finished jobs older than this are purged (BRIEF §14.6: 90 days).</summary>
public TimeSpan HistoryRetention { get; init; } = TimeSpan.FromDays(90);
}
/// <summary>
/// Persistent job queue runner (BRIEF §14.6): N workers claim queued jobs from the database by priority, run the
/// registered handler, checkpoint progress, and survive crashes (running jobs are requeued at start with their
/// checkpoints). Used by the desktop client (local mode) and the server.
/// </summary>
public sealed class JobEngine : IAsyncDisposable
{
private readonly TitanoDatabase _db;
private readonly Dictionary<JobKind, IJobHandler> _handlers;
private readonly ConcurrentDictionary<Guid, CancellationTokenSource> _running = new();
private readonly Channel<bool> _wake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
private readonly CancellationTokenSource _shutdown = new();
private readonly List<Task> _workers = [];
public JobEngine(TitanoDatabase db, IEnumerable<IJobHandler> handlers, JobEngineOptions? options = null)
{
_db = db;
Options = options ?? new JobEngineOptions();
_handlers = handlers.ToDictionary(h => h.Kind);
}
public JobEngineOptions Options { get; }
/// <summary>A job changed state (queued, running, done, failed, cancelled, paused).</summary>
public event Action<Job>? JobChanged;
/// <summary>Progress of a running job (high frequency; throttle in the UI).</summary>
public event Action<JobProgressInfo>? Progress;
public IReadOnlyCollection<Guid> RunningJobIds => [.. _running.Keys];
/// <summary>Requeues this worker's interrupted jobs, purges old history and starts the workers.</summary>
public async Task StartAsync(CancellationToken cancellationToken = default)
{
var requeued = await _db.Jobs.RequeueRunningAsync(Options.WorkerId, cancellationToken).ConfigureAwait(false);
if (requeued > 0) Log.Info("jobs", $"Requeued {requeued} interrupted job(s)");
await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false);
for (var i = 0; i < Options.Workers; i++)
{
var id = Options.Workers == 1 ? Options.WorkerId : $"{Options.WorkerId}#{i}";
_workers.Add(Task.Run(() => WorkerLoopAsync(id, _shutdown.Token), CancellationToken.None));
}
}
/// <summary>Queues a job and wakes a worker.</summary>
public async Task<Job> EnqueueAsync(JobKind kind, IReadOnlyList<Guid> targets, string? targetLabel = null, string? parametersJson = null,
JobPriority priority = JobPriority.Normal, Guid? createdBy = null, CancellationToken cancellationToken = default)
{
var job = new Job
{
Id = Guid.CreateVersion7(),
Kind = kind,
TargetIds = targets,
TargetLabel = targetLabel,
ParametersJson = parametersJson,
Priority = priority,
State = JobState.Queued,
CreatedBy = createdBy,
Created = DateTimeOffset.UtcNow,
};
await _db.Jobs.EnqueueAsync(job, cancellationToken).ConfigureAwait(false);
JobChanged?.Invoke(job);
Wake();
return job;
}
/// <summary>Cancels a queued or running job; a running handler observes its token.</summary>
public Task CancelAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Cancelled, cancellationToken);
/// <summary>Pauses a job: a running one stops at its next check and keeps its checkpoint.</summary>
public Task PauseAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Paused, cancellationToken);
/// <summary>Resumes a paused job or retries a failed/cancelled one from its last checkpoint.</summary>
public async Task ResumeAsync(Guid id, CancellationToken cancellationToken = default)
{
await _db.Jobs.SetStateAsync(id, JobState.Queued, null, cancellationToken).ConfigureAwait(false);
await NotifyAsync(id, cancellationToken).ConfigureAwait(false);
Wake();
}
/// <summary>Signals workers that new work may be available (also call on PostgreSQL NOTIFY).</summary>
public void Wake() => _wake.Writer.TryWrite(true);
public async ValueTask DisposeAsync()
{
await _shutdown.CancelAsync().ConfigureAwait(false);
try
{
await Task.WhenAll(_workers).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
_shutdown.Dispose();
}
internal void PublishProgress(JobContext context)
{
var (rate, eta) = context.Rate();
Progress?.Invoke(new JobProgressInfo(context.Job.Id, context.Job.Kind, context.Job.TargetLabel, context.Done, context.Total, rate, eta, context.Message));
}
internal async Task<bool> PersistAsync(JobContext context)
{
var (rate, eta) = context.Rate();
try
{
return await _db.Jobs.UpdateProgressAsync(context.Job.Id, context.Done, context.Total, rate, eta, context.Checkpoint, context.LogTail, CancellationToken.None)
.ConfigureAwait(false);
}
catch (DatabaseException ex) when (ex.IsTransient)
{
Log.Warning("jobs", "Progress not saved (transient database error)", ex);
return true;
}
}
internal void StopJob(Guid id)
{
if (_running.TryGetValue(id, out var cts)) cts.Cancel();
}
private async Task TransitionAsync(Guid id, JobState state, CancellationToken cancellationToken)
{
await _db.Jobs.SetStateAsync(id, state, null, cancellationToken).ConfigureAwait(false);
StopJob(id);
await NotifyAsync(id, cancellationToken).ConfigureAwait(false);
}
private async Task NotifyAsync(Guid id, CancellationToken cancellationToken)
{
if (await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } job) JobChanged?.Invoke(job);
}
private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown)
{
var kinds = Options.Kinds ?? [.. _handlers.Keys];
while (!shutdown.IsCancellationRequested)
{
Job? job;
try
{
job = await _db.Jobs.ClaimNextAsync(workerId, kinds, shutdown).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
catch (DatabaseException ex)
{
Log.Warning("jobs", "Claiming a job failed", ex);
job = null;
}
if (job is null)
{
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
timeout.CancelAfter(Options.PollInterval);
try
{
await _wake.Reader.ReadAsync(timeout.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
// Poll interval elapsed or shutdown.
}
continue;
}
await RunAsync(job, shutdown).ConfigureAwait(false);
}
}
private async Task RunAsync(Job job, CancellationToken shutdown)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
_running[job.Id] = cts;
JobChanged?.Invoke(job);
var context = new JobContext(this, job, cts.Token);
JobState final;
string? error = null;
try
{
if (!_handlers.TryGetValue(job.Kind, out var handler))
throw new InvalidOperationException($"No handler for {job.Kind} jobs on this worker.");
await handler.RunAsync(context).ConfigureAwait(false);
await PersistAsync(context).ConfigureAwait(false);
final = JobState.Done;
}
catch (OperationCanceledException) when (cts.IsCancellationRequested)
{
await PersistAsync(context).ConfigureAwait(false);
if (shutdown.IsCancellationRequested)
{
// Engine stopping: leave the job running so the next start requeues it from its checkpoint.
_running.TryRemove(job.Id, out _);
return;
}
// Cancel or pause already wrote the state; keep it.
_running.TryRemove(job.Id, out _);
await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false);
return;
}
catch (Exception ex)
{
Log.Error("jobs", $"{job.Kind} job {job.Id} failed", ex);
context.Log(ex.Message);
await PersistAsync(context).ConfigureAwait(false);
final = JobState.Failed;
error = ex.Message;
}
finally
{
_running.TryRemove(job.Id, out _);
}
await _db.Jobs.SetStateAsync(job.Id, final, error, CancellationToken.None).ConfigureAwait(false);
await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false);
}
}