JobEngine claims queued jobs atomically, persists progress at most every checkpoint interval, stops a handler when its job is cancelled or paused elsewhere and requeues jobs a crash left running. Also installs the SQLite import resolver under a lock so a concurrent first caller cannot reach a P/Invoke before the resolver is in place. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
247 lines
10 KiB
C#
247 lines
10 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Threading.Channels;
|
|
using Titano.Core.Diagnostics;
|
|
using Titano.Core.Model;
|
|
|
|
namespace Titano.Data.Jobs;
|
|
|
|
public sealed record JobEngineOptions
|
|
{
|
|
/// <summary>Concurrent workers (BRIEF §14.6: default cores / 2).</summary>
|
|
public int Workers { get; init; } = Math.Max(1, Environment.ProcessorCount / 2);
|
|
|
|
/// <summary>Identity stamped on claimed jobs (node id); also used for crash recovery.</summary>
|
|
public string WorkerId { get; init; } = Environment.MachineName;
|
|
|
|
/// <summary>Kinds this engine runs; null = every kind with a registered handler.</summary>
|
|
public IReadOnlyCollection<JobKind>? Kinds { get; init; }
|
|
|
|
public TimeSpan CheckpointInterval { get; init; } = TimeSpan.FromSeconds(5);
|
|
|
|
/// <summary>Fallback poll for jobs enqueued by other processes.</summary>
|
|
public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(3);
|
|
|
|
/// <summary>Finished jobs older than this are purged (BRIEF §14.6: 90 days).</summary>
|
|
public TimeSpan HistoryRetention { get; init; } = TimeSpan.FromDays(90);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Persistent job queue runner (BRIEF §14.6): N workers claim queued jobs from the database by priority, run the
|
|
/// registered handler, checkpoint progress, and survive crashes (running jobs are requeued at start with their
|
|
/// checkpoints). Used by the desktop client (local mode) and the server.
|
|
/// </summary>
|
|
public sealed class JobEngine : IAsyncDisposable
|
|
{
|
|
private readonly TitanoDatabase _db;
|
|
private readonly Dictionary<JobKind, IJobHandler> _handlers;
|
|
private readonly ConcurrentDictionary<Guid, CancellationTokenSource> _running = new();
|
|
private readonly Channel<bool> _wake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
|
|
private readonly CancellationTokenSource _shutdown = new();
|
|
private readonly List<Task> _workers = [];
|
|
|
|
public JobEngine(TitanoDatabase db, IEnumerable<IJobHandler> handlers, JobEngineOptions? options = null)
|
|
{
|
|
_db = db;
|
|
Options = options ?? new JobEngineOptions();
|
|
_handlers = handlers.ToDictionary(h => h.Kind);
|
|
}
|
|
|
|
public JobEngineOptions Options { get; }
|
|
|
|
/// <summary>A job changed state (queued, running, done, failed, cancelled, paused).</summary>
|
|
public event Action<Job>? JobChanged;
|
|
|
|
/// <summary>Progress of a running job (high frequency; throttle in the UI).</summary>
|
|
public event Action<JobProgressInfo>? Progress;
|
|
|
|
public IReadOnlyCollection<Guid> RunningJobIds => [.. _running.Keys];
|
|
|
|
/// <summary>Requeues this worker's interrupted jobs, purges old history and starts the workers.</summary>
|
|
public async Task StartAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
var requeued = await _db.Jobs.RequeueRunningAsync(Options.WorkerId, cancellationToken).ConfigureAwait(false);
|
|
if (requeued > 0) Log.Info("jobs", $"Requeued {requeued} interrupted job(s)");
|
|
await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false);
|
|
for (var i = 0; i < Options.Workers; i++)
|
|
{
|
|
var id = Options.Workers == 1 ? Options.WorkerId : $"{Options.WorkerId}#{i}";
|
|
_workers.Add(Task.Run(() => WorkerLoopAsync(id, _shutdown.Token), CancellationToken.None));
|
|
}
|
|
}
|
|
|
|
/// <summary>Queues a job and wakes a worker.</summary>
|
|
public async Task<Job> EnqueueAsync(JobKind kind, IReadOnlyList<Guid> targets, string? targetLabel = null, string? parametersJson = null,
|
|
JobPriority priority = JobPriority.Normal, Guid? createdBy = null, CancellationToken cancellationToken = default)
|
|
{
|
|
var job = new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = kind,
|
|
TargetIds = targets,
|
|
TargetLabel = targetLabel,
|
|
ParametersJson = parametersJson,
|
|
Priority = priority,
|
|
State = JobState.Queued,
|
|
CreatedBy = createdBy,
|
|
Created = DateTimeOffset.UtcNow,
|
|
};
|
|
await _db.Jobs.EnqueueAsync(job, cancellationToken).ConfigureAwait(false);
|
|
JobChanged?.Invoke(job);
|
|
Wake();
|
|
return job;
|
|
}
|
|
|
|
/// <summary>Cancels a queued or running job; a running handler observes its token.</summary>
|
|
public Task CancelAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Cancelled, cancellationToken);
|
|
|
|
/// <summary>Pauses a job: a running one stops at its next check and keeps its checkpoint.</summary>
|
|
public Task PauseAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Paused, cancellationToken);
|
|
|
|
/// <summary>Resumes a paused job or retries a failed/cancelled one from its last checkpoint.</summary>
|
|
public async Task ResumeAsync(Guid id, CancellationToken cancellationToken = default)
|
|
{
|
|
await _db.Jobs.SetStateAsync(id, JobState.Queued, null, cancellationToken).ConfigureAwait(false);
|
|
await NotifyAsync(id, cancellationToken).ConfigureAwait(false);
|
|
Wake();
|
|
}
|
|
|
|
/// <summary>Signals workers that new work may be available (also call on PostgreSQL NOTIFY).</summary>
|
|
public void Wake() => _wake.Writer.TryWrite(true);
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
await _shutdown.CancelAsync().ConfigureAwait(false);
|
|
try
|
|
{
|
|
await Task.WhenAll(_workers).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
}
|
|
_shutdown.Dispose();
|
|
}
|
|
|
|
internal void PublishProgress(JobContext context)
|
|
{
|
|
var (rate, eta) = context.Rate();
|
|
Progress?.Invoke(new JobProgressInfo(context.Job.Id, context.Job.Kind, context.Job.TargetLabel, context.Done, context.Total, rate, eta, context.Message));
|
|
}
|
|
|
|
internal async Task<bool> PersistAsync(JobContext context)
|
|
{
|
|
var (rate, eta) = context.Rate();
|
|
try
|
|
{
|
|
return await _db.Jobs.UpdateProgressAsync(context.Job.Id, context.Done, context.Total, rate, eta, context.Checkpoint, context.LogTail, CancellationToken.None)
|
|
.ConfigureAwait(false);
|
|
}
|
|
catch (DatabaseException ex) when (ex.IsTransient)
|
|
{
|
|
Log.Warning("jobs", "Progress not saved (transient database error)", ex);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
internal void StopJob(Guid id)
|
|
{
|
|
if (_running.TryGetValue(id, out var cts)) cts.Cancel();
|
|
}
|
|
|
|
private async Task TransitionAsync(Guid id, JobState state, CancellationToken cancellationToken)
|
|
{
|
|
await _db.Jobs.SetStateAsync(id, state, null, cancellationToken).ConfigureAwait(false);
|
|
StopJob(id);
|
|
await NotifyAsync(id, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task NotifyAsync(Guid id, CancellationToken cancellationToken)
|
|
{
|
|
if (await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } job) JobChanged?.Invoke(job);
|
|
}
|
|
|
|
private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown)
|
|
{
|
|
var kinds = Options.Kinds ?? [.. _handlers.Keys];
|
|
while (!shutdown.IsCancellationRequested)
|
|
{
|
|
Job? job;
|
|
try
|
|
{
|
|
job = await _db.Jobs.ClaimNextAsync(workerId, kinds, shutdown).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
return;
|
|
}
|
|
catch (DatabaseException ex)
|
|
{
|
|
Log.Warning("jobs", "Claiming a job failed", ex);
|
|
job = null;
|
|
}
|
|
|
|
if (job is null)
|
|
{
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
|
|
timeout.CancelAfter(Options.PollInterval);
|
|
try
|
|
{
|
|
await _wake.Reader.ReadAsync(timeout.Token).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
// Poll interval elapsed or shutdown.
|
|
}
|
|
continue;
|
|
}
|
|
|
|
await RunAsync(job, shutdown).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
private async Task RunAsync(Job job, CancellationToken shutdown)
|
|
{
|
|
using var cts = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
|
|
_running[job.Id] = cts;
|
|
JobChanged?.Invoke(job);
|
|
var context = new JobContext(this, job, cts.Token);
|
|
JobState final;
|
|
string? error = null;
|
|
try
|
|
{
|
|
if (!_handlers.TryGetValue(job.Kind, out var handler))
|
|
throw new InvalidOperationException($"No handler for {job.Kind} jobs on this worker.");
|
|
await handler.RunAsync(context).ConfigureAwait(false);
|
|
await PersistAsync(context).ConfigureAwait(false);
|
|
final = JobState.Done;
|
|
}
|
|
catch (OperationCanceledException) when (cts.IsCancellationRequested)
|
|
{
|
|
await PersistAsync(context).ConfigureAwait(false);
|
|
if (shutdown.IsCancellationRequested)
|
|
{
|
|
// Engine stopping: leave the job running so the next start requeues it from its checkpoint.
|
|
_running.TryRemove(job.Id, out _);
|
|
return;
|
|
}
|
|
// Cancel or pause already wrote the state; keep it.
|
|
_running.TryRemove(job.Id, out _);
|
|
await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false);
|
|
return;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Error("jobs", $"{job.Kind} job {job.Id} failed", ex);
|
|
context.Log(ex.Message);
|
|
await PersistAsync(context).ConfigureAwait(false);
|
|
final = JobState.Failed;
|
|
error = ex.Message;
|
|
}
|
|
finally
|
|
{
|
|
_running.TryRemove(job.Id, out _);
|
|
}
|
|
await _db.Jobs.SetStateAsync(job.Id, final, error, CancellationToken.None).ConfigureAwait(false);
|
|
await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
}
|