feat(data): persistent job engine with checkpoints, pause/resume and crash recovery

JobEngine claims queued jobs atomically, persists progress at most every
checkpoint interval, stops a handler when its job is cancelled or paused
elsewhere and requeues jobs a crash left running.

Also installs the SQLite import resolver under a lock so a concurrent
first caller cannot reach a P/Invoke before the resolver is in place.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 10:40:01 +00:00
1 parent e6938d6cf3
commit 11a38f745a
4 files changed
+521 -2

No files matched your search

+246
View File
@@ -0,0 +1,246 @@
using System.Collections.Concurrent;
using System.Threading.Channels;
using Titano.Core.Diagnostics;
using Titano.Core.Model;
namespace Titano.Data.Jobs;
public sealed record JobEngineOptions
{
/// <summary>Concurrent workers (BRIEF §14.6: default cores / 2).</summary>
public int Workers { get; init; } = Math.Max(1, Environment.ProcessorCount / 2);
/// <summary>Identity stamped on claimed jobs (node id); also used for crash recovery.</summary>
public string WorkerId { get; init; } = Environment.MachineName;
/// <summary>Kinds this engine runs; null = every kind with a registered handler.</summary>
public IReadOnlyCollection<JobKind>? Kinds { get; init; }
public TimeSpan CheckpointInterval { get; init; } = TimeSpan.FromSeconds(5);
/// <summary>Fallback poll for jobs enqueued by other processes.</summary>
public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(3);
/// <summary>Finished jobs older than this are purged (BRIEF §14.6: 90 days).</summary>
public TimeSpan HistoryRetention { get; init; } = TimeSpan.FromDays(90);
}
/// <summary>
/// Persistent job queue runner (BRIEF §14.6): N workers claim queued jobs from the database by priority, run the
/// registered handler, checkpoint progress, and survive crashes (running jobs are requeued at start with their
/// checkpoints). Used by the desktop client (local mode) and the server.
/// </summary>
public sealed class JobEngine : IAsyncDisposable
{
private readonly TitanoDatabase _db;
private readonly Dictionary<JobKind, IJobHandler> _handlers;
private readonly ConcurrentDictionary<Guid, CancellationTokenSource> _running = new();
private readonly Channel<bool> _wake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
private readonly CancellationTokenSource _shutdown = new();
private readonly List<Task> _workers = [];
public JobEngine(TitanoDatabase db, IEnumerable<IJobHandler> handlers, JobEngineOptions? options = null)
{
_db = db;
Options = options ?? new JobEngineOptions();
_handlers = handlers.ToDictionary(h => h.Kind);
}
public JobEngineOptions Options { get; }
/// <summary>A job changed state (queued, running, done, failed, cancelled, paused).</summary>
public event Action<Job>? JobChanged;
/// <summary>Progress of a running job (high frequency; throttle in the UI).</summary>
public event Action<JobProgressInfo>? Progress;
public IReadOnlyCollection<Guid> RunningJobIds => [.. _running.Keys];
/// <summary>Requeues this worker's interrupted jobs, purges old history and starts the workers.</summary>
public async Task StartAsync(CancellationToken cancellationToken = default)
{
var requeued = await _db.Jobs.RequeueRunningAsync(Options.WorkerId, cancellationToken).ConfigureAwait(false);
if (requeued > 0) Log.Info("jobs", $"Requeued {requeued} interrupted job(s)");
await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false);
for (var i = 0; i < Options.Workers; i++)
{
var id = Options.Workers == 1 ? Options.WorkerId : $"{Options.WorkerId}#{i}";
_workers.Add(Task.Run(() => WorkerLoopAsync(id, _shutdown.Token), CancellationToken.None));
}
}
/// <summary>Queues a job and wakes a worker.</summary>
public async Task<Job> EnqueueAsync(JobKind kind, IReadOnlyList<Guid> targets, string? targetLabel = null, string? parametersJson = null,
JobPriority priority = JobPriority.Normal, Guid? createdBy = null, CancellationToken cancellationToken = default)
{
var job = new Job
{
Id = Guid.CreateVersion7(),
Kind = kind,
TargetIds = targets,
TargetLabel = targetLabel,
ParametersJson = parametersJson,
Priority = priority,
State = JobState.Queued,
CreatedBy = createdBy,
Created = DateTimeOffset.UtcNow,
};
await _db.Jobs.EnqueueAsync(job, cancellationToken).ConfigureAwait(false);
JobChanged?.Invoke(job);
Wake();
return job;
}
/// <summary>Cancels a queued or running job; a running handler observes its token.</summary>
public Task CancelAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Cancelled, cancellationToken);
/// <summary>Pauses a job: a running one stops at its next check and keeps its checkpoint.</summary>
public Task PauseAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Paused, cancellationToken);
/// <summary>Resumes a paused job or retries a failed/cancelled one from its last checkpoint.</summary>
public async Task ResumeAsync(Guid id, CancellationToken cancellationToken = default)
{
await _db.Jobs.SetStateAsync(id, JobState.Queued, null, cancellationToken).ConfigureAwait(false);
await NotifyAsync(id, cancellationToken).ConfigureAwait(false);
Wake();
}
/// <summary>Signals workers that new work may be available (also call on PostgreSQL NOTIFY).</summary>
public void Wake() => _wake.Writer.TryWrite(true);
public async ValueTask DisposeAsync()
{
await _shutdown.CancelAsync().ConfigureAwait(false);
try
{
await Task.WhenAll(_workers).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
_shutdown.Dispose();
}
internal void PublishProgress(JobContext context)
{
var (rate, eta) = context.Rate();
Progress?.Invoke(new JobProgressInfo(context.Job.Id, context.Job.Kind, context.Job.TargetLabel, context.Done, context.Total, rate, eta, context.Message));
}
internal async Task<bool> PersistAsync(JobContext context)
{
var (rate, eta) = context.Rate();
try
{
return await _db.Jobs.UpdateProgressAsync(context.Job.Id, context.Done, context.Total, rate, eta, context.Checkpoint, context.LogTail, CancellationToken.None)
.ConfigureAwait(false);
}
catch (DatabaseException ex) when (ex.IsTransient)
{
Log.Warning("jobs", "Progress not saved (transient database error)", ex);
return true;
}
}
internal void StopJob(Guid id)
{
if (_running.TryGetValue(id, out var cts)) cts.Cancel();
}
private async Task TransitionAsync(Guid id, JobState state, CancellationToken cancellationToken)
{
await _db.Jobs.SetStateAsync(id, state, null, cancellationToken).ConfigureAwait(false);
StopJob(id);
await NotifyAsync(id, cancellationToken).ConfigureAwait(false);
}
private async Task NotifyAsync(Guid id, CancellationToken cancellationToken)
{
if (await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } job) JobChanged?.Invoke(job);
}
private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown)
{
var kinds = Options.Kinds ?? [.. _handlers.Keys];
while (!shutdown.IsCancellationRequested)
{
Job? job;
try
{
job = await _db.Jobs.ClaimNextAsync(workerId, kinds, shutdown).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
catch (DatabaseException ex)
{
Log.Warning("jobs", "Claiming a job failed", ex);
job = null;
}
if (job is null)
{
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
timeout.CancelAfter(Options.PollInterval);
try
{
await _wake.Reader.ReadAsync(timeout.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
// Poll interval elapsed or shutdown.
}
continue;
}
await RunAsync(job, shutdown).ConfigureAwait(false);
}
}
private async Task RunAsync(Job job, CancellationToken shutdown)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
_running[job.Id] = cts;
JobChanged?.Invoke(job);
var context = new JobContext(this, job, cts.Token);
JobState final;
string? error = null;
try
{
if (!_handlers.TryGetValue(job.Kind, out var handler))
throw new InvalidOperationException($"No handler for {job.Kind} jobs on this worker.");
await handler.RunAsync(context).ConfigureAwait(false);
await PersistAsync(context).ConfigureAwait(false);
final = JobState.Done;
}
catch (OperationCanceledException) when (cts.IsCancellationRequested)
{
await PersistAsync(context).ConfigureAwait(false);
if (shutdown.IsCancellationRequested)
{
// Engine stopping: leave the job running so the next start requeues it from its checkpoint.
_running.TryRemove(job.Id, out _);
return;
}
// Cancel or pause already wrote the state; keep it.
_running.TryRemove(job.Id, out _);
await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false);
return;
}
catch (Exception ex)
{
Log.Error("jobs", $"{job.Kind} job {job.Id} failed", ex);
context.Log(ex.Message);
await PersistAsync(context).ConfigureAwait(false);
final = JobState.Failed;
error = ex.Message;
}
finally
{
_running.TryRemove(job.Id, out _);
}
await _db.Jobs.SetStateAsync(job.Id, final, error, CancellationToken.None).ConfigureAwait(false);
await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false);
}
}
+131
View File
@@ -0,0 +1,131 @@
using System.Diagnostics;
using System.Text;
using Titano.Core.Model;
namespace Titano.Data.Jobs;
/// <summary>Runs one kind of job (scan, verify, render, …). Handlers must honour <see cref="JobContext.Cancellation"/>
/// and resume from <see cref="JobContext.Checkpoint"/> when it is set.</summary>
public interface IJobHandler
{
JobKind Kind { get; }
Task RunAsync(JobContext context);
}
/// <summary>Live progress of a running job (pushed to the job bar, the Jobs screen and WebSocket clients).</summary>
public sealed record JobProgressInfo(Guid JobId, JobKind Kind, string? TargetLabel, long Done, long Total, double? ItemsPerSecond, TimeSpan? Eta, string? Message);
/// <summary>Everything a handler needs while it runs. Progress is cheap to report; the engine persists it at most every
/// checkpoint interval (BRIEF §14.6: checkpoints every 5 s).</summary>
public sealed class JobContext
{
private const int LogTailLines = 40;
private readonly JobEngine _engine;
private readonly Stopwatch _clock = Stopwatch.StartNew();
private readonly Queue<string> _log = new();
private readonly Lock _gate = new();
private long _lastPersistTicks;
private long _rateDone;
private double _rateSeconds;
private double? _rate;
internal JobContext(JobEngine engine, Job job, CancellationToken cancellation)
{
_engine = engine;
Job = job;
Cancellation = cancellation;
Checkpoint = job.Checkpoint;
Done = job.ProgressDone;
Total = job.ProgressTotal;
_rateDone = Done;
if (job.LogTail is { Length: > 0 } tail)
{
foreach (var line in tail.Split('\n')) _log.Enqueue(line);
}
}
public Job Job { get; }
/// <summary>Signalled on cancel, pause and engine shutdown.</summary>
public CancellationToken Cancellation { get; }
/// <summary>Resume cursor saved by a previous attempt (null on the first run).</summary>
public string? Checkpoint { get; private set; }
public long Done { get; private set; }
public long Total { get; private set; }
public string? Message { get; private set; }
/// <summary>Reports progress; persisted and broadcast at most every checkpoint interval.</summary>
public ValueTask ReportAsync(long done, long total, string? message = null) => ReportCoreAsync(done, total, message, force: false);
/// <summary>Saves a resume cursor together with the current progress (persisted immediately).</summary>
public ValueTask CheckpointAsync(string checkpoint, long done, long total)
{
Checkpoint = checkpoint;
return ReportCoreAsync(done, total, Message, force: true);
}
/// <summary>Appends a line to the job log (the last lines are kept with the job).</summary>
public void Log(string line)
{
lock (_gate)
{
_log.Enqueue($"{DateTime.UtcNow:HH:mm:ss} {line}");
while (_log.Count > LogTailLines) _log.Dequeue();
}
}
internal string LogTail
{
get
{
lock (_gate)
{
var sb = new StringBuilder();
foreach (var l in _log)
{
if (sb.Length > 0) sb.Append('\n');
sb.Append(l);
}
return sb.ToString();
}
}
}
/// <summary>Throughput as a moving average over windows of at least two seconds, and the time left at that rate.</summary>
internal (double? Rate, TimeSpan? Eta) Rate()
{
lock (_gate)
{
var seconds = _clock.Elapsed.TotalSeconds;
var span = seconds - _rateSeconds;
if (span >= 2 && Done >= _rateDone)
{
var sample = (Done - _rateDone) / span;
_rate = _rate is { } previous ? previous * 0.7 + sample * 0.3 : sample;
_rateDone = Done;
_rateSeconds = seconds;
}
TimeSpan? eta = _rate is > 0 && Total > Done ? TimeSpan.FromSeconds((Total - Done) / _rate.Value) : null;
return (_rate, eta);
}
}
private async ValueTask ReportCoreAsync(long done, long total, string? message, bool force)
{
Done = done;
Total = total;
Message = message;
var now = _clock.ElapsedTicks;
var due = now - Interlocked.Read(ref _lastPersistTicks) >= _engine.Options.CheckpointInterval.TotalSeconds * Stopwatch.Frequency;
_engine.PublishProgress(this);
if (!force && !due) return;
Interlocked.Exchange(ref _lastPersistTicks, now);
if (!await _engine.PersistAsync(this).ConfigureAwait(false))
_engine.StopJob(Job.Id); // cancelled or paused by someone else
}
}
+9 -2
View File
@@ -11,7 +11,9 @@ namespace Titano.Data.Sqlite;
internal static unsafe class SqliteNative
{
private const string Library = "titano-sqlite3";
private static int _resolverInstalled;
// Set only after the resolver is in place: a concurrent first caller must not reach a P/Invoke before that.
private static readonly Lock ResolverGate = new();
private static bool _resolverInstalled;
public const int SQLITE_OK = 0;
public const int SQLITE_INTERRUPT = 9;
@@ -43,8 +45,13 @@ internal static unsafe class SqliteNative
/// <summary>Routes the logical library name to the platform's system SQLite (once per process).</summary>
public static void EnsureResolver()
{
if (Interlocked.Exchange(ref _resolverInstalled, 1) == 0)
if (Volatile.Read(ref _resolverInstalled)) return;
lock (ResolverGate)
{
if (_resolverInstalled) return;
NativeLibrary.SetDllImportResolver(typeof(SqliteNative).Assembly, Resolve);
Volatile.Write(ref _resolverInstalled, true);
}
}
private static nint Resolve(string name, Assembly assembly, DllImportSearchPath? searchPath)
@@ -0,0 +1,135 @@
using Titano.Core.Model;
using Titano.Data.Jobs;
using Titano.Testing;
namespace Titano.Data.Tests.Jobs;
[Category("jobs")]
public sealed class JobEngineTests(TestContext context)
{
private sealed class CountingHandler(JobKind kind, int steps, Func<int, CancellationToken, Task>? onStep = null) : IJobHandler
{
public JobKind Kind => kind;
public List<string?> SeenCheckpoints { get; } = [];
public int Completed;
public async Task RunAsync(JobContext c)
{
lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint);
var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0;
for (var i = start; i < steps; i++)
{
c.Cancellation.ThrowIfCancellationRequested();
if (onStep is not null) await onStep(i, c.Cancellation);
await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps);
}
Interlocked.Increment(ref Completed);
}
}
private async Task<TitanoDatabase> OpenAsync() =>
await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
private static async Task<Job> WaitForStateAsync(TitanoDatabase db, Guid id, JobState state, TimeSpan timeout)
{
var until = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < until)
{
var job = await db.Jobs.GetAsync(id);
if (job?.State == state) return job;
await Task.Delay(20);
}
Assert.Fail($"Job {id} did not reach {state}");
return null!;
}
[Test]
public async Task RunsQueuedJobsToCompletion()
{
await using var db = await OpenAsync();
var handler = new CountingHandler(JobKind.Verify, 5);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
var progress = 0;
engine.Progress += _ => Interlocked.Increment(ref progress);
await engine.StartAsync();
var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A");
var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B");
var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10));
await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal(2, handler.Completed);
Assert.Equal(5L, doneA.ProgressDone);
Assert.Greater(progress, 0);
}
[Test]
public async Task CancelStopsARunningJob()
{
await using var db = await OpenAsync();
var started = new TaskCompletionSource();
var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) =>
{
if (i == 0) started.TrySetResult();
await Task.Delay(10, ct);
});
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
await engine.StartAsync();
var job = await engine.EnqueueAsync(JobKind.Scan, []);
await started.Task.WaitAsync(TimeSpan.FromSeconds(10));
await engine.CancelAsync(job.Id);
await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5));
Assert.Equal(0, handler.Completed);
}
[Test]
public async Task PauseAndResumeContinueFromCheckpoint()
{
await using var db = await OpenAsync();
var gate = new SemaphoreSlim(0);
var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) =>
{
if (i == 10) await gate.WaitAsync(ct);
});
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
await engine.StartAsync();
var job = await engine.EnqueueAsync(JobKind.Proxies, []);
var until = DateTime.UtcNow.AddSeconds(10);
while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20);
await engine.PauseAsync(job.Id);
var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5));
Assert.Equal("10", paused.Checkpoint);
gate.Release(10);
await engine.ResumeAsync(job.Id);
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal("10", handler.SeenCheckpoints[^1]);
}
[Test]
public async Task CrashedJobsResumeAfterRestart()
{
await using var db = await OpenAsync();
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
await db.Jobs.EnqueueAsync(job);
var claimed = await db.Jobs.ClaimNextAsync("t");
Assert.NotNull(claimed);
await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null);
// Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it.
var handler = new CountingHandler(JobKind.Thumbs, 12);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
await engine.StartAsync();
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal("7", handler.SeenCheckpoints.Single());
}
[Test]
public async Task HandlerFailureMarksJobFailedWithMessage()
{
await using var db = await OpenAsync();
var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
await engine.StartAsync();
var job = await engine.EnqueueAsync(JobKind.Render, []);
var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10));
Assert.Equal("encoder lost", failed.Error);
Assert.Contains("encoder lost", failed.LogTail);
}
}