From 11a38f745a73d415cdbb923a7d24edfd911d59d4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 10:40:01 +0000 Subject: [PATCH] feat(data): persistent job engine with checkpoints, pause/resume and crash recovery JobEngine claims queued jobs atomically, persists progress at most every checkpoint interval, stops a handler when its job is cancelled or paused elsewhere and requeues jobs a crash left running. Also installs the SQLite import resolver under a lock so a concurrent first caller cannot reach a P/Invoke before the resolver is in place. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Jobs/JobEngine.cs | 246 ++++++++++++++++++ src/Titano.Data/Jobs/JobHandler.cs | 131 ++++++++++ src/Titano.Data/Sqlite/SqliteNative.cs | 11 +- .../Titano.Data.Tests/Jobs/JobEngineTests.cs | 135 ++++++++++ 4 files changed, 521 insertions(+), 2 deletions(-) create mode 100644 src/Titano.Data/Jobs/JobEngine.cs create mode 100644 src/Titano.Data/Jobs/JobHandler.cs create mode 100644 tests/Titano.Data.Tests/Jobs/JobEngineTests.cs diff --git a/src/Titano.Data/Jobs/JobEngine.cs b/src/Titano.Data/Jobs/JobEngine.cs new file mode 100644 index 00000000..b2b279f4 --- /dev/null +++ b/src/Titano.Data/Jobs/JobEngine.cs @@ -0,0 +1,246 @@ +using System.Collections.Concurrent; +using System.Threading.Channels; +using Titano.Core.Diagnostics; +using Titano.Core.Model; + +namespace Titano.Data.Jobs; + +public sealed record JobEngineOptions +{ + /// Concurrent workers (BRIEF §14.6: default cores / 2). + public int Workers { get; init; } = Math.Max(1, Environment.ProcessorCount / 2); + + /// Identity stamped on claimed jobs (node id); also used for crash recovery. + public string WorkerId { get; init; } = Environment.MachineName; + + /// Kinds this engine runs; null = every kind with a registered handler. + public IReadOnlyCollection? Kinds { get; init; } + + public TimeSpan CheckpointInterval { get; init; } = TimeSpan.FromSeconds(5); + + /// Fallback poll for jobs enqueued by other processes. + public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(3); + + /// Finished jobs older than this are purged (BRIEF §14.6: 90 days). + public TimeSpan HistoryRetention { get; init; } = TimeSpan.FromDays(90); +} + +/// +/// Persistent job queue runner (BRIEF §14.6): N workers claim queued jobs from the database by priority, run the +/// registered handler, checkpoint progress, and survive crashes (running jobs are requeued at start with their +/// checkpoints). Used by the desktop client (local mode) and the server. +/// +public sealed class JobEngine : IAsyncDisposable +{ + private readonly TitanoDatabase _db; + private readonly Dictionary _handlers; + private readonly ConcurrentDictionary _running = new(); + private readonly Channel _wake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite }); + private readonly CancellationTokenSource _shutdown = new(); + private readonly List _workers = []; + + public JobEngine(TitanoDatabase db, IEnumerable handlers, JobEngineOptions? options = null) + { + _db = db; + Options = options ?? new JobEngineOptions(); + _handlers = handlers.ToDictionary(h => h.Kind); + } + + public JobEngineOptions Options { get; } + + /// A job changed state (queued, running, done, failed, cancelled, paused). + public event Action? JobChanged; + + /// Progress of a running job (high frequency; throttle in the UI). + public event Action? Progress; + + public IReadOnlyCollection RunningJobIds => [.. _running.Keys]; + + /// Requeues this worker's interrupted jobs, purges old history and starts the workers. + public async Task StartAsync(CancellationToken cancellationToken = default) + { + var requeued = await _db.Jobs.RequeueRunningAsync(Options.WorkerId, cancellationToken).ConfigureAwait(false); + if (requeued > 0) Log.Info("jobs", $"Requeued {requeued} interrupted job(s)"); + await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false); + for (var i = 0; i < Options.Workers; i++) + { + var id = Options.Workers == 1 ? Options.WorkerId : $"{Options.WorkerId}#{i}"; + _workers.Add(Task.Run(() => WorkerLoopAsync(id, _shutdown.Token), CancellationToken.None)); + } + } + + /// Queues a job and wakes a worker. + public async Task EnqueueAsync(JobKind kind, IReadOnlyList targets, string? targetLabel = null, string? parametersJson = null, + JobPriority priority = JobPriority.Normal, Guid? createdBy = null, CancellationToken cancellationToken = default) + { + var job = new Job + { + Id = Guid.CreateVersion7(), + Kind = kind, + TargetIds = targets, + TargetLabel = targetLabel, + ParametersJson = parametersJson, + Priority = priority, + State = JobState.Queued, + CreatedBy = createdBy, + Created = DateTimeOffset.UtcNow, + }; + await _db.Jobs.EnqueueAsync(job, cancellationToken).ConfigureAwait(false); + JobChanged?.Invoke(job); + Wake(); + return job; + } + + /// Cancels a queued or running job; a running handler observes its token. + public Task CancelAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Cancelled, cancellationToken); + + /// Pauses a job: a running one stops at its next check and keeps its checkpoint. + public Task PauseAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Paused, cancellationToken); + + /// Resumes a paused job or retries a failed/cancelled one from its last checkpoint. + public async Task ResumeAsync(Guid id, CancellationToken cancellationToken = default) + { + await _db.Jobs.SetStateAsync(id, JobState.Queued, null, cancellationToken).ConfigureAwait(false); + await NotifyAsync(id, cancellationToken).ConfigureAwait(false); + Wake(); + } + + /// Signals workers that new work may be available (also call on PostgreSQL NOTIFY). + public void Wake() => _wake.Writer.TryWrite(true); + + public async ValueTask DisposeAsync() + { + await _shutdown.CancelAsync().ConfigureAwait(false); + try + { + await Task.WhenAll(_workers).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + _shutdown.Dispose(); + } + + internal void PublishProgress(JobContext context) + { + var (rate, eta) = context.Rate(); + Progress?.Invoke(new JobProgressInfo(context.Job.Id, context.Job.Kind, context.Job.TargetLabel, context.Done, context.Total, rate, eta, context.Message)); + } + + internal async Task PersistAsync(JobContext context) + { + var (rate, eta) = context.Rate(); + try + { + return await _db.Jobs.UpdateProgressAsync(context.Job.Id, context.Done, context.Total, rate, eta, context.Checkpoint, context.LogTail, CancellationToken.None) + .ConfigureAwait(false); + } + catch (DatabaseException ex) when (ex.IsTransient) + { + Log.Warning("jobs", "Progress not saved (transient database error)", ex); + return true; + } + } + + internal void StopJob(Guid id) + { + if (_running.TryGetValue(id, out var cts)) cts.Cancel(); + } + + private async Task TransitionAsync(Guid id, JobState state, CancellationToken cancellationToken) + { + await _db.Jobs.SetStateAsync(id, state, null, cancellationToken).ConfigureAwait(false); + StopJob(id); + await NotifyAsync(id, cancellationToken).ConfigureAwait(false); + } + + private async Task NotifyAsync(Guid id, CancellationToken cancellationToken) + { + if (await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } job) JobChanged?.Invoke(job); + } + + private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown) + { + var kinds = Options.Kinds ?? [.. _handlers.Keys]; + while (!shutdown.IsCancellationRequested) + { + Job? job; + try + { + job = await _db.Jobs.ClaimNextAsync(workerId, kinds, shutdown).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + catch (DatabaseException ex) + { + Log.Warning("jobs", "Claiming a job failed", ex); + job = null; + } + + if (job is null) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(shutdown); + timeout.CancelAfter(Options.PollInterval); + try + { + await _wake.Reader.ReadAsync(timeout.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Poll interval elapsed or shutdown. + } + continue; + } + + await RunAsync(job, shutdown).ConfigureAwait(false); + } + } + + private async Task RunAsync(Job job, CancellationToken shutdown) + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(shutdown); + _running[job.Id] = cts; + JobChanged?.Invoke(job); + var context = new JobContext(this, job, cts.Token); + JobState final; + string? error = null; + try + { + if (!_handlers.TryGetValue(job.Kind, out var handler)) + throw new InvalidOperationException($"No handler for {job.Kind} jobs on this worker."); + await handler.RunAsync(context).ConfigureAwait(false); + await PersistAsync(context).ConfigureAwait(false); + final = JobState.Done; + } + catch (OperationCanceledException) when (cts.IsCancellationRequested) + { + await PersistAsync(context).ConfigureAwait(false); + if (shutdown.IsCancellationRequested) + { + // Engine stopping: leave the job running so the next start requeues it from its checkpoint. + _running.TryRemove(job.Id, out _); + return; + } + // Cancel or pause already wrote the state; keep it. + _running.TryRemove(job.Id, out _); + await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false); + return; + } + catch (Exception ex) + { + Log.Error("jobs", $"{job.Kind} job {job.Id} failed", ex); + context.Log(ex.Message); + await PersistAsync(context).ConfigureAwait(false); + final = JobState.Failed; + error = ex.Message; + } + finally + { + _running.TryRemove(job.Id, out _); + } + await _db.Jobs.SetStateAsync(job.Id, final, error, CancellationToken.None).ConfigureAwait(false); + await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false); + } +} diff --git a/src/Titano.Data/Jobs/JobHandler.cs b/src/Titano.Data/Jobs/JobHandler.cs new file mode 100644 index 00000000..a8c18705 --- /dev/null +++ b/src/Titano.Data/Jobs/JobHandler.cs @@ -0,0 +1,131 @@ +using System.Diagnostics; +using System.Text; +using Titano.Core.Model; + +namespace Titano.Data.Jobs; + +/// Runs one kind of job (scan, verify, render, …). Handlers must honour +/// and resume from when it is set. +public interface IJobHandler +{ + JobKind Kind { get; } + + Task RunAsync(JobContext context); +} + +/// Live progress of a running job (pushed to the job bar, the Jobs screen and WebSocket clients). +public sealed record JobProgressInfo(Guid JobId, JobKind Kind, string? TargetLabel, long Done, long Total, double? ItemsPerSecond, TimeSpan? Eta, string? Message); + +/// Everything a handler needs while it runs. Progress is cheap to report; the engine persists it at most every +/// checkpoint interval (BRIEF §14.6: checkpoints every 5 s). +public sealed class JobContext +{ + private const int LogTailLines = 40; + private readonly JobEngine _engine; + private readonly Stopwatch _clock = Stopwatch.StartNew(); + private readonly Queue _log = new(); + private readonly Lock _gate = new(); + private long _lastPersistTicks; + private long _rateDone; + private double _rateSeconds; + private double? _rate; + + internal JobContext(JobEngine engine, Job job, CancellationToken cancellation) + { + _engine = engine; + Job = job; + Cancellation = cancellation; + Checkpoint = job.Checkpoint; + Done = job.ProgressDone; + Total = job.ProgressTotal; + _rateDone = Done; + if (job.LogTail is { Length: > 0 } tail) + { + foreach (var line in tail.Split('\n')) _log.Enqueue(line); + } + } + + public Job Job { get; } + + /// Signalled on cancel, pause and engine shutdown. + public CancellationToken Cancellation { get; } + + /// Resume cursor saved by a previous attempt (null on the first run). + public string? Checkpoint { get; private set; } + + public long Done { get; private set; } + + public long Total { get; private set; } + + public string? Message { get; private set; } + + /// Reports progress; persisted and broadcast at most every checkpoint interval. + public ValueTask ReportAsync(long done, long total, string? message = null) => ReportCoreAsync(done, total, message, force: false); + + /// Saves a resume cursor together with the current progress (persisted immediately). + public ValueTask CheckpointAsync(string checkpoint, long done, long total) + { + Checkpoint = checkpoint; + return ReportCoreAsync(done, total, Message, force: true); + } + + /// Appends a line to the job log (the last lines are kept with the job). + public void Log(string line) + { + lock (_gate) + { + _log.Enqueue($"{DateTime.UtcNow:HH:mm:ss} {line}"); + while (_log.Count > LogTailLines) _log.Dequeue(); + } + } + + internal string LogTail + { + get + { + lock (_gate) + { + var sb = new StringBuilder(); + foreach (var l in _log) + { + if (sb.Length > 0) sb.Append('\n'); + sb.Append(l); + } + return sb.ToString(); + } + } + } + + /// Throughput as a moving average over windows of at least two seconds, and the time left at that rate. + internal (double? Rate, TimeSpan? Eta) Rate() + { + lock (_gate) + { + var seconds = _clock.Elapsed.TotalSeconds; + var span = seconds - _rateSeconds; + if (span >= 2 && Done >= _rateDone) + { + var sample = (Done - _rateDone) / span; + _rate = _rate is { } previous ? previous * 0.7 + sample * 0.3 : sample; + _rateDone = Done; + _rateSeconds = seconds; + } + TimeSpan? eta = _rate is > 0 && Total > Done ? TimeSpan.FromSeconds((Total - Done) / _rate.Value) : null; + return (_rate, eta); + } + } + + private async ValueTask ReportCoreAsync(long done, long total, string? message, bool force) + { + Done = done; + Total = total; + Message = message; + var now = _clock.ElapsedTicks; + var due = now - Interlocked.Read(ref _lastPersistTicks) >= _engine.Options.CheckpointInterval.TotalSeconds * Stopwatch.Frequency; + _engine.PublishProgress(this); + if (!force && !due) return; + Interlocked.Exchange(ref _lastPersistTicks, now); + if (!await _engine.PersistAsync(this).ConfigureAwait(false)) + _engine.StopJob(Job.Id); // cancelled or paused by someone else + } +} diff --git a/src/Titano.Data/Sqlite/SqliteNative.cs b/src/Titano.Data/Sqlite/SqliteNative.cs index 97938b91..b86a4c77 100644 --- a/src/Titano.Data/Sqlite/SqliteNative.cs +++ b/src/Titano.Data/Sqlite/SqliteNative.cs @@ -11,7 +11,9 @@ namespace Titano.Data.Sqlite; internal static unsafe class SqliteNative { private const string Library = "titano-sqlite3"; - private static int _resolverInstalled; + // Set only after the resolver is in place: a concurrent first caller must not reach a P/Invoke before that. + private static readonly Lock ResolverGate = new(); + private static bool _resolverInstalled; public const int SQLITE_OK = 0; public const int SQLITE_INTERRUPT = 9; @@ -43,8 +45,13 @@ internal static unsafe class SqliteNative /// Routes the logical library name to the platform's system SQLite (once per process). public static void EnsureResolver() { - if (Interlocked.Exchange(ref _resolverInstalled, 1) == 0) + if (Volatile.Read(ref _resolverInstalled)) return; + lock (ResolverGate) + { + if (_resolverInstalled) return; NativeLibrary.SetDllImportResolver(typeof(SqliteNative).Assembly, Resolve); + Volatile.Write(ref _resolverInstalled, true); + } } private static nint Resolve(string name, Assembly assembly, DllImportSearchPath? searchPath) diff --git a/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs new file mode 100644 index 00000000..eec610a2 --- /dev/null +++ b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs @@ -0,0 +1,135 @@ +using Titano.Core.Model; +using Titano.Data.Jobs; +using Titano.Testing; + +namespace Titano.Data.Tests.Jobs; + +[Category("jobs")] +public sealed class JobEngineTests(TestContext context) +{ + private sealed class CountingHandler(JobKind kind, int steps, Func? onStep = null) : IJobHandler + { + public JobKind Kind => kind; + public List SeenCheckpoints { get; } = []; + public int Completed; + + public async Task RunAsync(JobContext c) + { + lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint); + var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0; + for (var i = start; i < steps; i++) + { + c.Cancellation.ThrowIfCancellationRequested(); + if (onStep is not null) await onStep(i, c.Cancellation); + await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps); + } + Interlocked.Increment(ref Completed); + } + } + + private async Task OpenAsync() => + await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation); + + private static async Task WaitForStateAsync(TitanoDatabase db, Guid id, JobState state, TimeSpan timeout) + { + var until = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < until) + { + var job = await db.Jobs.GetAsync(id); + if (job?.State == state) return job; + await Task.Delay(20); + } + Assert.Fail($"Job {id} did not reach {state}"); + return null!; + } + + [Test] + public async Task RunsQueuedJobsToCompletion() + { + await using var db = await OpenAsync(); + var handler = new CountingHandler(JobKind.Verify, 5); + await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) }); + var progress = 0; + engine.Progress += _ => Interlocked.Increment(ref progress); + await engine.StartAsync(); + var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A"); + var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B"); + var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10)); + await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10)); + Assert.Equal(2, handler.Completed); + Assert.Equal(5L, doneA.ProgressDone); + Assert.Greater(progress, 0); + } + + [Test] + public async Task CancelStopsARunningJob() + { + await using var db = await OpenAsync(); + var started = new TaskCompletionSource(); + var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) => + { + if (i == 0) started.TrySetResult(); + await Task.Delay(10, ct); + }); + await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); + await engine.StartAsync(); + var job = await engine.EnqueueAsync(JobKind.Scan, []); + await started.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await engine.CancelAsync(job.Id); + await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5)); + Assert.Equal(0, handler.Completed); + } + + [Test] + public async Task PauseAndResumeContinueFromCheckpoint() + { + await using var db = await OpenAsync(); + var gate = new SemaphoreSlim(0); + var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) => + { + if (i == 10) await gate.WaitAsync(ct); + }); + await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) }); + await engine.StartAsync(); + var job = await engine.EnqueueAsync(JobKind.Proxies, []); + var until = DateTime.UtcNow.AddSeconds(10); + while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20); + await engine.PauseAsync(job.Id); + var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5)); + Assert.Equal("10", paused.Checkpoint); + gate.Release(10); + await engine.ResumeAsync(job.Id); + await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10)); + Assert.Equal("10", handler.SeenCheckpoints[^1]); + } + + [Test] + public async Task CrashedJobsResumeAfterRestart() + { + await using var db = await OpenAsync(); + var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow }; + await db.Jobs.EnqueueAsync(job); + var claimed = await db.Jobs.ClaimNextAsync("t"); + Assert.NotNull(claimed); + await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null); + // Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it. + var handler = new CountingHandler(JobKind.Thumbs, 12); + await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); + await engine.StartAsync(); + await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10)); + Assert.Equal("7", handler.SeenCheckpoints.Single()); + } + + [Test] + public async Task HandlerFailureMarksJobFailedWithMessage() + { + await using var db = await OpenAsync(); + var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask); + await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); + await engine.StartAsync(); + var job = await engine.EnqueueAsync(JobKind.Render, []); + var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10)); + Assert.Equal("encoder lost", failed.Error); + Assert.Contains("encoder lost", failed.LogTail); + } +}