diff --git a/src/Titano.Data/Jobs/JobEngine.cs b/src/Titano.Data/Jobs/JobEngine.cs
new file mode 100644
index 00000000..b2b279f4
--- /dev/null
+++ b/src/Titano.Data/Jobs/JobEngine.cs
@@ -0,0 +1,246 @@
+using System.Collections.Concurrent;
+using System.Threading.Channels;
+using Titano.Core.Diagnostics;
+using Titano.Core.Model;
+
+namespace Titano.Data.Jobs;
+
+public sealed record JobEngineOptions
+{
+ /// Concurrent workers (BRIEF §14.6: default cores / 2).
+ public int Workers { get; init; } = Math.Max(1, Environment.ProcessorCount / 2);
+
+ /// Identity stamped on claimed jobs (node id); also used for crash recovery.
+ public string WorkerId { get; init; } = Environment.MachineName;
+
+ /// Kinds this engine runs; null = every kind with a registered handler.
+ public IReadOnlyCollection? Kinds { get; init; }
+
+ public TimeSpan CheckpointInterval { get; init; } = TimeSpan.FromSeconds(5);
+
+ /// Fallback poll for jobs enqueued by other processes.
+ public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(3);
+
+ /// Finished jobs older than this are purged (BRIEF §14.6: 90 days).
+ public TimeSpan HistoryRetention { get; init; } = TimeSpan.FromDays(90);
+}
+
+///
+/// Persistent job queue runner (BRIEF §14.6): N workers claim queued jobs from the database by priority, run the
+/// registered handler, checkpoint progress, and survive crashes (running jobs are requeued at start with their
+/// checkpoints). Used by the desktop client (local mode) and the server.
+///
+public sealed class JobEngine : IAsyncDisposable
+{
+ private readonly TitanoDatabase _db;
+ private readonly Dictionary _handlers;
+ private readonly ConcurrentDictionary _running = new();
+ private readonly Channel _wake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
+ private readonly CancellationTokenSource _shutdown = new();
+ private readonly List _workers = [];
+
+ public JobEngine(TitanoDatabase db, IEnumerable handlers, JobEngineOptions? options = null)
+ {
+ _db = db;
+ Options = options ?? new JobEngineOptions();
+ _handlers = handlers.ToDictionary(h => h.Kind);
+ }
+
+ public JobEngineOptions Options { get; }
+
+ /// A job changed state (queued, running, done, failed, cancelled, paused).
+ public event Action? JobChanged;
+
+ /// Progress of a running job (high frequency; throttle in the UI).
+ public event Action? Progress;
+
+ public IReadOnlyCollection RunningJobIds => [.. _running.Keys];
+
+ /// Requeues this worker's interrupted jobs, purges old history and starts the workers.
+ public async Task StartAsync(CancellationToken cancellationToken = default)
+ {
+ var requeued = await _db.Jobs.RequeueRunningAsync(Options.WorkerId, cancellationToken).ConfigureAwait(false);
+ if (requeued > 0) Log.Info("jobs", $"Requeued {requeued} interrupted job(s)");
+ await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false);
+ for (var i = 0; i < Options.Workers; i++)
+ {
+ var id = Options.Workers == 1 ? Options.WorkerId : $"{Options.WorkerId}#{i}";
+ _workers.Add(Task.Run(() => WorkerLoopAsync(id, _shutdown.Token), CancellationToken.None));
+ }
+ }
+
+ /// Queues a job and wakes a worker.
+ public async Task EnqueueAsync(JobKind kind, IReadOnlyList targets, string? targetLabel = null, string? parametersJson = null,
+ JobPriority priority = JobPriority.Normal, Guid? createdBy = null, CancellationToken cancellationToken = default)
+ {
+ var job = new Job
+ {
+ Id = Guid.CreateVersion7(),
+ Kind = kind,
+ TargetIds = targets,
+ TargetLabel = targetLabel,
+ ParametersJson = parametersJson,
+ Priority = priority,
+ State = JobState.Queued,
+ CreatedBy = createdBy,
+ Created = DateTimeOffset.UtcNow,
+ };
+ await _db.Jobs.EnqueueAsync(job, cancellationToken).ConfigureAwait(false);
+ JobChanged?.Invoke(job);
+ Wake();
+ return job;
+ }
+
+ /// Cancels a queued or running job; a running handler observes its token.
+ public Task CancelAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Cancelled, cancellationToken);
+
+ /// Pauses a job: a running one stops at its next check and keeps its checkpoint.
+ public Task PauseAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Paused, cancellationToken);
+
+ /// Resumes a paused job or retries a failed/cancelled one from its last checkpoint.
+ public async Task ResumeAsync(Guid id, CancellationToken cancellationToken = default)
+ {
+ await _db.Jobs.SetStateAsync(id, JobState.Queued, null, cancellationToken).ConfigureAwait(false);
+ await NotifyAsync(id, cancellationToken).ConfigureAwait(false);
+ Wake();
+ }
+
+ /// Signals workers that new work may be available (also call on PostgreSQL NOTIFY).
+ public void Wake() => _wake.Writer.TryWrite(true);
+
+ public async ValueTask DisposeAsync()
+ {
+ await _shutdown.CancelAsync().ConfigureAwait(false);
+ try
+ {
+ await Task.WhenAll(_workers).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ }
+ _shutdown.Dispose();
+ }
+
+ internal void PublishProgress(JobContext context)
+ {
+ var (rate, eta) = context.Rate();
+ Progress?.Invoke(new JobProgressInfo(context.Job.Id, context.Job.Kind, context.Job.TargetLabel, context.Done, context.Total, rate, eta, context.Message));
+ }
+
+ internal async Task PersistAsync(JobContext context)
+ {
+ var (rate, eta) = context.Rate();
+ try
+ {
+ return await _db.Jobs.UpdateProgressAsync(context.Job.Id, context.Done, context.Total, rate, eta, context.Checkpoint, context.LogTail, CancellationToken.None)
+ .ConfigureAwait(false);
+ }
+ catch (DatabaseException ex) when (ex.IsTransient)
+ {
+ Log.Warning("jobs", "Progress not saved (transient database error)", ex);
+ return true;
+ }
+ }
+
+ internal void StopJob(Guid id)
+ {
+ if (_running.TryGetValue(id, out var cts)) cts.Cancel();
+ }
+
+ private async Task TransitionAsync(Guid id, JobState state, CancellationToken cancellationToken)
+ {
+ await _db.Jobs.SetStateAsync(id, state, null, cancellationToken).ConfigureAwait(false);
+ StopJob(id);
+ await NotifyAsync(id, cancellationToken).ConfigureAwait(false);
+ }
+
+ private async Task NotifyAsync(Guid id, CancellationToken cancellationToken)
+ {
+ if (await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } job) JobChanged?.Invoke(job);
+ }
+
+ private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown)
+ {
+ var kinds = Options.Kinds ?? [.. _handlers.Keys];
+ while (!shutdown.IsCancellationRequested)
+ {
+ Job? job;
+ try
+ {
+ job = await _db.Jobs.ClaimNextAsync(workerId, kinds, shutdown).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ return;
+ }
+ catch (DatabaseException ex)
+ {
+ Log.Warning("jobs", "Claiming a job failed", ex);
+ job = null;
+ }
+
+ if (job is null)
+ {
+ using var timeout = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
+ timeout.CancelAfter(Options.PollInterval);
+ try
+ {
+ await _wake.Reader.ReadAsync(timeout.Token).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ // Poll interval elapsed or shutdown.
+ }
+ continue;
+ }
+
+ await RunAsync(job, shutdown).ConfigureAwait(false);
+ }
+ }
+
+ private async Task RunAsync(Job job, CancellationToken shutdown)
+ {
+ using var cts = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
+ _running[job.Id] = cts;
+ JobChanged?.Invoke(job);
+ var context = new JobContext(this, job, cts.Token);
+ JobState final;
+ string? error = null;
+ try
+ {
+ if (!_handlers.TryGetValue(job.Kind, out var handler))
+ throw new InvalidOperationException($"No handler for {job.Kind} jobs on this worker.");
+ await handler.RunAsync(context).ConfigureAwait(false);
+ await PersistAsync(context).ConfigureAwait(false);
+ final = JobState.Done;
+ }
+ catch (OperationCanceledException) when (cts.IsCancellationRequested)
+ {
+ await PersistAsync(context).ConfigureAwait(false);
+ if (shutdown.IsCancellationRequested)
+ {
+ // Engine stopping: leave the job running so the next start requeues it from its checkpoint.
+ _running.TryRemove(job.Id, out _);
+ return;
+ }
+ // Cancel or pause already wrote the state; keep it.
+ _running.TryRemove(job.Id, out _);
+ await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false);
+ return;
+ }
+ catch (Exception ex)
+ {
+ Log.Error("jobs", $"{job.Kind} job {job.Id} failed", ex);
+ context.Log(ex.Message);
+ await PersistAsync(context).ConfigureAwait(false);
+ final = JobState.Failed;
+ error = ex.Message;
+ }
+ finally
+ {
+ _running.TryRemove(job.Id, out _);
+ }
+ await _db.Jobs.SetStateAsync(job.Id, final, error, CancellationToken.None).ConfigureAwait(false);
+ await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false);
+ }
+}
diff --git a/src/Titano.Data/Jobs/JobHandler.cs b/src/Titano.Data/Jobs/JobHandler.cs
new file mode 100644
index 00000000..a8c18705
--- /dev/null
+++ b/src/Titano.Data/Jobs/JobHandler.cs
@@ -0,0 +1,131 @@
+using System.Diagnostics;
+using System.Text;
+using Titano.Core.Model;
+
+namespace Titano.Data.Jobs;
+
+/// Runs one kind of job (scan, verify, render, …). Handlers must honour
+/// and resume from when it is set.
+public interface IJobHandler
+{
+ JobKind Kind { get; }
+
+ Task RunAsync(JobContext context);
+}
+
+/// Live progress of a running job (pushed to the job bar, the Jobs screen and WebSocket clients).
+public sealed record JobProgressInfo(Guid JobId, JobKind Kind, string? TargetLabel, long Done, long Total, double? ItemsPerSecond, TimeSpan? Eta, string? Message);
+
+/// Everything a handler needs while it runs. Progress is cheap to report; the engine persists it at most every
+/// checkpoint interval (BRIEF §14.6: checkpoints every 5 s).
+public sealed class JobContext
+{
+ private const int LogTailLines = 40;
+ private readonly JobEngine _engine;
+ private readonly Stopwatch _clock = Stopwatch.StartNew();
+ private readonly Queue _log = new();
+ private readonly Lock _gate = new();
+ private long _lastPersistTicks;
+ private long _rateDone;
+ private double _rateSeconds;
+ private double? _rate;
+
+ internal JobContext(JobEngine engine, Job job, CancellationToken cancellation)
+ {
+ _engine = engine;
+ Job = job;
+ Cancellation = cancellation;
+ Checkpoint = job.Checkpoint;
+ Done = job.ProgressDone;
+ Total = job.ProgressTotal;
+ _rateDone = Done;
+ if (job.LogTail is { Length: > 0 } tail)
+ {
+ foreach (var line in tail.Split('\n')) _log.Enqueue(line);
+ }
+ }
+
+ public Job Job { get; }
+
+ /// Signalled on cancel, pause and engine shutdown.
+ public CancellationToken Cancellation { get; }
+
+ /// Resume cursor saved by a previous attempt (null on the first run).
+ public string? Checkpoint { get; private set; }
+
+ public long Done { get; private set; }
+
+ public long Total { get; private set; }
+
+ public string? Message { get; private set; }
+
+ /// Reports progress; persisted and broadcast at most every checkpoint interval.
+ public ValueTask ReportAsync(long done, long total, string? message = null) => ReportCoreAsync(done, total, message, force: false);
+
+ /// Saves a resume cursor together with the current progress (persisted immediately).
+ public ValueTask CheckpointAsync(string checkpoint, long done, long total)
+ {
+ Checkpoint = checkpoint;
+ return ReportCoreAsync(done, total, Message, force: true);
+ }
+
+ /// Appends a line to the job log (the last lines are kept with the job).
+ public void Log(string line)
+ {
+ lock (_gate)
+ {
+ _log.Enqueue($"{DateTime.UtcNow:HH:mm:ss} {line}");
+ while (_log.Count > LogTailLines) _log.Dequeue();
+ }
+ }
+
+ internal string LogTail
+ {
+ get
+ {
+ lock (_gate)
+ {
+ var sb = new StringBuilder();
+ foreach (var l in _log)
+ {
+ if (sb.Length > 0) sb.Append('\n');
+ sb.Append(l);
+ }
+ return sb.ToString();
+ }
+ }
+ }
+
+ /// Throughput as a moving average over windows of at least two seconds, and the time left at that rate.
+ internal (double? Rate, TimeSpan? Eta) Rate()
+ {
+ lock (_gate)
+ {
+ var seconds = _clock.Elapsed.TotalSeconds;
+ var span = seconds - _rateSeconds;
+ if (span >= 2 && Done >= _rateDone)
+ {
+ var sample = (Done - _rateDone) / span;
+ _rate = _rate is { } previous ? previous * 0.7 + sample * 0.3 : sample;
+ _rateDone = Done;
+ _rateSeconds = seconds;
+ }
+ TimeSpan? eta = _rate is > 0 && Total > Done ? TimeSpan.FromSeconds((Total - Done) / _rate.Value) : null;
+ return (_rate, eta);
+ }
+ }
+
+ private async ValueTask ReportCoreAsync(long done, long total, string? message, bool force)
+ {
+ Done = done;
+ Total = total;
+ Message = message;
+ var now = _clock.ElapsedTicks;
+ var due = now - Interlocked.Read(ref _lastPersistTicks) >= _engine.Options.CheckpointInterval.TotalSeconds * Stopwatch.Frequency;
+ _engine.PublishProgress(this);
+ if (!force && !due) return;
+ Interlocked.Exchange(ref _lastPersistTicks, now);
+ if (!await _engine.PersistAsync(this).ConfigureAwait(false))
+ _engine.StopJob(Job.Id); // cancelled or paused by someone else
+ }
+}
diff --git a/src/Titano.Data/Sqlite/SqliteNative.cs b/src/Titano.Data/Sqlite/SqliteNative.cs
index 97938b91..b86a4c77 100644
--- a/src/Titano.Data/Sqlite/SqliteNative.cs
+++ b/src/Titano.Data/Sqlite/SqliteNative.cs
@@ -11,7 +11,9 @@ namespace Titano.Data.Sqlite;
internal static unsafe class SqliteNative
{
private const string Library = "titano-sqlite3";
- private static int _resolverInstalled;
+ // Set only after the resolver is in place: a concurrent first caller must not reach a P/Invoke before that.
+ private static readonly Lock ResolverGate = new();
+ private static bool _resolverInstalled;
public const int SQLITE_OK = 0;
public const int SQLITE_INTERRUPT = 9;
@@ -43,8 +45,13 @@ internal static unsafe class SqliteNative
/// Routes the logical library name to the platform's system SQLite (once per process).
public static void EnsureResolver()
{
- if (Interlocked.Exchange(ref _resolverInstalled, 1) == 0)
+ if (Volatile.Read(ref _resolverInstalled)) return;
+ lock (ResolverGate)
+ {
+ if (_resolverInstalled) return;
NativeLibrary.SetDllImportResolver(typeof(SqliteNative).Assembly, Resolve);
+ Volatile.Write(ref _resolverInstalled, true);
+ }
}
private static nint Resolve(string name, Assembly assembly, DllImportSearchPath? searchPath)
diff --git a/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs
new file mode 100644
index 00000000..eec610a2
--- /dev/null
+++ b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs
@@ -0,0 +1,135 @@
+using Titano.Core.Model;
+using Titano.Data.Jobs;
+using Titano.Testing;
+
+namespace Titano.Data.Tests.Jobs;
+
+[Category("jobs")]
+public sealed class JobEngineTests(TestContext context)
+{
+ private sealed class CountingHandler(JobKind kind, int steps, Func? onStep = null) : IJobHandler
+ {
+ public JobKind Kind => kind;
+ public List SeenCheckpoints { get; } = [];
+ public int Completed;
+
+ public async Task RunAsync(JobContext c)
+ {
+ lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint);
+ var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0;
+ for (var i = start; i < steps; i++)
+ {
+ c.Cancellation.ThrowIfCancellationRequested();
+ if (onStep is not null) await onStep(i, c.Cancellation);
+ await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps);
+ }
+ Interlocked.Increment(ref Completed);
+ }
+ }
+
+ private async Task OpenAsync() =>
+ await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
+
+ private static async Task WaitForStateAsync(TitanoDatabase db, Guid id, JobState state, TimeSpan timeout)
+ {
+ var until = DateTime.UtcNow + timeout;
+ while (DateTime.UtcNow < until)
+ {
+ var job = await db.Jobs.GetAsync(id);
+ if (job?.State == state) return job;
+ await Task.Delay(20);
+ }
+ Assert.Fail($"Job {id} did not reach {state}");
+ return null!;
+ }
+
+ [Test]
+ public async Task RunsQueuedJobsToCompletion()
+ {
+ await using var db = await OpenAsync();
+ var handler = new CountingHandler(JobKind.Verify, 5);
+ await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
+ var progress = 0;
+ engine.Progress += _ => Interlocked.Increment(ref progress);
+ await engine.StartAsync();
+ var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A");
+ var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B");
+ var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10));
+ await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10));
+ Assert.Equal(2, handler.Completed);
+ Assert.Equal(5L, doneA.ProgressDone);
+ Assert.Greater(progress, 0);
+ }
+
+ [Test]
+ public async Task CancelStopsARunningJob()
+ {
+ await using var db = await OpenAsync();
+ var started = new TaskCompletionSource();
+ var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) =>
+ {
+ if (i == 0) started.TrySetResult();
+ await Task.Delay(10, ct);
+ });
+ await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
+ await engine.StartAsync();
+ var job = await engine.EnqueueAsync(JobKind.Scan, []);
+ await started.Task.WaitAsync(TimeSpan.FromSeconds(10));
+ await engine.CancelAsync(job.Id);
+ await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5));
+ Assert.Equal(0, handler.Completed);
+ }
+
+ [Test]
+ public async Task PauseAndResumeContinueFromCheckpoint()
+ {
+ await using var db = await OpenAsync();
+ var gate = new SemaphoreSlim(0);
+ var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) =>
+ {
+ if (i == 10) await gate.WaitAsync(ct);
+ });
+ await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
+ await engine.StartAsync();
+ var job = await engine.EnqueueAsync(JobKind.Proxies, []);
+ var until = DateTime.UtcNow.AddSeconds(10);
+ while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20);
+ await engine.PauseAsync(job.Id);
+ var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5));
+ Assert.Equal("10", paused.Checkpoint);
+ gate.Release(10);
+ await engine.ResumeAsync(job.Id);
+ await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
+ Assert.Equal("10", handler.SeenCheckpoints[^1]);
+ }
+
+ [Test]
+ public async Task CrashedJobsResumeAfterRestart()
+ {
+ await using var db = await OpenAsync();
+ var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
+ await db.Jobs.EnqueueAsync(job);
+ var claimed = await db.Jobs.ClaimNextAsync("t");
+ Assert.NotNull(claimed);
+ await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null);
+ // Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it.
+ var handler = new CountingHandler(JobKind.Thumbs, 12);
+ await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
+ await engine.StartAsync();
+ await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
+ Assert.Equal("7", handler.SeenCheckpoints.Single());
+ }
+
+ [Test]
+ public async Task HandlerFailureMarksJobFailedWithMessage()
+ {
+ await using var db = await OpenAsync();
+ var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask);
+ await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
+ await engine.StartAsync();
+ var job = await engine.EnqueueAsync(JobKind.Render, []);
+ var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10));
+ Assert.Equal("encoder lost", failed.Error);
+ Assert.Contains("encoder lost", failed.LogTail);
+ }
+}