feat(data): persistent job engine with checkpoints, pause/resume and crash recovery
JobEngine claims queued jobs atomically, persists progress at most every checkpoint interval, stops a handler when its job is cancelled or paused elsewhere and requeues jobs a crash left running. Also installs the SQLite import resolver under a lock so a concurrent first caller cannot reach a P/Invoke before the resolver is in place. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
4 files changed
+521
-2
No files matched your search
@@ -0,0 +1,246 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Threading.Channels;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Jobs;
|
||||
|
||||
public sealed record JobEngineOptions
|
||||
{
|
||||
/// <summary>Concurrent workers (BRIEF §14.6: default cores / 2).</summary>
|
||||
public int Workers { get; init; } = Math.Max(1, Environment.ProcessorCount / 2);
|
||||
|
||||
/// <summary>Identity stamped on claimed jobs (node id); also used for crash recovery.</summary>
|
||||
public string WorkerId { get; init; } = Environment.MachineName;
|
||||
|
||||
/// <summary>Kinds this engine runs; null = every kind with a registered handler.</summary>
|
||||
public IReadOnlyCollection<JobKind>? Kinds { get; init; }
|
||||
|
||||
public TimeSpan CheckpointInterval { get; init; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Fallback poll for jobs enqueued by other processes.</summary>
|
||||
public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(3);
|
||||
|
||||
/// <summary>Finished jobs older than this are purged (BRIEF §14.6: 90 days).</summary>
|
||||
public TimeSpan HistoryRetention { get; init; } = TimeSpan.FromDays(90);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Persistent job queue runner (BRIEF §14.6): N workers claim queued jobs from the database by priority, run the
|
||||
/// registered handler, checkpoint progress, and survive crashes (running jobs are requeued at start with their
|
||||
/// checkpoints). Used by the desktop client (local mode) and the server.
|
||||
/// </summary>
|
||||
public sealed class JobEngine : IAsyncDisposable
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly Dictionary<JobKind, IJobHandler> _handlers;
|
||||
private readonly ConcurrentDictionary<Guid, CancellationTokenSource> _running = new();
|
||||
private readonly Channel<bool> _wake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
|
||||
private readonly CancellationTokenSource _shutdown = new();
|
||||
private readonly List<Task> _workers = [];
|
||||
|
||||
public JobEngine(TitanoDatabase db, IEnumerable<IJobHandler> handlers, JobEngineOptions? options = null)
|
||||
{
|
||||
_db = db;
|
||||
Options = options ?? new JobEngineOptions();
|
||||
_handlers = handlers.ToDictionary(h => h.Kind);
|
||||
}
|
||||
|
||||
public JobEngineOptions Options { get; }
|
||||
|
||||
/// <summary>A job changed state (queued, running, done, failed, cancelled, paused).</summary>
|
||||
public event Action<Job>? JobChanged;
|
||||
|
||||
/// <summary>Progress of a running job (high frequency; throttle in the UI).</summary>
|
||||
public event Action<JobProgressInfo>? Progress;
|
||||
|
||||
public IReadOnlyCollection<Guid> RunningJobIds => [.. _running.Keys];
|
||||
|
||||
/// <summary>Requeues this worker's interrupted jobs, purges old history and starts the workers.</summary>
|
||||
public async Task StartAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var requeued = await _db.Jobs.RequeueRunningAsync(Options.WorkerId, cancellationToken).ConfigureAwait(false);
|
||||
if (requeued > 0) Log.Info("jobs", $"Requeued {requeued} interrupted job(s)");
|
||||
await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false);
|
||||
for (var i = 0; i < Options.Workers; i++)
|
||||
{
|
||||
var id = Options.Workers == 1 ? Options.WorkerId : $"{Options.WorkerId}#{i}";
|
||||
_workers.Add(Task.Run(() => WorkerLoopAsync(id, _shutdown.Token), CancellationToken.None));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Queues a job and wakes a worker.</summary>
|
||||
public async Task<Job> EnqueueAsync(JobKind kind, IReadOnlyList<Guid> targets, string? targetLabel = null, string? parametersJson = null,
|
||||
JobPriority priority = JobPriority.Normal, Guid? createdBy = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var job = new Job
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Kind = kind,
|
||||
TargetIds = targets,
|
||||
TargetLabel = targetLabel,
|
||||
ParametersJson = parametersJson,
|
||||
Priority = priority,
|
||||
State = JobState.Queued,
|
||||
CreatedBy = createdBy,
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
};
|
||||
await _db.Jobs.EnqueueAsync(job, cancellationToken).ConfigureAwait(false);
|
||||
JobChanged?.Invoke(job);
|
||||
Wake();
|
||||
return job;
|
||||
}
|
||||
|
||||
/// <summary>Cancels a queued or running job; a running handler observes its token.</summary>
|
||||
public Task CancelAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Cancelled, cancellationToken);
|
||||
|
||||
/// <summary>Pauses a job: a running one stops at its next check and keeps its checkpoint.</summary>
|
||||
public Task PauseAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Paused, cancellationToken);
|
||||
|
||||
/// <summary>Resumes a paused job or retries a failed/cancelled one from its last checkpoint.</summary>
|
||||
public async Task ResumeAsync(Guid id, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await _db.Jobs.SetStateAsync(id, JobState.Queued, null, cancellationToken).ConfigureAwait(false);
|
||||
await NotifyAsync(id, cancellationToken).ConfigureAwait(false);
|
||||
Wake();
|
||||
}
|
||||
|
||||
/// <summary>Signals workers that new work may be available (also call on PostgreSQL NOTIFY).</summary>
|
||||
public void Wake() => _wake.Writer.TryWrite(true);
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _shutdown.CancelAsync().ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(_workers).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
_shutdown.Dispose();
|
||||
}
|
||||
|
||||
internal void PublishProgress(JobContext context)
|
||||
{
|
||||
var (rate, eta) = context.Rate();
|
||||
Progress?.Invoke(new JobProgressInfo(context.Job.Id, context.Job.Kind, context.Job.TargetLabel, context.Done, context.Total, rate, eta, context.Message));
|
||||
}
|
||||
|
||||
internal async Task<bool> PersistAsync(JobContext context)
|
||||
{
|
||||
var (rate, eta) = context.Rate();
|
||||
try
|
||||
{
|
||||
return await _db.Jobs.UpdateProgressAsync(context.Job.Id, context.Done, context.Total, rate, eta, context.Checkpoint, context.LogTail, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (DatabaseException ex) when (ex.IsTransient)
|
||||
{
|
||||
Log.Warning("jobs", "Progress not saved (transient database error)", ex);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
internal void StopJob(Guid id)
|
||||
{
|
||||
if (_running.TryGetValue(id, out var cts)) cts.Cancel();
|
||||
}
|
||||
|
||||
private async Task TransitionAsync(Guid id, JobState state, CancellationToken cancellationToken)
|
||||
{
|
||||
await _db.Jobs.SetStateAsync(id, state, null, cancellationToken).ConfigureAwait(false);
|
||||
StopJob(id);
|
||||
await NotifyAsync(id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task NotifyAsync(Guid id, CancellationToken cancellationToken)
|
||||
{
|
||||
if (await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } job) JobChanged?.Invoke(job);
|
||||
}
|
||||
|
||||
private async Task WorkerLoopAsync(string workerId, CancellationToken shutdown)
|
||||
{
|
||||
var kinds = Options.Kinds ?? [.. _handlers.Keys];
|
||||
while (!shutdown.IsCancellationRequested)
|
||||
{
|
||||
Job? job;
|
||||
try
|
||||
{
|
||||
job = await _db.Jobs.ClaimNextAsync(workerId, kinds, shutdown).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (DatabaseException ex)
|
||||
{
|
||||
Log.Warning("jobs", "Claiming a job failed", ex);
|
||||
job = null;
|
||||
}
|
||||
|
||||
if (job is null)
|
||||
{
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
|
||||
timeout.CancelAfter(Options.PollInterval);
|
||||
try
|
||||
{
|
||||
await _wake.Reader.ReadAsync(timeout.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Poll interval elapsed or shutdown.
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
await RunAsync(job, shutdown).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunAsync(Job job, CancellationToken shutdown)
|
||||
{
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
|
||||
_running[job.Id] = cts;
|
||||
JobChanged?.Invoke(job);
|
||||
var context = new JobContext(this, job, cts.Token);
|
||||
JobState final;
|
||||
string? error = null;
|
||||
try
|
||||
{
|
||||
if (!_handlers.TryGetValue(job.Kind, out var handler))
|
||||
throw new InvalidOperationException($"No handler for {job.Kind} jobs on this worker.");
|
||||
await handler.RunAsync(context).ConfigureAwait(false);
|
||||
await PersistAsync(context).ConfigureAwait(false);
|
||||
final = JobState.Done;
|
||||
}
|
||||
catch (OperationCanceledException) when (cts.IsCancellationRequested)
|
||||
{
|
||||
await PersistAsync(context).ConfigureAwait(false);
|
||||
if (shutdown.IsCancellationRequested)
|
||||
{
|
||||
// Engine stopping: leave the job running so the next start requeues it from its checkpoint.
|
||||
_running.TryRemove(job.Id, out _);
|
||||
return;
|
||||
}
|
||||
// Cancel or pause already wrote the state; keep it.
|
||||
_running.TryRemove(job.Id, out _);
|
||||
await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Error("jobs", $"{job.Kind} job {job.Id} failed", ex);
|
||||
context.Log(ex.Message);
|
||||
await PersistAsync(context).ConfigureAwait(false);
|
||||
final = JobState.Failed;
|
||||
error = ex.Message;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_running.TryRemove(job.Id, out _);
|
||||
}
|
||||
await _db.Jobs.SetStateAsync(job.Id, final, error, CancellationToken.None).ConfigureAwait(false);
|
||||
await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Jobs;
|
||||
|
||||
/// <summary>Runs one kind of job (scan, verify, render, …). Handlers must honour <see cref="JobContext.Cancellation"/>
|
||||
/// and resume from <see cref="JobContext.Checkpoint"/> when it is set.</summary>
|
||||
public interface IJobHandler
|
||||
{
|
||||
JobKind Kind { get; }
|
||||
|
||||
Task RunAsync(JobContext context);
|
||||
}
|
||||
|
||||
/// <summary>Live progress of a running job (pushed to the job bar, the Jobs screen and WebSocket clients).</summary>
|
||||
public sealed record JobProgressInfo(Guid JobId, JobKind Kind, string? TargetLabel, long Done, long Total, double? ItemsPerSecond, TimeSpan? Eta, string? Message);
|
||||
|
||||
/// <summary>Everything a handler needs while it runs. Progress is cheap to report; the engine persists it at most every
|
||||
/// checkpoint interval (BRIEF §14.6: checkpoints every 5 s).</summary>
|
||||
public sealed class JobContext
|
||||
{
|
||||
private const int LogTailLines = 40;
|
||||
private readonly JobEngine _engine;
|
||||
private readonly Stopwatch _clock = Stopwatch.StartNew();
|
||||
private readonly Queue<string> _log = new();
|
||||
private readonly Lock _gate = new();
|
||||
private long _lastPersistTicks;
|
||||
private long _rateDone;
|
||||
private double _rateSeconds;
|
||||
private double? _rate;
|
||||
|
||||
internal JobContext(JobEngine engine, Job job, CancellationToken cancellation)
|
||||
{
|
||||
_engine = engine;
|
||||
Job = job;
|
||||
Cancellation = cancellation;
|
||||
Checkpoint = job.Checkpoint;
|
||||
Done = job.ProgressDone;
|
||||
Total = job.ProgressTotal;
|
||||
_rateDone = Done;
|
||||
if (job.LogTail is { Length: > 0 } tail)
|
||||
{
|
||||
foreach (var line in tail.Split('\n')) _log.Enqueue(line);
|
||||
}
|
||||
}
|
||||
|
||||
public Job Job { get; }
|
||||
|
||||
/// <summary>Signalled on cancel, pause and engine shutdown.</summary>
|
||||
public CancellationToken Cancellation { get; }
|
||||
|
||||
/// <summary>Resume cursor saved by a previous attempt (null on the first run).</summary>
|
||||
public string? Checkpoint { get; private set; }
|
||||
|
||||
public long Done { get; private set; }
|
||||
|
||||
public long Total { get; private set; }
|
||||
|
||||
public string? Message { get; private set; }
|
||||
|
||||
/// <summary>Reports progress; persisted and broadcast at most every checkpoint interval.</summary>
|
||||
public ValueTask ReportAsync(long done, long total, string? message = null) => ReportCoreAsync(done, total, message, force: false);
|
||||
|
||||
/// <summary>Saves a resume cursor together with the current progress (persisted immediately).</summary>
|
||||
public ValueTask CheckpointAsync(string checkpoint, long done, long total)
|
||||
{
|
||||
Checkpoint = checkpoint;
|
||||
return ReportCoreAsync(done, total, Message, force: true);
|
||||
}
|
||||
|
||||
/// <summary>Appends a line to the job log (the last lines are kept with the job).</summary>
|
||||
public void Log(string line)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_log.Enqueue($"{DateTime.UtcNow:HH:mm:ss} {line}");
|
||||
while (_log.Count > LogTailLines) _log.Dequeue();
|
||||
}
|
||||
}
|
||||
|
||||
internal string LogTail
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
foreach (var l in _log)
|
||||
{
|
||||
if (sb.Length > 0) sb.Append('\n');
|
||||
sb.Append(l);
|
||||
}
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Throughput as a moving average over windows of at least two seconds, and the time left at that rate.</summary>
|
||||
internal (double? Rate, TimeSpan? Eta) Rate()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var seconds = _clock.Elapsed.TotalSeconds;
|
||||
var span = seconds - _rateSeconds;
|
||||
if (span >= 2 && Done >= _rateDone)
|
||||
{
|
||||
var sample = (Done - _rateDone) / span;
|
||||
_rate = _rate is { } previous ? previous * 0.7 + sample * 0.3 : sample;
|
||||
_rateDone = Done;
|
||||
_rateSeconds = seconds;
|
||||
}
|
||||
TimeSpan? eta = _rate is > 0 && Total > Done ? TimeSpan.FromSeconds((Total - Done) / _rate.Value) : null;
|
||||
return (_rate, eta);
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask ReportCoreAsync(long done, long total, string? message, bool force)
|
||||
{
|
||||
Done = done;
|
||||
Total = total;
|
||||
Message = message;
|
||||
var now = _clock.ElapsedTicks;
|
||||
var due = now - Interlocked.Read(ref _lastPersistTicks) >= _engine.Options.CheckpointInterval.TotalSeconds * Stopwatch.Frequency;
|
||||
_engine.PublishProgress(this);
|
||||
if (!force && !due) return;
|
||||
Interlocked.Exchange(ref _lastPersistTicks, now);
|
||||
if (!await _engine.PersistAsync(this).ConfigureAwait(false))
|
||||
_engine.StopJob(Job.Id); // cancelled or paused by someone else
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,9 @@ namespace Titano.Data.Sqlite;
|
||||
internal static unsafe class SqliteNative
|
||||
{
|
||||
private const string Library = "titano-sqlite3";
|
||||
private static int _resolverInstalled;
|
||||
// Set only after the resolver is in place: a concurrent first caller must not reach a P/Invoke before that.
|
||||
private static readonly Lock ResolverGate = new();
|
||||
private static bool _resolverInstalled;
|
||||
|
||||
public const int SQLITE_OK = 0;
|
||||
public const int SQLITE_INTERRUPT = 9;
|
||||
@@ -43,8 +45,13 @@ internal static unsafe class SqliteNative
|
||||
/// <summary>Routes the logical library name to the platform's system SQLite (once per process).</summary>
|
||||
public static void EnsureResolver()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _resolverInstalled, 1) == 0)
|
||||
if (Volatile.Read(ref _resolverInstalled)) return;
|
||||
lock (ResolverGate)
|
||||
{
|
||||
if (_resolverInstalled) return;
|
||||
NativeLibrary.SetDllImportResolver(typeof(SqliteNative).Assembly, Resolve);
|
||||
Volatile.Write(ref _resolverInstalled, true);
|
||||
}
|
||||
}
|
||||
|
||||
private static nint Resolve(string name, Assembly assembly, DllImportSearchPath? searchPath)
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Data.Tests.Jobs;
|
||||
|
||||
[Category("jobs")]
|
||||
public sealed class JobEngineTests(TestContext context)
|
||||
{
|
||||
private sealed class CountingHandler(JobKind kind, int steps, Func<int, CancellationToken, Task>? onStep = null) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => kind;
|
||||
public List<string?> SeenCheckpoints { get; } = [];
|
||||
public int Completed;
|
||||
|
||||
public async Task RunAsync(JobContext c)
|
||||
{
|
||||
lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint);
|
||||
var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0;
|
||||
for (var i = start; i < steps; i++)
|
||||
{
|
||||
c.Cancellation.ThrowIfCancellationRequested();
|
||||
if (onStep is not null) await onStep(i, c.Cancellation);
|
||||
await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps);
|
||||
}
|
||||
Interlocked.Increment(ref Completed);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<TitanoDatabase> OpenAsync() =>
|
||||
await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
|
||||
|
||||
private static async Task<Job> WaitForStateAsync(TitanoDatabase db, Guid id, JobState state, TimeSpan timeout)
|
||||
{
|
||||
var until = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < until)
|
||||
{
|
||||
var job = await db.Jobs.GetAsync(id);
|
||||
if (job?.State == state) return job;
|
||||
await Task.Delay(20);
|
||||
}
|
||||
Assert.Fail($"Job {id} did not reach {state}");
|
||||
return null!;
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RunsQueuedJobsToCompletion()
|
||||
{
|
||||
await using var db = await OpenAsync();
|
||||
var handler = new CountingHandler(JobKind.Verify, 5);
|
||||
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
||||
var progress = 0;
|
||||
engine.Progress += _ => Interlocked.Increment(ref progress);
|
||||
await engine.StartAsync();
|
||||
var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A");
|
||||
var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B");
|
||||
var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
||||
await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
||||
Assert.Equal(2, handler.Completed);
|
||||
Assert.Equal(5L, doneA.ProgressDone);
|
||||
Assert.Greater(progress, 0);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task CancelStopsARunningJob()
|
||||
{
|
||||
await using var db = await OpenAsync();
|
||||
var started = new TaskCompletionSource();
|
||||
var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) =>
|
||||
{
|
||||
if (i == 0) started.TrySetResult();
|
||||
await Task.Delay(10, ct);
|
||||
});
|
||||
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
||||
await engine.StartAsync();
|
||||
var job = await engine.EnqueueAsync(JobKind.Scan, []);
|
||||
await started.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
||||
await engine.CancelAsync(job.Id);
|
||||
await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(0, handler.Completed);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task PauseAndResumeContinueFromCheckpoint()
|
||||
{
|
||||
await using var db = await OpenAsync();
|
||||
var gate = new SemaphoreSlim(0);
|
||||
var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) =>
|
||||
{
|
||||
if (i == 10) await gate.WaitAsync(ct);
|
||||
});
|
||||
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
||||
await engine.StartAsync();
|
||||
var job = await engine.EnqueueAsync(JobKind.Proxies, []);
|
||||
var until = DateTime.UtcNow.AddSeconds(10);
|
||||
while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20);
|
||||
await engine.PauseAsync(job.Id);
|
||||
var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5));
|
||||
Assert.Equal("10", paused.Checkpoint);
|
||||
gate.Release(10);
|
||||
await engine.ResumeAsync(job.Id);
|
||||
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
||||
Assert.Equal("10", handler.SeenCheckpoints[^1]);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task CrashedJobsResumeAfterRestart()
|
||||
{
|
||||
await using var db = await OpenAsync();
|
||||
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
||||
await db.Jobs.EnqueueAsync(job);
|
||||
var claimed = await db.Jobs.ClaimNextAsync("t");
|
||||
Assert.NotNull(claimed);
|
||||
await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null);
|
||||
// Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it.
|
||||
var handler = new CountingHandler(JobKind.Thumbs, 12);
|
||||
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
||||
await engine.StartAsync();
|
||||
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
||||
Assert.Equal("7", handler.SeenCheckpoints.Single());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task HandlerFailureMarksJobFailedWithMessage()
|
||||
{
|
||||
await using var db = await OpenAsync();
|
||||
var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask);
|
||||
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
||||
await engine.StartAsync();
|
||||
var job = await engine.EnqueueAsync(JobKind.Render, []);
|
||||
var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10));
|
||||
Assert.Equal("encoder lost", failed.Error);
|
||||
Assert.Contains("encoder lost", failed.LogTail);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user