using Titano.Core.Model; using Titano.Data.Jobs; using Titano.Testing; namespace Titano.Data.Tests.Jobs; [Category("jobs")] public sealed class JobEngineTests(TestContext context) { private sealed class CountingHandler(JobKind kind, int steps, Func? onStep = null) : IJobHandler { public JobKind Kind => kind; public List SeenCheckpoints { get; } = []; public int Completed; public async Task RunAsync(JobContext c) { lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint); var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0; for (var i = start; i < steps; i++) { c.Cancellation.ThrowIfCancellationRequested(); if (onStep is not null) await onStep(i, c.Cancellation); await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps); } Interlocked.Increment(ref Completed); } } private async Task OpenAsync() => await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation); private static async Task WaitForStateAsync(TitanoDatabase db, Guid id, JobState state, TimeSpan timeout) { var until = DateTime.UtcNow + timeout; while (DateTime.UtcNow < until) { var job = await db.Jobs.GetAsync(id); if (job?.State == state) return job; await Task.Delay(20); } Assert.Fail($"Job {id} did not reach {state}"); return null!; } [Test] public async Task RunsQueuedJobsToCompletion() { await using var db = await OpenAsync(); var handler = new CountingHandler(JobKind.Verify, 5); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) }); var progress = 0; engine.Progress += _ => Interlocked.Increment(ref progress); await engine.StartAsync(); var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A"); var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B"); var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10)); await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.Equal(2, handler.Completed); Assert.Equal(5L, doneA.ProgressDone); Assert.Greater(progress, 0); } [Test] public async Task CancelStopsARunningJob() { await using var db = await OpenAsync(); var started = new TaskCompletionSource(); var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) => { if (i == 0) started.TrySetResult(); await Task.Delay(10, ct); }); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); await engine.StartAsync(); var job = await engine.EnqueueAsync(JobKind.Scan, []); await started.Task.WaitAsync(TimeSpan.FromSeconds(10)); await engine.CancelAsync(job.Id); await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5)); Assert.Equal(0, handler.Completed); } [Test] public async Task PauseAndResumeContinueFromCheckpoint() { await using var db = await OpenAsync(); var gate = new SemaphoreSlim(0); var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) => { if (i == 10) await gate.WaitAsync(ct); }); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) }); await engine.StartAsync(); var job = await engine.EnqueueAsync(JobKind.Proxies, []); var until = DateTime.UtcNow.AddSeconds(10); while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20); await engine.PauseAsync(job.Id); var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5)); Assert.Equal("10", paused.Checkpoint); gate.Release(10); await engine.ResumeAsync(job.Id); await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.Equal("10", handler.SeenCheckpoints[^1]); } [Test] public async Task CrashedJobsResumeAfterRestart() { await using var db = await OpenAsync(); var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow }; await db.Jobs.EnqueueAsync(job); var claimed = await db.Jobs.ClaimNextAsync("t"); Assert.NotNull(claimed); await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null); // Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it. var handler = new CountingHandler(JobKind.Thumbs, 12); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); await engine.StartAsync(); await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10)); Assert.Equal("7", handler.SeenCheckpoints.Single()); } [Test] public async Task HandlerFailureMarksJobFailedWithMessage() { await using var db = await OpenAsync(); var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); await engine.StartAsync(); var job = await engine.EnqueueAsync(JobKind.Render, []); var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10)); Assert.Equal("encoder lost", failed.Error); Assert.Contains("encoder lost", failed.LogTail); } }