CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
The owner renamed the product and moved the repository to the self-hosted Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control, and asked for Actions and a first automatic release with the server package and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md). - Chronos everywhere: a byte-exact rename of contents and paths, then a per-language review of the forms the old name's ending had dictated. - Gitea instead of GitHub: the update check, licence, issue and support links read ChronosReleasesUrl; the update check takes the highest stable version, offers a pre-release build its final release and reports a repository it cannot see as a failed check. EULA, security policy and Gitea issue forms live at the repository root. - Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release, nightly image and performance gates in the .NET SDK container with git checkout and the system SQLite. The release cross-compiles the Windows programs, checks their icon, version and DPI manifest, packs installers and zips, publishes the Gitea release and pushes the server image (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows runner (WINDOWS_RUNNER). - New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the docker-exec script, which Gitea's service containers do not allow; the throw-away server runs without durability) and pe-check. - Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR, 3 on the shared runner; the weekly performance gates keep theirs). BREAKING CHANGE: programs, settings, data folders, file types, environment variables and the Docker image use the new name; nothing is migrated from Titano, which never had a release. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
450 lines
23 KiB
C#
450 lines
23 KiB
C#
using Chronos.Core.Model;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Jobs;
|
|
|
|
[Category("jobs")]
|
|
public sealed class JobEngineTests(TestContext context)
|
|
{
|
|
private sealed class CountingHandler(JobKind kind, int steps, Func<int, CancellationToken, Task>? onStep = null) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
public List<string?> SeenCheckpoints { get; } = [];
|
|
public int Completed;
|
|
|
|
public async Task RunAsync(JobContext c)
|
|
{
|
|
lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint);
|
|
var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0;
|
|
for (var i = start; i < steps; i++)
|
|
{
|
|
c.Cancellation.ThrowIfCancellationRequested();
|
|
if (onStep is not null) await onStep(i, c.Cancellation);
|
|
await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps);
|
|
}
|
|
Interlocked.Increment(ref Completed);
|
|
}
|
|
}
|
|
|
|
/// <summary>Takes only jobs whose parameters say they are its own.</summary>
|
|
private sealed class SelectiveHandler(JobKind kind) : ISelectiveJobHandler
|
|
{
|
|
public const string Mine = """{"mine":true}""";
|
|
|
|
public JobKind Kind => kind;
|
|
|
|
public ValueTask<bool> AcceptsAsync(Job job, CancellationToken cancellationToken) => ValueTask.FromResult(job.ParametersJson == Mine);
|
|
|
|
public Task RunAsync(JobContext c) => Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>Cannot judge jobs without parameters (as a render handler with a malformed preset) and fails them when run.</summary>
|
|
private sealed class FragileHandler(JobKind kind) : ISelectiveJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
|
|
public ValueTask<bool> AcceptsAsync(Job job, CancellationToken cancellationToken) =>
|
|
job.ParametersJson?.Length > 0 ? ValueTask.FromResult(true) : throw new ArgumentNullException(nameof(job));
|
|
|
|
public Task RunAsync(JobContext c) =>
|
|
c.Job.ParametersJson is null ? throw new InvalidDataException("no parameters") : Task.CompletedTask;
|
|
}
|
|
|
|
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
|
|
public Task RunAsync(JobContext context) => run(context);
|
|
}
|
|
|
|
private async Task<ChronosDatabase> OpenAsync() =>
|
|
await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
|
|
|
|
private static async Task<Job> WaitForStateAsync(ChronosDatabase db, Guid id, JobState state, TimeSpan timeout)
|
|
{
|
|
var until = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < until)
|
|
{
|
|
var job = await db.Jobs.GetAsync(id);
|
|
if (job?.State == state) return job;
|
|
await Task.Delay(20);
|
|
}
|
|
Assert.Fail($"Job {id} did not reach {state}");
|
|
return null!;
|
|
}
|
|
|
|
[Test]
|
|
public async Task RunsQueuedJobsToCompletion()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var handler = new CountingHandler(JobKind.Verify, 5);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
|
var progress = 0;
|
|
engine.Progress += _ => Interlocked.Increment(ref progress);
|
|
await engine.StartAsync();
|
|
var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A");
|
|
var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B");
|
|
var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(2, handler.Completed);
|
|
Assert.Equal(5L, doneA.ProgressDone);
|
|
Assert.Greater(progress, 0);
|
|
}
|
|
|
|
[Test]
|
|
public async Task CancelStopsARunningJob()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var started = new TaskCompletionSource();
|
|
var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) =>
|
|
{
|
|
if (i == 0) started.TrySetResult();
|
|
await Task.Delay(10, ct);
|
|
});
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await started.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
await engine.CancelAsync(job.Id);
|
|
await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5));
|
|
Assert.Equal(0, handler.Completed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task PauseAndResumeContinueFromCheckpoint()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var gate = new SemaphoreSlim(0);
|
|
var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) =>
|
|
{
|
|
if (i == 10) await gate.WaitAsync(ct);
|
|
});
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Proxies, []);
|
|
var until = DateTime.UtcNow.AddSeconds(10);
|
|
while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20);
|
|
await engine.PauseAsync(job.Id);
|
|
var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5));
|
|
Assert.Equal("10", paused.Checkpoint);
|
|
gate.Release(10);
|
|
await engine.ResumeAsync(job.Id);
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("10", handler.SeenCheckpoints[^1]);
|
|
}
|
|
|
|
[Test]
|
|
public async Task CrashedJobsResumeAfterRestart()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(job);
|
|
var claimed = await db.Jobs.ClaimNextAsync("t");
|
|
Assert.NotNull(claimed);
|
|
await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null);
|
|
// Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it.
|
|
var handler = new CountingHandler(JobKind.Thumbs, 12);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("7", handler.SeenCheckpoints.Single());
|
|
}
|
|
|
|
[Test]
|
|
public async Task JobsOfNumberedWorkersResumeAfterRestart()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(job);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc#1"));
|
|
var other = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(other);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc2#0"));
|
|
// Restarting node "pc" (two workers: pc#0, pc#1) requeues pc#1's job, never another node's.
|
|
var handler = new CountingHandler(JobKind.Thumbs, 3);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "pc" });
|
|
await engine.StartAsync();
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(other.Id))!.State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OnlyKindsWithAHandlerAreClaimed()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
await using (var idle = new JobEngine(db, [], new JobEngineOptions { Workers = 1, WorkerId = "idle" }))
|
|
{
|
|
await idle.StartAsync();
|
|
var render = await idle.EnqueueAsync(JobKind.Render, []);
|
|
await Task.Delay(300);
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(render.Id))!.State);
|
|
}
|
|
var handler = new CountingHandler(JobKind.Thumbs, 1);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Kinds = [JobKind.Thumbs, JobKind.Render] });
|
|
await engine.StartAsync();
|
|
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
|
|
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
var pending = (await db.Jobs.ListAsync(JobState.Queued)).Items;
|
|
Assert.Count(1, pending);
|
|
Assert.Equal(JobKind.Render, pending[0].Kind);
|
|
}
|
|
|
|
[Test]
|
|
public async Task SelectiveHandlersLeaveDeclinedJobsQueuedWithoutHoldingUpOthers()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
await using var engine = new JobEngine(db, [new SelectiveHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
|
|
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
var declined = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: """{"mine":false}""", priority: JobPriority.User);
|
|
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.Background);
|
|
var accepted = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: SelectiveHandler.Mine, priority: JobPriority.User);
|
|
await engine.StartAsync();
|
|
|
|
var render = await WaitForStateAsync(db, accepted.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
var other = await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.True(render.Started <= other.Started, "claim order: priority first, across selective and other kinds");
|
|
await Task.Delay(200);
|
|
var waiting = (await db.Jobs.GetAsync(declined.Id))!;
|
|
Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASelectiveHandlerThatThrowsDoesNotStopTheWorkers()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
await using var engine = new JobEngine(db, [new FragileHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
|
|
new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
var broken = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
|
|
await engine.StartAsync();
|
|
|
|
var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("no parameters", failed.Error);
|
|
// The workers are still there for every kind.
|
|
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
|
|
var render = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: "{}");
|
|
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, render.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
[Test]
|
|
public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var target = Guid.CreateVersion7();
|
|
Job? thumbs = null, light = null;
|
|
var scan = new CallbackHandler(JobKind.Scan, async c =>
|
|
{
|
|
thumbs = await c.EnqueueFollowUpAsync(JobKind.Thumbs, [target], "Roof", null, JobPriority.Background);
|
|
light = await c.EnqueueFollowUpAsync(JobKind.LightAnalyse, [target], "Roof", null, JobPriority.Background);
|
|
});
|
|
await using var engine = new JobEngine(db, [scan, new CountingHandler(JobKind.Thumbs, 1)],
|
|
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
await engine.StartAsync();
|
|
var user = Guid.CreateVersion7();
|
|
var job = await engine.EnqueueAsync(JobKind.Scan, [], createdBy: user);
|
|
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
|
|
Assert.Null(light);
|
|
var followUp = await WaitForStateAsync(db, Assert.NotNull(thumbs).Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal((user, JobPriority.Background, "Roof"), (followUp.CreatedBy!.Value, followUp.Priority, followUp.TargetLabel!));
|
|
Assert.SequenceEqual([target], followUp.TargetIds);
|
|
Assert.Empty((await db.Jobs.ListAsync(null, JobKind.LightAnalyse)).Items);
|
|
}
|
|
|
|
/// <summary>Waits until <paramref name="condition"/> holds, failing with <paramref name="what"/> after ten seconds.</summary>
|
|
private static async Task UntilAsync(Func<bool> condition, string what)
|
|
{
|
|
var until = DateTime.UtcNow.AddSeconds(10);
|
|
while (!condition())
|
|
{
|
|
Assert.True(DateTime.UtcNow < until, what);
|
|
await Task.Delay(20);
|
|
}
|
|
}
|
|
|
|
/// <summary>A handler whose jobs run until released, counting those running and started, by kind.</summary>
|
|
private sealed class GatedHandler(JobKind kind, SemaphoreSlim release) : IJobHandler
|
|
{
|
|
public int Running;
|
|
public int Started;
|
|
|
|
public JobKind Kind => kind;
|
|
|
|
public async Task RunAsync(JobContext c)
|
|
{
|
|
Interlocked.Increment(ref Running);
|
|
Interlocked.Increment(ref Started);
|
|
try
|
|
{
|
|
await release.WaitAsync(c.Cancellation);
|
|
}
|
|
finally
|
|
{
|
|
Interlocked.Decrement(ref Running);
|
|
}
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task JobsRunOneAtATimeAndPausingStartsTheNext()
|
|
{
|
|
// Owner review 1 (R2): one job at a time by default; pausing the running job lets the next one start.
|
|
await using var db = await OpenAsync();
|
|
var release = new SemaphoreSlim(0);
|
|
var handler = new GatedHandler(JobKind.Thumbs, release);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
Assert.Equal(1, engine.Options.Workers);
|
|
await engine.StartAsync();
|
|
var jobs = new List<Job>();
|
|
for (var i = 0; i < 3; i++) jobs.Add(await engine.EnqueueAsync(JobKind.Thumbs, []));
|
|
|
|
await UntilAsync(() => Volatile.Read(ref handler.Running) == 1, "the first job runs");
|
|
await Task.Delay(300);
|
|
Assert.Equal((1, 1), (Volatile.Read(ref handler.Running), Volatile.Read(ref handler.Started)));
|
|
Assert.Count(1, engine.RunningJobIds);
|
|
var first = engine.RunningJobIds.First();
|
|
Assert.Equal("pc", (await db.Jobs.GetAsync(first))!.NodeId);
|
|
|
|
await engine.PauseAsync(first);
|
|
await UntilAsync(() => Volatile.Read(ref handler.Started) == 2 && Volatile.Read(ref handler.Running) == 1, "the next job starts");
|
|
Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(first))!.State);
|
|
|
|
release.Release(10);
|
|
foreach (var job in jobs.Where(j => j.Id != first)) await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(first))!.State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task WithoutThePriorityQueueARunningJobIsNeverInterrupted()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var release = new SemaphoreSlim(0);
|
|
var scan = new GatedHandler(JobKind.Scan, release);
|
|
var render = new GatedHandler(JobKind.Render, release);
|
|
await using var engine = new JobEngine(db, [scan, render], new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
await engine.StartAsync();
|
|
var scanning = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await UntilAsync(() => Volatile.Read(ref scan.Running) == 1, "the scan runs");
|
|
var rendering = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
|
|
await Task.Delay(400);
|
|
Assert.Equal((1, 0), (Volatile.Read(ref scan.Running), Volatile.Read(ref render.Started)));
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(rendering.Id))!.State);
|
|
|
|
release.Release(10);
|
|
await WaitForStateAsync(db, scanning.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, rendering.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
Assert.Equal(1, Volatile.Read(ref scan.Started));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AHigherJobPausesTheRunningOneWhichResumesFromItsCheckpoint()
|
|
{
|
|
// Owner review 1 (R2): with render at the highest level, starting a render while a scan runs pauses the scan
|
|
// without asking, renders, and resumes the scan by itself where it was.
|
|
await using var db = await OpenAsync();
|
|
var checkpointed = new TaskCompletionSource();
|
|
var go = new SemaphoreSlim(0);
|
|
var scan = new CountingHandler(JobKind.Scan, 4, async (i, ct) =>
|
|
{
|
|
if (i < 2) return;
|
|
checkpointed.TrySetResult();
|
|
await go.WaitAsync(ct);
|
|
});
|
|
var renderRelease = new SemaphoreSlim(0);
|
|
var render = new GatedHandler(JobKind.Render, renderRelease);
|
|
var queue = new JobPriorityQueue { Enabled = true };
|
|
Assert.Equal((JobKindPriority.High, JobKindPriority.Normal), (queue.LevelOf(JobKind.Render), queue.LevelOf(JobKind.Scan)));
|
|
await using var engine = new JobEngine(db, [scan, render],
|
|
new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50), PriorityQueue = () => queue });
|
|
await engine.StartAsync();
|
|
var scanning = await engine.EnqueueAsync(JobKind.Scan, []);
|
|
await checkpointed.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
|
|
var rendering = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
|
|
await UntilAsync(() => Volatile.Read(ref render.Running) == 1, "the render starts at once");
|
|
var paused = await db.Jobs.GetAsync(scanning.Id);
|
|
Assert.Equal((JobState.Queued, "2", (string?)null), (paused!.State, paused.Checkpoint, paused.NodeId));
|
|
Assert.Contains(Chronos.Core.Localization.Loc.L("Jobs.Kind.Render"), paused.LogTail);
|
|
|
|
// The scan waits while the render runs, then resumes from its checkpoint without being asked.
|
|
go.Release(10);
|
|
await Task.Delay(300);
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(scanning.Id))!.State);
|
|
renderRelease.Release();
|
|
await WaitForStateAsync(db, rendering.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
await WaitForStateAsync(db, scanning.Id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
lock (scan.SeenCheckpoints) Assert.SequenceEqual([null, "2"], scan.SeenCheckpoints);
|
|
Assert.Equal(1, scan.Completed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OnlyAHigherLevelInterruptsAndNeverTheJobsOwnFollowUps()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var release = new SemaphoreSlim(0);
|
|
var queued = new TaskCompletionSource<Job?>();
|
|
// The light analysis queues a deep analysis of the same sequence, which the user put higher.
|
|
var light = new CallbackHandler(JobKind.LightAnalyse, async c =>
|
|
{
|
|
queued.TrySetResult(await c.EnqueueFollowUpAsync(JobKind.DeepAnalyse, [Guid.NewGuid()], "Roof", null, JobPriority.Background));
|
|
await release.WaitAsync(c.Cancellation);
|
|
});
|
|
var deep = new GatedHandler(JobKind.DeepAnalyse, release);
|
|
var thumbs = new GatedHandler(JobKind.Thumbs, release);
|
|
var queue = new JobPriorityQueue
|
|
{
|
|
Enabled = true,
|
|
Levels = new Dictionary<JobKind, JobKindPriority> { [JobKind.DeepAnalyse] = JobKindPriority.High, [JobKind.Thumbs] = JobKindPriority.Normal },
|
|
};
|
|
await using var engine = new JobEngine(db, [light, deep, thumbs],
|
|
new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50), PriorityQueue = () => queue });
|
|
await engine.StartAsync();
|
|
var analysing = await engine.EnqueueAsync(JobKind.LightAnalyse, []);
|
|
var followUp = Assert.NotNull(await queued.Task.WaitAsync(TimeSpan.FromSeconds(10)));
|
|
// A job of the same level waits too.
|
|
var drawing = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.User);
|
|
await Task.Delay(400);
|
|
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(analysing.Id))!.State);
|
|
Assert.Equal((0, 0), (Volatile.Read(ref deep.Started), Volatile.Read(ref thumbs.Started)));
|
|
Assert.False(await engine.ReviewAsync(context.Cancellation));
|
|
|
|
// Another deep analysis, not its own, interrupts it.
|
|
var other = await engine.EnqueueAsync(JobKind.DeepAnalyse, [Guid.NewGuid()], priority: JobPriority.User);
|
|
await UntilAsync(() => Volatile.Read(ref deep.Running) == 1, "a higher job runs");
|
|
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(analysing.Id))!.State);
|
|
release.Release(20);
|
|
foreach (var id in new[] { analysing.Id, followUp.Id, drawing.Id, other.Id }) await WaitForStateAsync(db, id, JobState.Done, TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
[Test]
|
|
public async Task RequeueingAYieldedJobLeavesPausesAndOtherNodesAlone()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
|
|
await db.Jobs.EnqueueAsync(job);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc"));
|
|
Assert.False(await db.Jobs.RequeueAsync(job.Id, "other"));
|
|
await db.Jobs.SetStateAsync(job.Id, JobState.Paused);
|
|
Assert.False(await db.Jobs.RequeueAsync(job.Id, "pc"));
|
|
Assert.Equal(JobState.Paused, (await db.Jobs.GetAsync(job.Id))!.State);
|
|
await db.Jobs.SetStateAsync(job.Id, JobState.Queued);
|
|
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc"));
|
|
Assert.True(await db.Jobs.RequeueAsync(job.Id, "pc"));
|
|
Assert.Equal((JobState.Queued, (string?)null), ((await db.Jobs.GetAsync(job.Id))!.State, (await db.Jobs.GetAsync(job.Id))!.NodeId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task HandlerFailureMarksJobFailedWithMessage()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Render, []);
|
|
var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10));
|
|
Assert.Equal("encoder lost", failed.Error);
|
|
Assert.Contains("encoder lost", failed.LogTail);
|
|
}
|
|
}
|