Files
Chronos/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs
T
Claude 3e146c9f9f feat(settings): make every Settings row real; archive settings, index backups and restore
Settings (UI-01, UI-02, UI-03, UI-09, RF-4):
- New Import, Analysis and Formats pages, in the order of Settings.html. Import holds
  the defaults of new cards (session rules, subfolders, verification, duplicates,
  after-import actions); Analysis the deep-analysis width cap, night and star-field
  thresholds and deep analysis after import or scan; Formats the decode tier of every
  format on this PC with a notice and Store install link per missing codec extension.
- Archive gains thumbnail size and rescan interval; its templates, proxy size and
  thumbnail size are the open archive's own (archive.json) and new archives' defaults.
- Rendering quality, parallelism and background priority now drive the jobs: quality
  maps to preparation size, read-ahead and default flow quality; parallelism sets the
  engine's workers live and caps per-job threads; background priority raises the UI
  thread while Titano is in front.
- Advanced gains Restore (daily backups, dry-run summary, destructive confirmation,
  staged for the next start) and Reset all.
- The naming preview's sample title is a catalog string.

Engine (ENG-03, ENG-04, ENG-06):
- ArchiveService.GetSettingsAsync/UpdateSettingsAsync rewrite archive.json atomically
  under a lease, then the index row; the server endpoint and the desktop use them, and
  the desktop creates archives with Settings › Archive. ThumbnailPolicy in archive.json
  sizes the thumbnails.
- DatabaseBackups: daily VACUUM INTO copies kept x3 on the desktop and on a server on
  SQLite (DatabaseBackupService), staged restore checked and preceded by a fresh copy.
- Maintenance vacuums the SQLite index when 10 % or more is free (dry run reports it).
- JobEngine.SetWorkers; ProcessingOptions.Tune applies host tuning per job.

Archive setup toasts say why in a localized sentence (RF-1).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 10:06:34 +00:00

327 lines
16 KiB
C#

using Titano.Core.Model;
using Titano.Data.Jobs;
using Titano.Testing;
namespace Titano.Data.Tests.Jobs;
[Category("jobs")]
public sealed class JobEngineTests(TestContext context)
{
private sealed class CountingHandler(JobKind kind, int steps, Func<int, CancellationToken, Task>? onStep = null) : IJobHandler
{
public JobKind Kind => kind;
public List<string?> SeenCheckpoints { get; } = [];
public int Completed;
public async Task RunAsync(JobContext c)
{
lock (SeenCheckpoints) SeenCheckpoints.Add(c.Checkpoint);
var start = c.Checkpoint is { } cp ? int.Parse(cp, System.Globalization.CultureInfo.InvariantCulture) : 0;
for (var i = start; i < steps; i++)
{
c.Cancellation.ThrowIfCancellationRequested();
if (onStep is not null) await onStep(i, c.Cancellation);
await c.CheckpointAsync((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), i + 1, steps);
}
Interlocked.Increment(ref Completed);
}
}
/// <summary>Takes only jobs whose parameters say they are its own.</summary>
private sealed class SelectiveHandler(JobKind kind) : ISelectiveJobHandler
{
public const string Mine = """{"mine":true}""";
public JobKind Kind => kind;
public ValueTask<bool> AcceptsAsync(Job job, CancellationToken cancellationToken) => ValueTask.FromResult(job.ParametersJson == Mine);
public Task RunAsync(JobContext c) => Task.CompletedTask;
}
/// <summary>Cannot judge jobs without parameters (as a render handler with a malformed preset) and fails them when run.</summary>
private sealed class FragileHandler(JobKind kind) : ISelectiveJobHandler
{
public JobKind Kind => kind;
public ValueTask<bool> AcceptsAsync(Job job, CancellationToken cancellationToken) =>
job.ParametersJson?.Length > 0 ? ValueTask.FromResult(true) : throw new ArgumentNullException(nameof(job));
public Task RunAsync(JobContext c) =>
c.Job.ParametersJson is null ? throw new InvalidDataException("no parameters") : Task.CompletedTask;
}
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
{
public JobKind Kind => kind;
public Task RunAsync(JobContext context) => run(context);
}
private async Task<TitanoDatabase> OpenAsync() =>
await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
private static async Task<Job> WaitForStateAsync(TitanoDatabase db, Guid id, JobState state, TimeSpan timeout)
{
var until = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < until)
{
var job = await db.Jobs.GetAsync(id);
if (job?.State == state) return job;
await Task.Delay(20);
}
Assert.Fail($"Job {id} did not reach {state}");
return null!;
}
[Test]
public async Task RunsQueuedJobsToCompletion()
{
await using var db = await OpenAsync();
var handler = new CountingHandler(JobKind.Verify, 5);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
var progress = 0;
engine.Progress += _ => Interlocked.Increment(ref progress);
await engine.StartAsync();
var a = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "A");
var b = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], "B");
var doneA = await WaitForStateAsync(db, a.Id, JobState.Done, TimeSpan.FromSeconds(10));
await WaitForStateAsync(db, b.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal(2, handler.Completed);
Assert.Equal(5L, doneA.ProgressDone);
Assert.Greater(progress, 0);
}
[Test]
public async Task CancelStopsARunningJob()
{
await using var db = await OpenAsync();
var started = new TaskCompletionSource();
var handler = new CountingHandler(JobKind.Scan, 1000, async (i, ct) =>
{
if (i == 0) started.TrySetResult();
await Task.Delay(10, ct);
});
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
await engine.StartAsync();
var job = await engine.EnqueueAsync(JobKind.Scan, []);
await started.Task.WaitAsync(TimeSpan.FromSeconds(10));
await engine.CancelAsync(job.Id);
await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5));
Assert.Equal(0, handler.Completed);
}
[Test]
public async Task PauseAndResumeContinueFromCheckpoint()
{
await using var db = await OpenAsync();
var gate = new SemaphoreSlim(0);
var handler = new CountingHandler(JobKind.Proxies, 20, async (i, ct) =>
{
if (i == 10) await gate.WaitAsync(ct);
});
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
await engine.StartAsync();
var job = await engine.EnqueueAsync(JobKind.Proxies, []);
var until = DateTime.UtcNow.AddSeconds(10);
while ((await db.Jobs.GetAsync(job.Id))!.Checkpoint != "10" && DateTime.UtcNow < until) await Task.Delay(20);
await engine.PauseAsync(job.Id);
var paused = await WaitForStateAsync(db, job.Id, JobState.Paused, TimeSpan.FromSeconds(5));
Assert.Equal("10", paused.Checkpoint);
gate.Release(10);
await engine.ResumeAsync(job.Id);
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal("10", handler.SeenCheckpoints[^1]);
}
[Test]
public async Task CrashedJobsResumeAfterRestart()
{
await using var db = await OpenAsync();
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
await db.Jobs.EnqueueAsync(job);
var claimed = await db.Jobs.ClaimNextAsync("t");
Assert.NotNull(claimed);
await db.Jobs.UpdateProgressAsync(job.Id, 7, 12, null, null, "7", null);
// Simulated crash: the job stays "running" for worker t. A new engine with the same id requeues it.
var handler = new CountingHandler(JobKind.Thumbs, 12);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
await engine.StartAsync();
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal("7", handler.SeenCheckpoints.Single());
}
[Test]
public async Task JobsOfNumberedWorkersResumeAfterRestart()
{
await using var db = await OpenAsync();
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
await db.Jobs.EnqueueAsync(job);
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc#1"));
var other = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Thumbs, State = JobState.Queued, Created = DateTimeOffset.UtcNow };
await db.Jobs.EnqueueAsync(other);
Assert.NotNull(await db.Jobs.ClaimNextAsync("pc2#0"));
// Restarting node "pc" (two workers: pc#0, pc#1) requeues pc#1's job, never another node's.
var handler = new CountingHandler(JobKind.Thumbs, 3);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 2, WorkerId = "pc" });
await engine.StartAsync();
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(other.Id))!.State);
}
[Test]
public async Task OnlyKindsWithAHandlerAreClaimed()
{
await using var db = await OpenAsync();
await using (var idle = new JobEngine(db, [], new JobEngineOptions { Workers = 1, WorkerId = "idle" }))
{
await idle.StartAsync();
var render = await idle.EnqueueAsync(JobKind.Render, []);
await Task.Delay(300);
Assert.Equal(JobState.Queued, (await db.Jobs.GetAsync(render.Id))!.State);
}
var handler = new CountingHandler(JobKind.Thumbs, 1);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Kinds = [JobKind.Thumbs, JobKind.Render] });
await engine.StartAsync();
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
var pending = (await db.Jobs.ListAsync(JobState.Queued)).Items;
Assert.Count(1, pending);
Assert.Equal(JobKind.Render, pending[0].Kind);
}
[Test]
public async Task SelectiveHandlersLeaveDeclinedJobsQueuedWithoutHoldingUpOthers()
{
await using var db = await OpenAsync();
await using var engine = new JobEngine(db, [new SelectiveHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
var declined = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: """{"mine":false}""", priority: JobPriority.User);
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.Background);
var accepted = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: SelectiveHandler.Mine, priority: JobPriority.User);
await engine.StartAsync();
var render = await WaitForStateAsync(db, accepted.Id, JobState.Done, TimeSpan.FromSeconds(10));
var other = await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.True(render.Started <= other.Started, "claim order: priority first, across selective and other kinds");
await Task.Delay(200);
var waiting = (await db.Jobs.GetAsync(declined.Id))!;
Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId));
}
[Test]
public async Task ASelectiveHandlerThatThrowsDoesNotStopTheWorkers()
{
await using var db = await OpenAsync();
await using var engine = new JobEngine(db, [new FragileHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
new JobEngineOptions { Workers = 2, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
var broken = await engine.EnqueueAsync(JobKind.Render, [], priority: JobPriority.User);
await engine.StartAsync();
var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed, TimeSpan.FromSeconds(10));
Assert.Equal("no parameters", failed.Error);
// The workers are still there for every kind.
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, []);
var render = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: "{}");
await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
await WaitForStateAsync(db, render.Id, JobState.Done, TimeSpan.FromSeconds(10));
}
[Test]
public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns()
{
await using var db = await OpenAsync();
var target = Guid.CreateVersion7();
Job? thumbs = null, light = null;
var scan = new CallbackHandler(JobKind.Scan, async c =>
{
thumbs = await c.EnqueueFollowUpAsync(JobKind.Thumbs, [target], "Roof", null, JobPriority.Background);
light = await c.EnqueueFollowUpAsync(JobKind.LightAnalyse, [target], "Roof", null, JobPriority.Background);
});
await using var engine = new JobEngine(db, [scan, new CountingHandler(JobKind.Thumbs, 1)],
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync();
var user = Guid.CreateVersion7();
var job = await engine.EnqueueAsync(JobKind.Scan, [], createdBy: user);
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Null(light);
var followUp = await WaitForStateAsync(db, Assert.NotNull(thumbs).Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal((user, JobPriority.Background, "Roof"), (followUp.CreatedBy!.Value, followUp.Priority, followUp.TargetLabel!));
Assert.SequenceEqual([target], followUp.TargetIds);
Assert.Empty((await db.Jobs.ListAsync(null, JobKind.LightAnalyse)).Items);
}
[Test]
public async Task TheNumberOfWorkersChangesWhileTheEngineRuns()
{
await using var db = await OpenAsync();
var running = 0;
var started = 0;
var release = new SemaphoreSlim(0);
var handler = new CallbackHandler(JobKind.Thumbs, async c =>
{
Interlocked.Increment(ref running);
Interlocked.Increment(ref started);
try
{
await release.WaitAsync(c.Cancellation);
}
finally
{
Interlocked.Decrement(ref running);
}
});
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync();
var jobs = new List<Job>();
for (var i = 0; i < 6; i++) jobs.Add(await engine.EnqueueAsync(JobKind.Thumbs, []));
async Task UntilAsync(Func<bool> condition, string what)
{
var until = DateTime.UtcNow.AddSeconds(10);
while (!condition())
{
Assert.True(DateTime.UtcNow < until, what);
await Task.Delay(20);
}
}
// One worker: one job at a time.
await UntilAsync(() => Volatile.Read(ref running) == 1, "the first job runs");
await Task.Delay(300);
Assert.Equal(1, Volatile.Read(ref running));
// Three workers: two more jobs start at once, and no fourth.
engine.SetWorkers(3);
Assert.Equal(3, engine.Workers);
await UntilAsync(() => Volatile.Read(ref running) == 3, "three jobs run together");
await Task.Delay(300);
Assert.Equal(3, Volatile.Read(ref running));
// Back to one: the running jobs finish undisturbed, then a single worker carries on.
engine.SetWorkers(1);
release.Release(3);
await UntilAsync(() => Volatile.Read(ref started) == 4 && Volatile.Read(ref running) == 1, "one job after the three");
await Task.Delay(300);
Assert.Equal((4, 1), (Volatile.Read(ref started), Volatile.Read(ref running)));
release.Release(3);
foreach (var job in jobs) await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Throws<ArgumentOutOfRangeException>(() => engine.SetWorkers(0));
}
[Test]
public async Task HandlerFailureMarksJobFailedWithMessage()
{
await using var db = await OpenAsync();
var handler = new CountingHandler(JobKind.Render, 3, (i, _) => i == 1 ? throw new InvalidOperationException("encoder lost") : Task.CompletedTask);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
await engine.StartAsync();
var job = await engine.EnqueueAsync(JobKind.Render, []);
var failed = await WaitForStateAsync(db, job.Id, JobState.Failed, TimeSpan.FromSeconds(10));
Assert.Equal("encoder lost", failed.Error);
Assert.Contains("encoder lost", failed.LogTail);
}
}