using Chronos.Core.Model; using Chronos.Data.Jobs; using Chronos.Testing; namespace Chronos.Data.Tests.Jobs; /// /// How a running job's progress goes out (owner review 5, #65): at most twice a second, at once for the run's first report, /// a new step and a completed count, and with the persist that falls due when the last report was dropped; saved at most /// every checkpoint interval and at each checkpoint, with its detail. /// [Category("jobs")] public sealed class JobContextProgressTests(TestContext context) { private static readonly TimeSpan Tick = TimeSpan.FromMilliseconds(10); [Test] public async Task AHundredReportsInASecondGoOutAtMostThreeTimes() { await using var run = await StartAsync(); for (var i = 1; i <= 100; i++) { await run.Context.ReportAsync(i, 1000); run.Time.Advance(Tick); } Assert.True(run.Published.Count <= 3, $"{run.Published.Count} publishes"); // The first at once, then one per half second. Assert.SequenceEqual([1L, 51L], run.Published.Select(p => p.Done)); } [Test] public async Task ANewStepAndACompletedCountGoOutAtOnce() { await using var run = await StartAsync(); var index = new JobDetail(JobStep.Index, "2024/Harbour sunset", 1, 5, 1, 2, JobPartUnit.Folder); await run.Context.ReportAsync(1, 10, new JobDetail(JobStep.List)); run.Time.Advance(Tick); await run.Context.ReportAsync(2, 10, new JobDetail(JobStep.List)); run.Time.Advance(Tick); await run.Context.ReportAsync(3, 10, index); run.Time.Advance(Tick); await run.Context.ReportItemAsync(4, 10, 2, 5); run.Time.Advance(Tick); await run.Context.ReportAsync(10, 10); run.Time.Advance(Tick); await run.Context.ReportAsync(10, 10); Assert.SequenceEqual([1L, 3L, 10L], run.Published.Select(p => p.Done)); Assert.Equal(index, run.Published[1].Detail); // A report without a detail keeps the last one, with the item counts reported since. Assert.Equal(index with { ItemDone = 2 }, run.Published[2].Detail); Assert.Equal(index with { ItemDone = 2 }, run.Context.Detail); } [Test] public async Task ADroppedReportGoesOutWithThePersistThatFallsDue() { await using var run = await StartAsync(); await run.Context.ReportAsync(1, 100); run.Time.Advance(TimeSpan.FromSeconds(4.8)); await run.Context.ReportAsync(2, 100); Assert.Equal(0L, (await run.StoredAsync()).ProgressDone); run.Time.Advance(TimeSpan.FromSeconds(0.2)); // Too soon for a publish, but the persist falls due: it shows what it saves. await run.Context.ReportAsync(3, 100); Assert.SequenceEqual([1L, 2L, 3L], run.Published.Select(p => p.Done)); Assert.Equal(3L, (await run.StoredAsync()).ProgressDone); } [Test] public async Task ProgressIsSavedAtMostEveryCheckpointIntervalAndAtEachCheckpoint() { await using var run = await StartAsync(); for (var i = 1; i <= 50; i++) { await run.Context.ReportAsync(i, 100); run.Time.Advance(TimeSpan.FromMilliseconds(99)); } Assert.Equal(0L, (await run.StoredAsync()).ProgressDone); run.Time.Advance(TimeSpan.FromMilliseconds(50)); await run.Context.ReportAsync(51, 100); Assert.Equal(51L, (await run.StoredAsync()).ProgressDone); var published = run.Published.Count; // A checkpoint is saved at once; it goes out as any report would. run.Time.Advance(Tick); await run.Context.CheckpointAsync("52", 52, 100); var stored = await run.StoredAsync(); Assert.Equal((52L, "52"), (stored.ProgressDone, stored.Checkpoint)); Assert.Null(stored.Detail); Assert.Equal(published, run.Published.Count); var verify = new JobDetail(JobStep.Verify, "Harbour sunset", 1, 9, 1, 3, JobPartUnit.Sequence); run.Time.Advance(Tick); await run.Context.CheckpointAsync("53", 53, 100, verify); stored = await run.StoredAsync(); Assert.Equal((53L, "53", verify), (stored.ProgressDone, stored.Checkpoint, stored.Detail)); Assert.Equal(verify, run.Published[^1].Detail); // A checkpoint without a detail keeps it. run.Time.Advance(Tick); await run.Context.CheckpointAsync("54", 54, 100); Assert.Equal(verify, (await run.StoredAsync()).Detail); } [Test] public async Task TheRateIsTheSameUnderThePublishThrottle() { await using var run = await StartAsync(); // 100 photos a second for ten seconds, every photo reported. for (var i = 1; i <= 1000; i++) { await run.Context.ReportAsync(i, 2000); run.Time.Advance(Tick); } Assert.True(run.Published.Count <= 22, $"{run.Published.Count} publishes"); var last = run.Published[^1]; Assert.Approximately(100, last.ItemsPerSecond!.Value, 0.5); Assert.Approximately((2000 - last.Done) / 100.0, last.Eta!.Value.TotalSeconds, 0.1); } [Test] public async Task ItemCountsWithoutADetailAreAPhotoPass() { await using var run = await StartAsync(); Assert.Null(run.Context.Detail); await run.Context.ReportItemAsync(1, 10, 3, 9); Assert.Equal(new JobDetail(JobStep.Process, null, 3, 9), run.Context.Detail); Assert.Equal(run.Context.Detail, run.Published.Single().Detail); // A plain report keeps the detail and drops the old free text. await run.Context.ReportAsync(2, 10, "IMG_0001.CR3"); await run.Context.ReportAsync(3, 10); Assert.Null(run.Context.Message); Assert.Equal(new JobDetail(JobStep.Process, null, 3, 9), run.Context.Detail); } [Test] public async Task ADroppedReportAllocatesNothing() { await using var run = await StartAsync(); var thumbs = new JobDetail(JobStep.Thumbs, "Harbour sunset", 0, 3600, 2, 7, JobPartUnit.Sequence); await run.Context.ReportAsync(1, 1_000_000, thumbs); var job = run.Context; var dropped = 0; for (var i = 0; i < 200; i++) dropped += Drop(job, thumbs, i) ? 1 : 0; var before = GC.GetAllocatedBytesForCurrentThread(); for (var i = 0; i < 20_000; i++) dropped += Drop(job, thumbs, i) ? 1 : 0; var allocated = GC.GetAllocatedBytesForCurrentThread() - before; Assert.Equal(20_200, dropped); Assert.Equal(0L, allocated); Assert.Count(1, run.Published); } [Test] public async Task TheEngineSendsAndSavesTheDetail() { await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation); var verify = new JobDetail(JobStep.Verify, "Harbour sunset", 5, 9, 1, 2, JobPartUnit.Sequence); var handler = new CallbackHandler(JobKind.Verify, async c => { await c.CheckpointAsync("1", 1, 2, verify); await c.ReportAsync(2, 2); }); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); var published = new List(); engine.Progress += p => { lock (published) published.Add(p); }; await engine.StartAsync(context.Cancellation); var job = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], cancellationToken: context.Cancellation); var until = DateTime.UtcNow.AddSeconds(15); Job? done = null; while (DateTime.UtcNow < until && (done = await db.Jobs.GetAsync(job.Id, context.Cancellation))?.State != JobState.Done) await Task.Delay(20, context.Cancellation); Assert.Equal(JobState.Done, done?.State); Assert.Equal(verify, done!.Detail); lock (published) Assert.SequenceEqual([verify, verify], published.Select(p => p.Detail)); } /// One report dropped by the throttle: no new step, no completed count, no time passed. private static bool Drop(JobContext job, JobDetail detail, int i) { var item = job.ReportItemAsync(2 + i, 1_000_000, i, 3600); var full = job.ReportAsync(3 + i, 1_000_000, detail); return item.IsCompletedSuccessfully && full.IsCompletedSuccessfully; } private async Task StartAsync() { var time = new ManualTime(); var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation); var engine = new JobEngine(db, [], new JobEngineOptions { WorkerId = "t", Time = time }); var run = new Run(db, engine, time); engine.Progress += run.Published.Add; await db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow }, context.Cancellation); var claimed = Assert.NotNull(await db.Jobs.ClaimNextAsync("t", null, context.Cancellation)); run.Context = new JobContext(engine, claimed, context.Cancellation); return run; } private sealed class Run(ChronosDatabase db, JobEngine engine, ManualTime time) : IAsyncDisposable { public ManualTime Time { get; } = time; public JobContext Context { get; set; } = null!; public List Published { get; } = []; public async Task StoredAsync() => (await db.Jobs.GetAsync(Context.Job.Id))!; public async ValueTask DisposeAsync() { await engine.DisposeAsync(); await db.DisposeAsync(); } } private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler { public JobKind Kind => kind; public Task RunAsync(JobContext context) => run(context); } }