using Chronos.Core.Model; using Chronos.Data.Jobs; using Chronos.Protocol; using Chronos.Testing; namespace Chronos.Data.Tests.Jobs; /// A clock the test moves by hand (BCL , no package). internal sealed class ManualTime : TimeProvider { private long _ticks; public override long TimestampFrequency => TimeSpan.TicksPerSecond; public override long GetTimestamp() => Interlocked.Read(ref _ticks); public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks); } [Category("jobs")] public sealed class JobThroughputTests(TestContext context) { [Test] public void TheFirstWindowWithProgressSeedsTheRate() { var throughput = new JobThroughput(0); Assert.Null(throughput.Sample(2, 0, 100).Rate); var (rate, eta) = throughput.Sample(4, 10, 100); Assert.Equal(5.0, rate); Assert.Equal(TimeSpan.FromSeconds(18), eta); } [Test] public void AStalledJobLosesItsRateAndTimeLeftWithoutOverflowing() { // The owner's log: one sequence of 17 done, then a long one checkpointed every 10 s without moving the count. var throughput = new JobThroughput(0); throughput.Sample(600, 1, 17); (double? Rate, TimeSpan? Eta) last = default; for (var i = 1; i <= 2000; i++) last = throughput.Sample(600 + i * 10, 1, 17); Assert.Null(last.Rate); Assert.Null(last.Eta); } [Test] public void ProgressThatGoesBackStartsANewWindow() { var throughput = new JobThroughput(0); throughput.Sample(2, 20, 100); throughput.Sample(3, 5, 100); var (rate, _) = throughput.Sample(5, 15, 100); Assert.Approximately(0.7 * 10 + 0.3 * 5, rate!.Value, 1e-9); } [Theory] [Case(double.NaN)] [Case(double.PositiveInfinity)] [Case(-1.0)] [Case(1e300)] [Case(4e7)] public void UnusableSecondsGiveNoTimeLeft(double seconds) => Assert.Null(JobEta.FromSeconds(seconds)); [Test] public void UsableSecondsAndRatesGiveTheTimeLeft() { Assert.Equal(TimeSpan.FromSeconds(90), JobEta.FromSeconds(90)); Assert.Equal(TimeSpan.FromSeconds(4), JobEta.Of(8, 2)); Assert.Null(JobEta.Of(0, 2)); Assert.Null(JobEta.Of(8, null)); Assert.Null(JobEta.Of(8, 1e-300)); } [Test] public void ServerTimeLeftBeyondTheMaximumReadsUnknown() { var dto = new JobDto(Guid.NewGuid(), "render", [], null, "running", "normal", 1, 17, 1e-11, 1e300, null, null, 0, null, DateTimeOffset.UtcNow, null, null); Assert.Null(JobWire.FromDto(dto)!.Eta); } [Test] public async Task AJobWhoseCountStallsForHoursStillEndsDone() { var time = new ManualTime(); var reported = new List(); await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation); var handler = new CallbackHandler(JobKind.Proxies, async c => { time.Advance(TimeSpan.FromSeconds(600)); await c.CheckpointAsync("1:0", 1, 17); for (var frame = 1; frame <= 400; frame++) { time.Advance(TimeSpan.FromSeconds(11)); await c.CheckpointAsync($"1:{frame}", 1, 17); } }); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Time = time }); engine.Progress += p => { lock (reported) reported.Add(p); }; await engine.StartAsync(); var job = await engine.EnqueueAsync(JobKind.Proxies, [Guid.NewGuid()]); var done = await WaitForAsync(db, job.Id, JobState.Done); Assert.Null(done.Eta); Assert.Null(done.ItemsPerSecond); lock (reported) Assert.Null(reported[^1].Eta); } [Test] public async Task AProgressSubscriberThatThrowsNeverFailsTheJob() { await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation); var handler = new CallbackHandler(JobKind.Verify, async c => { for (var i = 1; i <= 5; i++) await c.CheckpointAsync(i.ToString(System.Globalization.CultureInfo.InvariantCulture), i, 5); }); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); engine.Progress += _ => throw new InvalidOperationException("broken subscriber"); await engine.StartAsync(); var first = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()]); var second = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()]); Assert.Equal(5L, (await WaitForAsync(db, first.Id, JobState.Done)).ProgressDone); await WaitForAsync(db, second.Id, JobState.Done); } private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler { public JobKind Kind => kind; public Task RunAsync(JobContext context) => run(context); } private static async Task WaitForAsync(ChronosDatabase db, Guid id, JobState state) { var until = DateTime.UtcNow.AddSeconds(15); Job? job = null; while (DateTime.UtcNow < until) { job = await db.Jobs.GetAsync(id); if (job?.State == state) return job; await Task.Delay(20); } Assert.Fail($"Job {id} is {job?.State}, not {state}"); return null!; } }