using Chronos.Core.Model; using Chronos.Data.Jobs; using Chronos.Protocol; using Chronos.Testing; namespace Chronos.Data.Tests.Jobs; /// /// A clock the test moves by hand (BCL , no package), whose timers fire only when /// reaches them, on the test's thread. /// internal sealed class ManualTime : TimeProvider { private readonly Lock _gate = new(); private readonly List _timers = []; private long _ticks; public override long TimestampFrequency => TimeSpan.TicksPerSecond; public override long GetTimestamp() => Interlocked.Read(ref _ticks); public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) { var timer = new Timer(this, callback, state); timer.Change(dueTime, period); return timer; } /// Moves the clock on, then fires each one-shot timer due by then. public void Advance(TimeSpan by) { List due; lock (_gate) { var now = Interlocked.Add(ref _ticks, by.Ticks); due = [.. _timers.Where(t => t.Due <= now)]; foreach (var timer in due) _timers.Remove(timer); } foreach (var timer in due) timer.Fire(); } private void Schedule(Timer timer, TimeSpan dueTime) { lock (_gate) { _timers.Remove(timer); if (dueTime == Timeout.InfiniteTimeSpan) return; timer.Due = Interlocked.Read(ref _ticks) + dueTime.Ticks; _timers.Add(timer); } } private sealed class Timer(ManualTime clock, TimerCallback callback, object? state) : ITimer { public long Due { get; set; } public bool Change(TimeSpan dueTime, TimeSpan period) { clock.Schedule(this, dueTime); return true; } public void Fire() => callback(state); public void Dispose() => clock.Schedule(this, Timeout.InfiniteTimeSpan); public ValueTask DisposeAsync() { Dispose(); return ValueTask.CompletedTask; } } } [Category("jobs")] public sealed class JobThroughputTests(TestContext context) { [Test] public void TheFirstWindowWithProgressSeedsTheRate() { var throughput = new JobThroughput(0); Assert.Null(throughput.Sample(2, 0, 100).Rate); var (rate, eta) = throughput.Sample(4, 10, 100); Assert.Equal(5.0, rate); Assert.Equal(TimeSpan.FromSeconds(18), eta); } [Test] public void AStalledJobLosesItsRateAndTimeLeftWithoutOverflowing() { // The owner's log: one sequence of 17 done, then a long one checkpointed every 10 s without moving the count. var throughput = new JobThroughput(0); throughput.Sample(600, 1, 17); (double? Rate, TimeSpan? Eta) last = default; for (var i = 1; i <= 2000; i++) last = throughput.Sample(600 + i * 10, 1, 17); Assert.Null(last.Rate); Assert.Null(last.Eta); } [Test] public void ProgressThatGoesBackStartsANewWindow() { var throughput = new JobThroughput(0); throughput.Sample(2, 20, 100); throughput.Sample(3, 5, 100); var (rate, _) = throughput.Sample(5, 15, 100); Assert.Approximately(0.7 * 10 + 0.3 * 5, rate!.Value, 1e-9); } [Theory] [Case(double.NaN)] [Case(double.PositiveInfinity)] [Case(-1.0)] [Case(1e300)] [Case(4e7)] public void UnusableSecondsGiveNoTimeLeft(double seconds) => Assert.Null(JobEta.FromSeconds(seconds)); [Test] public void UsableSecondsAndRatesGiveTheTimeLeft() { Assert.Equal(TimeSpan.FromSeconds(90), JobEta.FromSeconds(90)); Assert.Equal(TimeSpan.FromSeconds(4), JobEta.Of(8, 2)); Assert.Null(JobEta.Of(0, 2)); Assert.Null(JobEta.Of(8, null)); Assert.Null(JobEta.Of(8, 1e-300)); } [Test] public void ServerTimeLeftBeyondTheMaximumReadsUnknown() { var dto = new JobDto(Guid.NewGuid(), "render", [], null, "running", "normal", 1, 17, 1e-11, 1e300, null, null, 0, null, DateTimeOffset.UtcNow, null, null); Assert.Null(JobWire.FromDto(dto)!.Eta); } [Test] public async Task AJobWhoseCountStallsForHoursStillEndsDone() { var time = new ManualTime(); var reported = new List(); await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation); var handler = new CallbackHandler(JobKind.Proxies, async c => { time.Advance(TimeSpan.FromSeconds(600)); await c.CheckpointAsync("1:0", 1, 17); for (var frame = 1; frame <= 400; frame++) { time.Advance(TimeSpan.FromSeconds(11)); await c.CheckpointAsync($"1:{frame}", 1, 17); } }); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Time = time }); engine.Progress += p => { lock (reported) reported.Add(p); }; await engine.StartAsync(); var job = await engine.EnqueueAsync(JobKind.Proxies, [Guid.NewGuid()]); var done = await WaitForAsync(db, job.Id, JobState.Done); Assert.Null(done.Eta); Assert.Null(done.ItemsPerSecond); lock (reported) Assert.Null(reported[^1].Eta); } [Test] public async Task AProgressSubscriberThatThrowsNeverFailsTheJob() { await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation); var handler = new CallbackHandler(JobKind.Verify, async c => { for (var i = 1; i <= 5; i++) await c.CheckpointAsync(i.ToString(System.Globalization.CultureInfo.InvariantCulture), i, 5); }); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" }); engine.Progress += _ => throw new InvalidOperationException("broken subscriber"); await engine.StartAsync(); var first = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()]); var second = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()]); Assert.Equal(5L, (await WaitForAsync(db, first.Id, JobState.Done)).ProgressDone); await WaitForAsync(db, second.Id, JobState.Done); } private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler { public JobKind Kind => kind; public Task RunAsync(JobContext context) => run(context); } private static async Task WaitForAsync(ChronosDatabase db, Guid id, JobState state) { var until = DateTime.UtcNow.AddSeconds(15); Job? job = null; while (DateTime.UtcNow < until) { job = await db.Jobs.GetAsync(id); if (job?.State == state) return job; await Task.Delay(20); } Assert.Fail($"Job {id} is {job?.State}, not {state}"); return null!; } }