CI / Build and test (push) Successful in 11m36s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 12s
Release / Build, test and publish (push) Successful in 14m56s
CI / PostgreSQL 16 integration (push) Successful in 10m11s
CI / PostgreSQL 17 integration (push) Successful in 10m15s
CI / Docker image (push) Successful in 45s
Release / Server image (push) Successful in 4m55s
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
203 lines
7.1 KiB
C#
203 lines
7.1 KiB
C#
using Chronos.Core.Model;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Protocol;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Jobs;
|
|
|
|
/// <summary>
|
|
/// A clock the test moves by hand (BCL <see cref="TimeProvider"/>, no package), whose timers fire only when
|
|
/// <see cref="Advance"/> reaches them, on the test's thread.
|
|
/// </summary>
|
|
internal sealed class ManualTime : TimeProvider
|
|
{
|
|
private readonly Lock _gate = new();
|
|
private readonly List<Timer> _timers = [];
|
|
private long _ticks;
|
|
|
|
public override long TimestampFrequency => TimeSpan.TicksPerSecond;
|
|
|
|
public override long GetTimestamp() => Interlocked.Read(ref _ticks);
|
|
|
|
public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period)
|
|
{
|
|
var timer = new Timer(this, callback, state);
|
|
timer.Change(dueTime, period);
|
|
return timer;
|
|
}
|
|
|
|
/// <summary>Moves the clock on, then fires each one-shot timer due by then.</summary>
|
|
public void Advance(TimeSpan by)
|
|
{
|
|
List<Timer> due;
|
|
lock (_gate)
|
|
{
|
|
var now = Interlocked.Add(ref _ticks, by.Ticks);
|
|
due = [.. _timers.Where(t => t.Due <= now)];
|
|
foreach (var timer in due) _timers.Remove(timer);
|
|
}
|
|
foreach (var timer in due) timer.Fire();
|
|
}
|
|
|
|
private void Schedule(Timer timer, TimeSpan dueTime)
|
|
{
|
|
lock (_gate)
|
|
{
|
|
_timers.Remove(timer);
|
|
if (dueTime == Timeout.InfiniteTimeSpan) return;
|
|
timer.Due = Interlocked.Read(ref _ticks) + dueTime.Ticks;
|
|
_timers.Add(timer);
|
|
}
|
|
}
|
|
|
|
private sealed class Timer(ManualTime clock, TimerCallback callback, object? state) : ITimer
|
|
{
|
|
public long Due { get; set; }
|
|
|
|
public bool Change(TimeSpan dueTime, TimeSpan period)
|
|
{
|
|
clock.Schedule(this, dueTime);
|
|
return true;
|
|
}
|
|
|
|
public void Fire() => callback(state);
|
|
|
|
public void Dispose() => clock.Schedule(this, Timeout.InfiniteTimeSpan);
|
|
|
|
public ValueTask DisposeAsync()
|
|
{
|
|
Dispose();
|
|
return ValueTask.CompletedTask;
|
|
}
|
|
}
|
|
}
|
|
|
|
[Category("jobs")]
|
|
public sealed class JobThroughputTests(TestContext context)
|
|
{
|
|
[Test]
|
|
public void TheFirstWindowWithProgressSeedsTheRate()
|
|
{
|
|
var throughput = new JobThroughput(0);
|
|
Assert.Null(throughput.Sample(2, 0, 100).Rate);
|
|
var (rate, eta) = throughput.Sample(4, 10, 100);
|
|
Assert.Equal(5.0, rate);
|
|
Assert.Equal(TimeSpan.FromSeconds(18), eta);
|
|
}
|
|
|
|
[Test]
|
|
public void AStalledJobLosesItsRateAndTimeLeftWithoutOverflowing()
|
|
{
|
|
// The owner's log: one sequence of 17 done, then a long one checkpointed every 10 s without moving the count.
|
|
var throughput = new JobThroughput(0);
|
|
throughput.Sample(600, 1, 17);
|
|
(double? Rate, TimeSpan? Eta) last = default;
|
|
for (var i = 1; i <= 2000; i++) last = throughput.Sample(600 + i * 10, 1, 17);
|
|
Assert.Null(last.Rate);
|
|
Assert.Null(last.Eta);
|
|
}
|
|
|
|
[Test]
|
|
public void ProgressThatGoesBackStartsANewWindow()
|
|
{
|
|
var throughput = new JobThroughput(0);
|
|
throughput.Sample(2, 20, 100);
|
|
throughput.Sample(3, 5, 100);
|
|
var (rate, _) = throughput.Sample(5, 15, 100);
|
|
Assert.Approximately(0.7 * 10 + 0.3 * 5, rate!.Value, 1e-9);
|
|
}
|
|
|
|
[Theory]
|
|
[Case(double.NaN)]
|
|
[Case(double.PositiveInfinity)]
|
|
[Case(-1.0)]
|
|
[Case(1e300)]
|
|
[Case(4e7)]
|
|
public void UnusableSecondsGiveNoTimeLeft(double seconds) => Assert.Null(JobEta.FromSeconds(seconds));
|
|
|
|
[Test]
|
|
public void UsableSecondsAndRatesGiveTheTimeLeft()
|
|
{
|
|
Assert.Equal(TimeSpan.FromSeconds(90), JobEta.FromSeconds(90));
|
|
Assert.Equal(TimeSpan.FromSeconds(4), JobEta.Of(8, 2));
|
|
Assert.Null(JobEta.Of(0, 2));
|
|
Assert.Null(JobEta.Of(8, null));
|
|
Assert.Null(JobEta.Of(8, 1e-300));
|
|
}
|
|
|
|
[Test]
|
|
public void ServerTimeLeftBeyondTheMaximumReadsUnknown()
|
|
{
|
|
var dto = new JobDto(Guid.NewGuid(), "render", [], null, "running", "normal", 1, 17, 1e-11, 1e300, null, null, 0, null,
|
|
DateTimeOffset.UtcNow, null, null);
|
|
Assert.Null(JobWire.FromDto(dto)!.Eta);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AJobWhoseCountStallsForHoursStillEndsDone()
|
|
{
|
|
var time = new ManualTime();
|
|
var reported = new List<JobProgressInfo>();
|
|
await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
|
|
var handler = new CallbackHandler(JobKind.Proxies, async c =>
|
|
{
|
|
time.Advance(TimeSpan.FromSeconds(600));
|
|
await c.CheckpointAsync("1:0", 1, 17);
|
|
for (var frame = 1; frame <= 400; frame++)
|
|
{
|
|
time.Advance(TimeSpan.FromSeconds(11));
|
|
await c.CheckpointAsync($"1:{frame}", 1, 17);
|
|
}
|
|
});
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", Time = time });
|
|
engine.Progress += p =>
|
|
{
|
|
lock (reported) reported.Add(p);
|
|
};
|
|
await engine.StartAsync();
|
|
var job = await engine.EnqueueAsync(JobKind.Proxies, [Guid.NewGuid()]);
|
|
var done = await WaitForAsync(db, job.Id, JobState.Done);
|
|
Assert.Null(done.Eta);
|
|
Assert.Null(done.ItemsPerSecond);
|
|
lock (reported) Assert.Null(reported[^1].Eta);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AProgressSubscriberThatThrowsNeverFailsTheJob()
|
|
{
|
|
await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
|
|
var handler = new CallbackHandler(JobKind.Verify, async c =>
|
|
{
|
|
for (var i = 1; i <= 5; i++) await c.CheckpointAsync(i.ToString(System.Globalization.CultureInfo.InvariantCulture), i, 5);
|
|
});
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
engine.Progress += _ => throw new InvalidOperationException("broken subscriber");
|
|
await engine.StartAsync();
|
|
var first = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()]);
|
|
var second = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()]);
|
|
Assert.Equal(5L, (await WaitForAsync(db, first.Id, JobState.Done)).ProgressDone);
|
|
await WaitForAsync(db, second.Id, JobState.Done);
|
|
}
|
|
|
|
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
|
|
public Task RunAsync(JobContext context) => run(context);
|
|
}
|
|
|
|
private static async Task<Job> WaitForAsync(ChronosDatabase db, Guid id, JobState state)
|
|
{
|
|
var until = DateTime.UtcNow.AddSeconds(15);
|
|
Job? job = null;
|
|
while (DateTime.UtcNow < until)
|
|
{
|
|
job = await db.Jobs.GetAsync(id);
|
|
if (job?.State == state) return job;
|
|
await Task.Delay(20);
|
|
}
|
|
Assert.Fail($"Job {id} is {job?.State}, not {state}");
|
|
return null!;
|
|
}
|
|
}
|