Setup: a kept 1.0 Uploader no longer fixes the scope and folder, the Remove tooltip promises kept files only with a hashed log, and a single-program setup preselects it. The Uploader always downloads the setup. Sync links move to the server id at every sign-in while folders wait for jobs and open editors; jobs of a pending homes sort still find their projects. A shared library exposes only the shared archive's client-visible jobs and events, render nodes take only client renders, and the share counts as its own while starting. Progress: a new step waits a quarter interval and a trailing timer publishes the last state; the job bar tooltip carries the whole line; Recent maintenance and the Server jobs list pick up jobs that start while they are shown. Refs #64 #65 #66 #67 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
268 lines
12 KiB
C#
268 lines
12 KiB
C#
using Chronos.Core.Model;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Jobs;
|
|
|
|
/// <summary>
|
|
/// How a running job's progress goes out (owner review 5, #65): at most twice a second, at once for the run's first report,
|
|
/// a new step and a completed count, and with the persist that falls due when the last report was dropped; saved at most
|
|
/// every checkpoint interval and at each checkpoint, with its detail.
|
|
/// </summary>
|
|
[Category("jobs")]
|
|
public sealed class JobContextProgressTests(TestContext context)
|
|
{
|
|
private static readonly TimeSpan Tick = TimeSpan.FromMilliseconds(10);
|
|
|
|
/// <summary>A quarter of the default publish interval: what a new step waits since the last publish.</summary>
|
|
private static readonly TimeSpan StepWait = TimeSpan.FromMilliseconds(125);
|
|
|
|
[Test]
|
|
public async Task AHundredReportsInASecondGoOutAtMostThreeTimes()
|
|
{
|
|
await using var run = await StartAsync();
|
|
for (var i = 1; i <= 100; i++)
|
|
{
|
|
await run.Context.ReportAsync(i, 1000);
|
|
run.Time.Advance(Tick);
|
|
}
|
|
Assert.True(run.Published.Count <= 3, $"{run.Published.Count} publishes");
|
|
// The first at once, then one per half second: the latest state when its time comes.
|
|
Assert.SequenceEqual([1L, 50L, 100L], run.Published.Select(p => p.Done));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AJobGoneQuietInALongStepShowsItsLatestStateOnTime()
|
|
{
|
|
await using var run = await StartAsync();
|
|
// Maintenance: its first sub-steps take milliseconds, the third runs for minutes without a report.
|
|
await run.Context.ReportAsync(0, 5, new JobDetail(JobStep.Maintenance, "leases", 0, 0, 1, 5, JobPartUnit.Step));
|
|
run.Time.Advance(TimeSpan.FromMilliseconds(30));
|
|
await run.Context.CheckpointAsync("leases", 1, 5, new JobDetail(JobStep.Maintenance, "temp", 0, 0, 2, 5, JobPartUnit.Step));
|
|
run.Time.Advance(TimeSpan.FromMilliseconds(30));
|
|
var packs = new JobDetail(JobStep.Maintenance, "packs", 0, 0, 3, 5, JobPartUnit.Step);
|
|
await run.Context.CheckpointAsync("temp", 2, 5, packs);
|
|
Assert.Count(1, run.Published);
|
|
|
|
run.Time.Advance(TimeSpan.FromMilliseconds(500));
|
|
Assert.Count(2, run.Published);
|
|
Assert.Equal((2L, packs), (run.Published[^1].Done, run.Published[^1].Detail));
|
|
// Nothing new: nothing more goes out.
|
|
run.Time.Advance(TimeSpan.FromSeconds(10));
|
|
Assert.Count(2, run.Published);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AJobThatAlternatesStepsGoesOutAFewTimesASecond()
|
|
{
|
|
await using var run = await StartAsync();
|
|
// An incremental scan: every other folder changed, checked and indexed in turn, 200 reports in a second.
|
|
for (var i = 0; i < 200; i++)
|
|
{
|
|
var step = i % 2 == 0 ? JobStep.Check : JobStep.Index;
|
|
await run.Context.ReportAsync(i, 10_000, new JobDetail(step, "folder " + i, 0, 0, i + 1, 10_000, JobPartUnit.Folder));
|
|
run.Time.Advance(TimeSpan.FromMilliseconds(5));
|
|
}
|
|
Assert.True(run.Published.Count <= 10, $"{run.Published.Count} publishes");
|
|
// The last report still goes out once its time comes.
|
|
run.Time.Advance(TimeSpan.FromSeconds(1));
|
|
Assert.Equal(199L, run.Published[^1].Done);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANewStepAndACompletedCountGoOutAtOnce()
|
|
{
|
|
await using var run = await StartAsync();
|
|
var index = new JobDetail(JobStep.Index, "2024/Harbour sunset", 1, 5, 1, 2, JobPartUnit.Folder);
|
|
await run.Context.ReportAsync(1, 10, new JobDetail(JobStep.List));
|
|
run.Time.Advance(Tick);
|
|
await run.Context.ReportAsync(2, 10, new JobDetail(JobStep.List));
|
|
// A new step goes out once a quarter of the interval has passed, not at every switch.
|
|
run.Time.Advance(StepWait);
|
|
await run.Context.ReportAsync(3, 10, index);
|
|
run.Time.Advance(Tick);
|
|
await run.Context.ReportItemAsync(4, 10, 2, 5);
|
|
run.Time.Advance(Tick);
|
|
await run.Context.ReportAsync(10, 10);
|
|
run.Time.Advance(Tick);
|
|
await run.Context.ReportAsync(10, 10);
|
|
|
|
Assert.SequenceEqual([1L, 3L, 10L], run.Published.Select(p => p.Done));
|
|
Assert.Equal(index, run.Published[1].Detail);
|
|
// A report without a detail keeps the last one, with the item counts reported since.
|
|
Assert.Equal(index with { ItemDone = 2 }, run.Published[2].Detail);
|
|
Assert.Equal(index with { ItemDone = 2 }, run.Context.Detail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADroppedReportGoesOutWithThePersistThatFallsDue()
|
|
{
|
|
await using var run = await StartAsync();
|
|
await run.Context.ReportAsync(1, 100);
|
|
run.Time.Advance(TimeSpan.FromSeconds(4.8));
|
|
await run.Context.ReportAsync(2, 100);
|
|
Assert.Equal(0L, (await run.StoredAsync()).ProgressDone);
|
|
run.Time.Advance(TimeSpan.FromSeconds(0.2));
|
|
// Too soon for a publish, but the persist falls due: it shows what it saves.
|
|
await run.Context.ReportAsync(3, 100);
|
|
Assert.SequenceEqual([1L, 2L, 3L], run.Published.Select(p => p.Done));
|
|
Assert.Equal(3L, (await run.StoredAsync()).ProgressDone);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProgressIsSavedAtMostEveryCheckpointIntervalAndAtEachCheckpoint()
|
|
{
|
|
await using var run = await StartAsync();
|
|
for (var i = 1; i <= 50; i++)
|
|
{
|
|
await run.Context.ReportAsync(i, 100);
|
|
run.Time.Advance(TimeSpan.FromMilliseconds(99));
|
|
}
|
|
Assert.Equal(0L, (await run.StoredAsync()).ProgressDone);
|
|
run.Time.Advance(TimeSpan.FromMilliseconds(50));
|
|
await run.Context.ReportAsync(51, 100);
|
|
Assert.Equal(51L, (await run.StoredAsync()).ProgressDone);
|
|
var published = run.Published.Count;
|
|
|
|
// A checkpoint is saved at once; it goes out as any report would.
|
|
run.Time.Advance(Tick);
|
|
await run.Context.CheckpointAsync("52", 52, 100);
|
|
var stored = await run.StoredAsync();
|
|
Assert.Equal((52L, "52"), (stored.ProgressDone, stored.Checkpoint));
|
|
Assert.Null(stored.Detail);
|
|
Assert.Equal(published, run.Published.Count);
|
|
|
|
var verify = new JobDetail(JobStep.Verify, "Harbour sunset", 1, 9, 1, 3, JobPartUnit.Sequence);
|
|
run.Time.Advance(StepWait);
|
|
await run.Context.CheckpointAsync("53", 53, 100, verify);
|
|
stored = await run.StoredAsync();
|
|
Assert.Equal((53L, "53", verify), (stored.ProgressDone, stored.Checkpoint, stored.Detail));
|
|
Assert.Equal(verify, run.Published[^1].Detail);
|
|
|
|
// A checkpoint without a detail keeps it.
|
|
run.Time.Advance(Tick);
|
|
await run.Context.CheckpointAsync("54", 54, 100);
|
|
Assert.Equal(verify, (await run.StoredAsync()).Detail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheRateIsTheSameUnderThePublishThrottle()
|
|
{
|
|
await using var run = await StartAsync();
|
|
// 100 photos a second for ten seconds, every photo reported.
|
|
for (var i = 1; i <= 1000; i++)
|
|
{
|
|
await run.Context.ReportAsync(i, 2000);
|
|
run.Time.Advance(Tick);
|
|
}
|
|
Assert.True(run.Published.Count <= 22, $"{run.Published.Count} publishes");
|
|
var last = run.Published[^1];
|
|
Assert.Approximately(100, last.ItemsPerSecond!.Value, 0.5);
|
|
Assert.Approximately((2000 - last.Done) / 100.0, last.Eta!.Value.TotalSeconds, 0.1);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ItemCountsWithoutADetailAreAPhotoPass()
|
|
{
|
|
await using var run = await StartAsync();
|
|
Assert.Null(run.Context.Detail);
|
|
await run.Context.ReportItemAsync(1, 10, 3, 9);
|
|
Assert.Equal(new JobDetail(JobStep.Process, null, 3, 9), run.Context.Detail);
|
|
Assert.Equal(run.Context.Detail, run.Published.Single().Detail);
|
|
// A plain report keeps the detail and drops the old free text.
|
|
await run.Context.ReportAsync(2, 10, "IMG_0001.CR3");
|
|
await run.Context.ReportAsync(3, 10);
|
|
Assert.Null(run.Context.Message);
|
|
Assert.Equal(new JobDetail(JobStep.Process, null, 3, 9), run.Context.Detail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADroppedReportAllocatesNothing()
|
|
{
|
|
await using var run = await StartAsync();
|
|
var thumbs = new JobDetail(JobStep.Thumbs, "Harbour sunset", 0, 3600, 2, 7, JobPartUnit.Sequence);
|
|
await run.Context.ReportAsync(1, 1_000_000, thumbs);
|
|
var job = run.Context;
|
|
var dropped = 0;
|
|
for (var i = 0; i < 200; i++) dropped += Drop(job, thumbs, i) ? 1 : 0;
|
|
var before = GC.GetAllocatedBytesForCurrentThread();
|
|
for (var i = 0; i < 20_000; i++) dropped += Drop(job, thumbs, i) ? 1 : 0;
|
|
var allocated = GC.GetAllocatedBytesForCurrentThread() - before;
|
|
Assert.Equal(20_200, dropped);
|
|
Assert.Equal(0L, allocated);
|
|
Assert.Count(1, run.Published);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheEngineSendsAndSavesTheDetail()
|
|
{
|
|
await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
|
|
var verify = new JobDetail(JobStep.Verify, "Harbour sunset", 5, 9, 1, 2, JobPartUnit.Sequence);
|
|
var handler = new CallbackHandler(JobKind.Verify, async c =>
|
|
{
|
|
await c.CheckpointAsync("1", 1, 2, verify);
|
|
await c.ReportAsync(2, 2);
|
|
});
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t" });
|
|
var published = new List<JobProgressInfo>();
|
|
engine.Progress += p =>
|
|
{
|
|
lock (published) published.Add(p);
|
|
};
|
|
await engine.StartAsync(context.Cancellation);
|
|
var job = await engine.EnqueueAsync(JobKind.Verify, [Guid.NewGuid()], cancellationToken: context.Cancellation);
|
|
var until = DateTime.UtcNow.AddSeconds(15);
|
|
Job? done = null;
|
|
while (DateTime.UtcNow < until && (done = await db.Jobs.GetAsync(job.Id, context.Cancellation))?.State != JobState.Done) await Task.Delay(20, context.Cancellation);
|
|
Assert.Equal(JobState.Done, done?.State);
|
|
Assert.Equal(verify, done!.Detail);
|
|
lock (published) Assert.SequenceEqual([verify, verify], published.Select(p => p.Detail));
|
|
}
|
|
|
|
/// <summary>One report dropped by the throttle: no new step, no completed count, no time passed.</summary>
|
|
private static bool Drop(JobContext job, JobDetail detail, int i)
|
|
{
|
|
var item = job.ReportItemAsync(2 + i, 1_000_000, i, 3600);
|
|
var full = job.ReportAsync(3 + i, 1_000_000, detail);
|
|
return item.IsCompletedSuccessfully && full.IsCompletedSuccessfully;
|
|
}
|
|
|
|
private async Task<Run> StartAsync()
|
|
{
|
|
var time = new ManualTime();
|
|
var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
|
|
var engine = new JobEngine(db, [], new JobEngineOptions { WorkerId = "t", Time = time });
|
|
var run = new Run(db, engine, time);
|
|
engine.Progress += run.Published.Add;
|
|
await db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow },
|
|
context.Cancellation);
|
|
var claimed = Assert.NotNull(await db.Jobs.ClaimNextAsync("t", null, context.Cancellation));
|
|
run.Context = new JobContext(engine, claimed, context.Cancellation);
|
|
return run;
|
|
}
|
|
|
|
private sealed class Run(ChronosDatabase db, JobEngine engine, ManualTime time) : IAsyncDisposable
|
|
{
|
|
public ManualTime Time { get; } = time;
|
|
|
|
public JobContext Context { get; set; } = null!;
|
|
|
|
public List<JobProgressInfo> Published { get; } = [];
|
|
|
|
public async Task<Job> StoredAsync() => (await db.Jobs.GetAsync(Context.Job.Id))!;
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
await engine.DisposeAsync();
|
|
await db.DisposeAsync();
|
|
}
|
|
}
|
|
|
|
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
|
|
public Task RunAsync(JobContext context) => run(context);
|
|
}
|
|
}
|