Files
Chronos/tests/Chronos.Server.Tests/Events/ProgressEventTests.cs
T
Alby96andClaude Opus 5.5 ee9505df22 feat(jobs): progress detail contract and a 2/s publish throttle
Mass passes need to say what they do now (photo x of y, sequence n of m) without
slowing down. This adds the contract and the engine side; handlers and displays
follow in their own parcels.

- Core: JobDetail (step, item, item x of y, part n of m with unit), JobStep and
  JobPartUnit appended-only and stored by kebab name; Job.Detail.
- Protocol: JobDetailDto, appended as JobDto.Detail, JobProgressEvent.Detail and
  NodeProgress.Detail, so older peers ignore it and newer ones fall back.
- Data: JobContext keeps a detail across reports (ReportAsync with a detail,
  ReportItemAsync without allocating, CheckpointAsync with a detail) and publishes
  at most once per ProgressInterval (500 ms on the engine's TimeProvider), at once
  for the first report, a new step or a completed count; a dropped report costs no
  allocation or lock, and the persist that falls due publishes what it saves.
  Persist stays at most every 5 s. JobWire maps steps, units and details; an
  unknown name reads as none.
- Migration 0011 adds jobs.progress_detail (nullable JSON text); an unreadable
  cell reads as no detail. Lists return it.
- Server: ProgressGate (500 ms per job, step-change and completion bypass)
  shared by the engine host and render nodes; node details are stored and
  broadcast, an older node without one still works.

Refs #65

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 13:02:59 +01:00

207 lines
11 KiB
C#

using System.Net;
using System.Net.Http.Json;
using System.Text.Json;
using Microsoft.Extensions.DependencyInjection;
using Chronos.Core.Model;
using Chronos.Data.Jobs;
using Chronos.Protocol;
using Chronos.Server.Events;
using Chronos.Server.Jobs;
using Chronos.Server.Tests.Jobs;
using Chronos.Testing;
namespace Chronos.Server.Tests.Events;
/// <summary>
/// What a job does now on the wire (owner review 5, #65): the progress event, the job and a render node's report carry an
/// optional detail, older and newer peers read each other, and the event socket gets a job's progress at most twice a second.
/// </summary>
[Category("server")]
public sealed class ProgressEventTests(TestContext context)
{
private static readonly JobDetailDto Verify = new("verify", "Harbour sunset", 812, 2400, 3, 41, "sequence");
[Test]
public void TheProgressEventCarriesItsDetailInCamelCase()
{
var id = Guid.CreateVersion7();
var json = JsonSerializer.Serialize<ServerEvent>(new JobProgressEvent(id, 3, 10, 2.5, 4, Verify), ProtocolJson.Default.ServerEvent);
Assert.Contains("\"type\":\"jobProgress\"", json);
Assert.Contains("\"detail\":{\"step\":\"verify\",\"item\":\"Harbour sunset\",\"itemDone\":812,\"itemTotal\":2400,\"part\":3,\"parts\":41,\"partUnit\":\"sequence\"}", json);
Assert.Equal(new JobProgressEvent(id, 3, 10, 2.5, 4, Verify), JsonSerializer.Deserialize(json, ProtocolJson.Default.ServerEvent));
// None is left out.
var plain = JsonSerializer.Serialize<ServerEvent>(new JobProgressEvent(id, 3, 10, null, null), ProtocolJson.Default.ServerEvent);
Assert.False(plain.Contains("detail", StringComparison.Ordinal), plain);
}
[Test]
public void AnOlderPeersPayloadReadsWithoutADetail()
{
var id = Guid.CreateVersion7();
var progress = JsonSerializer.Deserialize($$"""{"type":"jobProgress","jobId":"{{id}}","done":1,"total":4,"itemsPerSecond":null,"etaSeconds":null}""",
ProtocolJson.Default.ServerEvent);
Assert.Equal(new JobProgressEvent(id, 1, 4, null, null), progress);
Assert.Equal(new NodeProgress(120, 3600, 40, "encoding"),
JsonSerializer.Deserialize("""{"done":120,"total":3600,"itemsPerSecond":40,"logLine":"encoding"}""", ProtocolJson.Default.NodeProgress));
var job = JsonSerializer.Deserialize($$"""
{"id":"{{id}}","kind":"verify","targetIds":[],"state":"running","priority":"normal","progressDone":1,"progressTotal":4,
"attempts":1,"created":"2026-11-15T10:00:00Z"}
""", ProtocolJson.Default.JobDto)!;
Assert.Null(job.Detail);
Assert.Null(JobWire.FromDto(job)!.Detail);
// A detail of fewer fields (a newer peer's that drops some, or a sparse one) reads with zeros.
Assert.Equal(new JobDetailDto("backup"), JsonSerializer.Deserialize("""{"step":"backup"}""", ProtocolJson.Default.JobDetailDto));
}
[Test]
public void ANewerPayloadReadsIntoAnOlderRecord()
{
var id = Guid.CreateVersion7();
var web = new JsonSerializerOptions(JsonSerializerDefaults.Web);
var json = JsonSerializer.Serialize<ServerEvent>(new JobProgressEvent(id, 3, 10, 2.5, 4, Verify), ProtocolJson.Default.ServerEvent);
Assert.Equal(new JobProgressBefore(id, 3, 10, 2.5, 4), JsonSerializer.Deserialize<JobProgressBefore>(json, web));
var node = JsonSerializer.Serialize(new NodeProgress(120, 3600, 40, "encoding", new JobDetailDto("encode", null, 120, 3600)), ProtocolJson.Default.NodeProgress);
Assert.Equal(new NodeProgressBefore(120, 3600, 40, "encoding"), JsonSerializer.Deserialize<NodeProgressBefore>(node, web));
}
[Test]
public void TheGateLetsAJobsProgressOutTwiceASecondButForNewStepsAndCompletedCounts()
{
var clock = new TimestampClock();
var gate = new ProgressGate(clock);
var job = Guid.CreateVersion7();
Assert.True(gate.Passes(job, 1, 100, JobStep.Verify), "the first");
clock.Advance(TimeSpan.FromMilliseconds(100));
Assert.False(gate.Passes(job, 2, 100, JobStep.Verify));
Assert.True(gate.Passes(job, 3, 100, JobStep.Replace), "a new step");
Assert.True(gate.Passes(job, 100, 100, JobStep.Replace), "a completed count");
Assert.True(gate.Passes(Guid.CreateVersion7(), 1, 100, JobStep.Verify), "another job");
clock.Advance(TimeSpan.FromMilliseconds(300));
Assert.False(gate.Passes(job, 4, 100, JobStep.Replace));
// The engine spaces its reports by the same half second, measured a moment before: they are not dropped.
clock.Advance(TimeSpan.FromMilliseconds(160));
Assert.True(gate.Passes(job, 5, 100, JobStep.Replace), "on time");
Assert.False(gate.Passes(job, 6, 100, JobStep.Replace));
// A job that stops running is forgotten; its next run starts afresh.
gate.Forget(job);
Assert.True(gate.Passes(job, 7, 100, JobStep.Replace), "after a pause");
gate.Forget(job);
Assert.Equal(1, gate.Count);
Assert.Equal(TimeSpan.FromMilliseconds(500), ProgressGate.Interval);
}
[Test]
public async Task TheServerSendsAndSavesWhatItsJobsDo()
{
var handler = new DetailJobHandler();
await using var server = await TestServer.StartAsync(context, services: s => s.AddSingleton<IJobHandler>(handler));
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var sequence = await JobApiTests.CreateSequenceAsync(editor);
var hub = server.Services.GetRequiredService<EventHub>();
using var subscription = hub.Subscribe();
var job = await JobApiTests.CreateAsync(editor, "deep-analyse", sequence);
await handler.Started.Task.WaitAsync(TimeSpan.FromSeconds(10));
var running = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal(DetailJobHandler.Expected, running.Detail);
handler.Release();
var done = await JobApiTests.WaitForAsync(editor, job.Id, "done");
// Kept once the job ends, as it was last saved.
Assert.Equal(DetailJobHandler.Expected with { ItemDone = 4 }, done.Detail);
using (var list = await editor.GetAsync("/api/v1/jobs"))
{
var page = await list.ReadAsync(ProtocolJson.Default.PageJobDto);
Assert.Equal(DetailJobHandler.Expected with { ItemDone = 4 }, page.Items.Single(j => j.Id == job.Id).Detail);
}
var progress = new List<JobProgressEvent>();
while (subscription.Reader.TryRead(out var e))
{
if (e is JobProgressEvent p && p.JobId == job.Id) progress.Add(p);
}
Assert.SequenceEqual([1L, 4L], progress.Select(p => p.Done));
Assert.SequenceEqual([DetailJobHandler.Expected, DetailJobHandler.Expected with { ItemDone = 4 }], progress.Select(p => p.Detail));
// The ended job is forgotten once its state goes out.
var gate = server.Services.GetRequiredService<JobEngineHost>().Gate;
for (var i = 0; i < 200 && gate.Count > 0; i++) await Task.Delay(25, context.Cancellation);
Assert.Equal(0, gate.Count);
}
[Test]
public async Task ARenderNodeSendsWhatItDoesAndAnOlderNodeNothing()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequence = await JobApiTests.CreateSequenceAsync(editor);
var job = await JobApiTests.CreateAsync(editor, "render", sequence, RenderNodeTests.HevcJob);
var node = await RenderNodeTests.RegisterAsync(editor, "studio-pc", "hevc", 8192);
using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK);
using var subscription = server.Services.GetRequiredService<EventHub>().Subscribe();
var encode = new JobDetailDto("encode", "Harbour sunset", 120, 3600, 1, 1, "render");
await ReportAsync(new NodeProgress(120, 3600, 40, "encoding", encode));
Assert.Equal(encode, (await JobApiTests.GetAsync(editor, job.Id)).Detail);
// A step or unit this server does not know is stored as none, with its counts.
await ReportAsync(new NodeProgress(130, 3600, 40, null, new JobDetailDto("teleport", null, 130, 3600, 0, 0, "galaxy")));
Assert.Equal(new JobDetailDto("none", null, 130, 3600), (await JobApiTests.GetAsync(editor, job.Id)).Detail);
// An older node sends none: the counts alone.
await ReportAsync(new NodeProgress(140, 3600, 40, null));
var stored = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal((140L, (JobDetailDto?)null), (stored.ProgressDone, stored.Detail));
var events = new List<JobProgressEvent>();
while (subscription.Reader.TryRead(out var e))
{
if (e is JobProgressEvent p) events.Add(p);
}
Assert.Equal(encode, events[0].Detail);
Assert.True(events.All(e => e.JobId == job.Id), "only the node's job");
async Task ReportAsync(NodeProgress progress)
{
using var response = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/progress", progress, ProtocolJson.Default.NodeProgress);
await response.ExpectAsync(HttpStatusCode.NoContent);
}
}
/// <summary>A progress event as a 1.0 client reads it.</summary>
private sealed record JobProgressBefore(Guid JobId, long Done, long Total, double? ItemsPerSecond, double? EtaSeconds);
/// <summary>A node's report as a 1.0 server reads it.</summary>
private sealed record NodeProgressBefore(long Done, long Total, double? ItemsPerSecond, string? LogLine);
/// <summary>A monotonic clock the test moves by hand.</summary>
private sealed class TimestampClock : TimeProvider
{
private long _ticks;
public override long TimestampFrequency => TimeSpan.TicksPerSecond;
public override long GetTimestamp() => Interlocked.Read(ref _ticks);
public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks);
}
/// <summary>A deep analysis that says what it does, waits to be released, then completes.</summary>
private sealed class DetailJobHandler : IJobHandler
{
public static readonly JobDetailDto Expected = new("analyse", "Harbour sunset", 1, 4, 1, 1, "sequence");
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
public JobKind Kind => JobKind.DeepAnalyse;
public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void Release() => _release.TrySetResult();
public async Task RunAsync(JobContext context)
{
await context.CheckpointAsync("1", 1, 4, JobWire.FromDto(Expected)!);
Started.TrySetResult();
await _release.Task.WaitAsync(context.Cancellation);
await context.ReportItemAsync(4, 4, 4, 4);
}
}
}