CI / Build and test (push) Successful in 11m36s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 12s
Release / Build, test and publish (push) Successful in 14m56s
CI / PostgreSQL 16 integration (push) Successful in 10m11s
CI / PostgreSQL 17 integration (push) Successful in 10m15s
CI / Docker image (push) Successful in 45s
Release / Server image (push) Successful in 4m55s
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
224 lines
12 KiB
C#
224 lines
12 KiB
C#
using System.Net;
|
|
using System.Net.Http.Json;
|
|
using System.Text.Json;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Protocol;
|
|
using Chronos.Server.Events;
|
|
using Chronos.Server.Jobs;
|
|
using Chronos.Server.Tests.Jobs;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Server.Tests.Events;
|
|
|
|
/// <summary>
|
|
/// What a job does now on the wire (owner review 5, #65): the progress event, the job and a render node's report carry an
|
|
/// optional detail, older and newer peers read each other, and the event socket gets a job's progress at most twice a second.
|
|
/// </summary>
|
|
[Category("server")]
|
|
public sealed class ProgressEventTests(TestContext context)
|
|
{
|
|
private static readonly JobDetailDto Verify = new("verify", "Harbour sunset", 812, 2400, 3, 41, "sequence");
|
|
|
|
[Test]
|
|
public void TheProgressEventCarriesItsDetailInCamelCase()
|
|
{
|
|
var id = Guid.CreateVersion7();
|
|
var json = JsonSerializer.Serialize<ServerEvent>(new JobProgressEvent(id, 3, 10, 2.5, 4, Verify), ProtocolJson.Default.ServerEvent);
|
|
Assert.Contains("\"type\":\"jobProgress\"", json);
|
|
Assert.Contains("\"detail\":{\"step\":\"verify\",\"item\":\"Harbour sunset\",\"itemDone\":812,\"itemTotal\":2400,\"part\":3,\"parts\":41,\"partUnit\":\"sequence\"}", json);
|
|
Assert.Equal(new JobProgressEvent(id, 3, 10, 2.5, 4, Verify), JsonSerializer.Deserialize(json, ProtocolJson.Default.ServerEvent));
|
|
// None is left out.
|
|
var plain = JsonSerializer.Serialize<ServerEvent>(new JobProgressEvent(id, 3, 10, null, null), ProtocolJson.Default.ServerEvent);
|
|
Assert.False(plain.Contains("detail", StringComparison.Ordinal), plain);
|
|
}
|
|
|
|
[Test]
|
|
public void AnOlderPeersPayloadReadsWithoutADetail()
|
|
{
|
|
var id = Guid.CreateVersion7();
|
|
var progress = JsonSerializer.Deserialize($$"""{"type":"jobProgress","jobId":"{{id}}","done":1,"total":4,"itemsPerSecond":null,"etaSeconds":null}""",
|
|
ProtocolJson.Default.ServerEvent);
|
|
Assert.Equal(new JobProgressEvent(id, 1, 4, null, null), progress);
|
|
Assert.Equal(new NodeProgress(120, 3600, 40, "encoding"),
|
|
JsonSerializer.Deserialize("""{"done":120,"total":3600,"itemsPerSecond":40,"logLine":"encoding"}""", ProtocolJson.Default.NodeProgress));
|
|
var job = JsonSerializer.Deserialize($$"""
|
|
{"id":"{{id}}","kind":"verify","targetIds":[],"state":"running","priority":"normal","progressDone":1,"progressTotal":4,
|
|
"attempts":1,"created":"2026-11-15T10:00:00Z"}
|
|
""", ProtocolJson.Default.JobDto)!;
|
|
Assert.Null(job.Detail);
|
|
Assert.Null(JobWire.FromDto(job)!.Detail);
|
|
// A detail of fewer fields (a newer peer's that drops some, or a sparse one) reads with zeros.
|
|
Assert.Equal(new JobDetailDto("backup"), JsonSerializer.Deserialize("""{"step":"backup"}""", ProtocolJson.Default.JobDetailDto));
|
|
}
|
|
|
|
[Test]
|
|
public void ANewerPayloadReadsIntoAnOlderRecord()
|
|
{
|
|
var id = Guid.CreateVersion7();
|
|
var web = new JsonSerializerOptions(JsonSerializerDefaults.Web);
|
|
var json = JsonSerializer.Serialize<ServerEvent>(new JobProgressEvent(id, 3, 10, 2.5, 4, Verify), ProtocolJson.Default.ServerEvent);
|
|
Assert.Equal(new JobProgressBefore(id, 3, 10, 2.5, 4), JsonSerializer.Deserialize<JobProgressBefore>(json, web));
|
|
var node = JsonSerializer.Serialize(new NodeProgress(120, 3600, 40, "encoding", new JobDetailDto("encode", null, 120, 3600)), ProtocolJson.Default.NodeProgress);
|
|
Assert.Equal(new NodeProgressBefore(120, 3600, 40, "encoding"), JsonSerializer.Deserialize<NodeProgressBefore>(node, web));
|
|
}
|
|
|
|
[Test]
|
|
public void TheGateLetsAJobsProgressOutTwiceASecondButForNewStepsAndCompletedCounts()
|
|
{
|
|
var clock = new TimestampClock();
|
|
var gate = new ProgressGate(clock);
|
|
var job = Guid.CreateVersion7();
|
|
Assert.True(gate.Passes(job, 1, 100, JobStep.Verify), "the first");
|
|
clock.Advance(TimeSpan.FromMilliseconds(100));
|
|
Assert.False(gate.Passes(job, 2, 100, JobStep.Verify));
|
|
Assert.True(gate.Passes(job, 3, 100, JobStep.Replace), "a new step");
|
|
Assert.True(gate.Passes(job, 100, 100, JobStep.Replace), "a completed count");
|
|
Assert.True(gate.Passes(Guid.CreateVersion7(), 1, 100, JobStep.Verify), "another job");
|
|
clock.Advance(TimeSpan.FromMilliseconds(300));
|
|
Assert.False(gate.Passes(job, 4, 100, JobStep.Replace));
|
|
// The engine spaces its reports by the same half second, measured a moment before: they are not dropped.
|
|
clock.Advance(TimeSpan.FromMilliseconds(160));
|
|
Assert.True(gate.Passes(job, 5, 100, JobStep.Replace), "on time");
|
|
Assert.False(gate.Passes(job, 6, 100, JobStep.Replace));
|
|
// A job that stops running is forgotten; its next run starts afresh.
|
|
gate.Forget(job);
|
|
Assert.True(gate.Passes(job, 7, 100, JobStep.Replace), "after a pause");
|
|
gate.Forget(job);
|
|
Assert.Equal(1, gate.Count);
|
|
Assert.Equal(TimeSpan.FromMilliseconds(500), ProgressGate.Interval);
|
|
}
|
|
|
|
[Test]
|
|
public void AJobThatAlternatesStepsStillGoesOutAFewTimesASecond()
|
|
{
|
|
var clock = new TimestampClock();
|
|
var gate = new ProgressGate(clock);
|
|
var job = Guid.CreateVersion7();
|
|
// An incremental scan: every other folder changed, checked and indexed in turn, 200 reports in a second.
|
|
var passed = 0;
|
|
for (var i = 0; i < 200; i++)
|
|
{
|
|
if (gate.Passes(job, i, 10_000, i % 2 == 0 ? JobStep.Check : JobStep.Index)) passed++;
|
|
clock.Advance(TimeSpan.FromMilliseconds(5));
|
|
}
|
|
Assert.True(passed <= 14, $"{passed} passed");
|
|
Assert.True(passed >= 4, $"{passed} passed: a new step still goes out early");
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheServerSendsAndSavesWhatItsJobsDo()
|
|
{
|
|
var handler = new DetailJobHandler();
|
|
await using var server = await TestServer.StartAsync(context, services: s => s.AddSingleton<IJobHandler>(handler));
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var sequence = await JobApiTests.CreateSequenceAsync(editor);
|
|
var hub = server.Services.GetRequiredService<EventHub>();
|
|
using var subscription = hub.Subscribe();
|
|
|
|
var job = await JobApiTests.CreateAsync(editor, "deep-analyse", sequence);
|
|
await handler.Started.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
var running = await JobApiTests.GetAsync(editor, job.Id);
|
|
Assert.Equal(DetailJobHandler.Expected, running.Detail);
|
|
handler.Release();
|
|
var done = await JobApiTests.WaitForAsync(editor, job.Id, "done");
|
|
// Kept once the job ends, as it was last saved.
|
|
Assert.Equal(DetailJobHandler.Expected with { ItemDone = 4 }, done.Detail);
|
|
using (var list = await editor.GetAsync("/api/v1/jobs"))
|
|
{
|
|
var page = await list.ReadAsync(ProtocolJson.Default.PageJobDto);
|
|
Assert.Equal(DetailJobHandler.Expected with { ItemDone = 4 }, page.Items.Single(j => j.Id == job.Id).Detail);
|
|
}
|
|
|
|
var progress = new List<JobProgressEvent>();
|
|
while (subscription.Reader.TryRead(out var e))
|
|
{
|
|
if (e is JobProgressEvent p && p.JobId == job.Id) progress.Add(p);
|
|
}
|
|
Assert.SequenceEqual([1L, 4L], progress.Select(p => p.Done));
|
|
Assert.SequenceEqual([DetailJobHandler.Expected, DetailJobHandler.Expected with { ItemDone = 4 }], progress.Select(p => p.Detail));
|
|
// The ended job is forgotten once its state goes out.
|
|
var gate = server.Services.GetRequiredService<JobEngineHost>().Gate;
|
|
for (var i = 0; i < 200 && gate.Count > 0; i++) await Task.Delay(25, context.Cancellation);
|
|
Assert.Equal(0, gate.Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ARenderNodeSendsWhatItDoesAndAnOlderNodeNothing()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var sequence = await JobApiTests.CreateSequenceAsync(editor);
|
|
var job = await JobApiTests.CreateAsync(editor, "render", sequence, RenderNodeTests.HevcJob);
|
|
var node = await RenderNodeTests.RegisterAsync(editor, "studio-pc", "hevc", 8192);
|
|
using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK);
|
|
using var subscription = server.Services.GetRequiredService<EventHub>().Subscribe();
|
|
|
|
var encode = new JobDetailDto("encode", "Harbour sunset", 120, 3600, 1, 1, "render");
|
|
await ReportAsync(new NodeProgress(120, 3600, 40, "encoding", encode));
|
|
Assert.Equal(encode, (await JobApiTests.GetAsync(editor, job.Id)).Detail);
|
|
// A step or unit this server does not know is stored as none, with its counts.
|
|
await ReportAsync(new NodeProgress(130, 3600, 40, null, new JobDetailDto("teleport", null, 130, 3600, 0, 0, "galaxy")));
|
|
Assert.Equal(new JobDetailDto("none", null, 130, 3600), (await JobApiTests.GetAsync(editor, job.Id)).Detail);
|
|
// An older node sends none: the counts alone.
|
|
await ReportAsync(new NodeProgress(140, 3600, 40, null));
|
|
var stored = await JobApiTests.GetAsync(editor, job.Id);
|
|
Assert.Equal((140L, (JobDetailDto?)null), (stored.ProgressDone, stored.Detail));
|
|
|
|
var events = new List<JobProgressEvent>();
|
|
while (subscription.Reader.TryRead(out var e))
|
|
{
|
|
if (e is JobProgressEvent p) events.Add(p);
|
|
}
|
|
Assert.Equal(encode, events[0].Detail);
|
|
Assert.True(events.All(e => e.JobId == job.Id), "only the node's job");
|
|
|
|
async Task ReportAsync(NodeProgress progress)
|
|
{
|
|
using var response = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/progress", progress, ProtocolJson.Default.NodeProgress);
|
|
await response.ExpectAsync(HttpStatusCode.NoContent);
|
|
}
|
|
}
|
|
|
|
/// <summary>A progress event as a 1.0 client reads it.</summary>
|
|
private sealed record JobProgressBefore(Guid JobId, long Done, long Total, double? ItemsPerSecond, double? EtaSeconds);
|
|
|
|
/// <summary>A node's report as a 1.0 server reads it.</summary>
|
|
private sealed record NodeProgressBefore(long Done, long Total, double? ItemsPerSecond, string? LogLine);
|
|
|
|
/// <summary>A monotonic clock the test moves by hand.</summary>
|
|
private sealed class TimestampClock : TimeProvider
|
|
{
|
|
private long _ticks;
|
|
|
|
public override long TimestampFrequency => TimeSpan.TicksPerSecond;
|
|
|
|
public override long GetTimestamp() => Interlocked.Read(ref _ticks);
|
|
|
|
public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks);
|
|
}
|
|
|
|
/// <summary>A deep analysis that says what it does, waits to be released, then completes.</summary>
|
|
private sealed class DetailJobHandler : IJobHandler
|
|
{
|
|
public static readonly JobDetailDto Expected = new("analyse", "Harbour sunset", 1, 4, 1, 1, "sequence");
|
|
|
|
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
public JobKind Kind => JobKind.DeepAnalyse;
|
|
|
|
public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
public void Release() => _release.TrySetResult();
|
|
|
|
public async Task RunAsync(JobContext context)
|
|
{
|
|
await context.CheckpointAsync("1", 1, 4, JobWire.FromDto(Expected)!);
|
|
Started.TrySetResult();
|
|
await _release.Task.WaitAsync(context.Cancellation);
|
|
await context.ReportItemAsync(4, 4, 4, 4);
|
|
}
|
|
}
|
|
}
|