Files
Chronos/src/Chronos.Server/Jobs/RenderNodeService.cs
T
Alby96andClaude Opus 5.5 f2b58522b9
CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
refactor!: rename Titano to Chronos and move CI and releases to Gitea (#6)
The owner renamed the product and moved the repository to the self-hosted
Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control,
and asked for Actions and a first automatic release with the server package
and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md).

- Chronos everywhere: a byte-exact rename of contents and paths, then a
  per-language review of the forms the old name's ending had dictated.
- Gitea instead of GitHub: the update check, licence, issue and support
  links read ChronosReleasesUrl; the update check takes the highest stable
  version, offers a pre-release build its final release and reports a
  repository it cannot see as a failed check. EULA, security policy and
  Gitea issue forms live at the repository root.
- Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release,
  nightly image and performance gates in the .NET SDK container with git
  checkout and the system SQLite. The release cross-compiles the Windows
  programs, checks their icon, version and DPI manifest, packs installers
  and zips, publishes the Gitea release and pushes the server image
  (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows
  runner (WINDOWS_RUNNER).
- New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the
  docker-exec script, which Gitea's service containers do not allow; the
  throw-away server runs without durability) and pe-check.
- Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR,
  3 on the shared runner; the weekly performance gates keep theirs).

BREAKING CHANGE: programs, settings, data folders, file types, environment
variables and the Docker image use the new name; nothing is migrated from
Titano, which never had a release.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-10-23 12:45:18 +02:00

265 lines
16 KiB
C#

using System.Globalization;
using System.Text.Json;
using Chronos.Core.Model;
using Chronos.Data;
using Chronos.Data.Jobs;
using Chronos.Data.Repositories;
using Chronos.Protocol;
using Chronos.Server.Api;
using Chronos.Server.Configuration;
using Chronos.Server.Events;
using Chronos.Server.Security;
using Chronos.Server.Storage;
using UploadState = Chronos.Data.Repositories.UploadState;
namespace Chronos.Server.Jobs;
/// <summary>
/// Render nodes (BRIEF §14.7): clients with the render-node right register their encoders, heartbeat, claim queued
/// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of
/// the sequence. A claimed job lists every frame with its flags (a node renders none missing, corrupt or excluded) and
/// the proxies the server asks for (<see cref="ServerProxies"/>), which the node sends from the originals it fetched.
/// A node that stops heart-beating loses its jobs to the queue (<see cref="RenderNodeMonitor"/>).
/// </summary>
public sealed class RenderNodeService(
ChronosDatabase db,
ServerOptions options,
ServerArchive archive,
UploadService uploads,
UploadStore store,
RenderStore renders,
JobEngineHost jobs,
EventHub events,
TimeProvider time)
{
public const string Idle = "idle";
public const string Busy = "busy";
public const string Offline = "offline";
private const int LogTailLines = 40;
public async Task<Outcome<RenderNodeDto>> RegisterAsync(Caller caller, RegisterNodeRequest request, CancellationToken cancellationToken)
{
var name = request.ClientName?.Trim();
if (string.IsNullOrEmpty(name) || name.Length > 100) return Outcome<RenderNodeDto>.Fail(400, "bad_request", "clientName");
if (request.Encoders is null || request.Encoders.Count > 64 || request.Encoders.Any(e => string.IsNullOrWhiteSpace(e?.Codec)))
return Outcome<RenderNodeDto>.Fail(400, "bad_request", "encoders");
var existing = (await db.RenderNodes.ListAsync(cancellationToken)).FirstOrDefault(n => n.UserId == caller.Id && n.ClientName == name);
// A node registers when it starts, holding no job: what the server still lists as running on it was abandoned
// by its previous run (stopped, restarted or crashed within the heartbeat timeout) and goes back to the queue.
if (existing is not null) await ReleaseJobsAsync(db, jobs, existing, $"Render node {name} restarted.", cancellationToken);
var node = new RenderNode
{
Id = existing?.Id ?? Guid.CreateVersion7(),
UserId = caller.Id,
ClientName = name,
CapabilitiesJson = JsonSerializer.Serialize(request with { ClientName = name }, ProtocolJson.Default.RegisterNodeRequest),
LastHeartbeat = time.GetUtcNow(),
State = Idle,
};
await db.RenderNodes.RegisterAsync(node, cancellationToken);
events.Publish(new NodeStateEvent(node.Id, node.ClientName, node.State));
return Dto.Node(node, caller.User.Username);
}
public async Task<Outcome<bool>> HeartbeatAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
await TouchAsync(node, cancellationToken);
return true;
}
/// <summary>The next queued render job this node can encode (highest priority, then oldest), or null when none fits.</summary>
public async Task<Outcome<ClaimedJob?>> ClaimAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<ClaimedJob?>.Fail(404, "not_found", "node");
var capabilities = Dto.Capabilities(node);
foreach (var queued in await db.Jobs.ListQueuedAsync(JobKind.Render, 50, cancellationToken))
{
// A render the server's own engine started (it left a checkpoint) resumes there: only it knows that attempt's
// leftovers and the batch entries it recorded.
if (queued.Checkpoint is not null) continue;
if (JobEndpoints.ParseRender(queued.ParametersJson) is not { } parameters || capabilities is null || !CanEncode(capabilities, parameters)) continue;
if (await db.Jobs.TryClaimAsync(queued.Id, NodeKey(node), cancellationToken) is not { } job) continue;
jobs.Publish(job);
if (job.TargetIds.Count != 1 || await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken) is not { } sequence)
{
await FinishAsync(node, job, JobState.Failed, "The sequence no longer exists.", cancellationToken);
continue;
}
await TouchAsync(node, cancellationToken);
var spec = ServerProxies.Spec(archive.Current);
return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson,
parameters.UseProxies, await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, cancellationToken), spec);
}
await TouchAsync(node, cancellationToken);
return (ClaimedJob?)null;
}
public async Task<Outcome<bool>> ProgressAsync(Caller caller, Guid nodeId, Guid jobId, NodeProgress progress, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<bool>.Fail(409, "job_not_assigned");
// A node reports any rate (a tiny one when it stalls): only a usable time left is stored and broadcast.
var rate = progress.ItemsPerSecond is { } r && double.IsFinite(r) && r > 0 ? r : (double?)null;
var eta = JobEta.Of(progress.Total - progress.Done, rate);
var tail = AppendLog(job.LogTail, progress.LogLine);
if (!await db.Jobs.UpdateNodeProgressAsync(jobId, NodeKey(node), progress.Done, progress.Total, rate, eta, tail, cancellationToken))
return Outcome<bool>.Fail(409, "job_not_assigned");
await TouchAsync(node, cancellationToken);
events.Publish(new JobProgressEvent(jobId, progress.Done, progress.Total, rate, eta?.TotalSeconds));
return true;
}
public async Task<Outcome<bool>> FailAsync(Caller caller, Guid nodeId, Guid jobId, NodeJobFailure failure, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<bool>.Fail(409, "job_not_assigned");
var error = string.IsNullOrWhiteSpace(failure.Error) ? "Render failed on the node." : failure.Error.Trim()[..Math.Min(failure.Error.Trim().Length, 500)];
return await FinishAsync(node, job, JobState.Failed, error, cancellationToken) ? true : Outcome<bool>.Fail(409, "job_not_assigned");
}
/// <summary>Starts (or resumes) the chunked upload of a job's output.</summary>
public async Task<Outcome<StartUploadResponse>> StartResultAsync(Caller caller, Guid nodeId, Guid jobId, RenderResultRequest request, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<StartUploadResponse>.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<StartUploadResponse>.Fail(409, "job_not_assigned");
if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Outcome<StartUploadResponse>.Fail(400, "invalid_file_name");
if (!UploadRules.IsSha256(request.Sha256)) return Outcome<StartUploadResponse>.Fail(400, "invalid_hash");
if (request.Size <= 0) return Outcome<StartUploadResponse>.Fail(400, "bad_request", "size");
if (request.Size > options.MaxUploadBytes)
return Outcome<StartUploadResponse>.Fail(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture));
if (string.IsNullOrWhiteSpace(request.PresetName)) return Outcome<StartUploadResponse>.Fail(400, "bad_request", "presetName");
var sha = request.Sha256.ToLowerInvariant();
if (await db.Uploads.FindByJobAsync(jobId, cancellationToken) is { State: UploadState.Receiving } open && open.Sha256 == sha && open.Size == request.Size
&& store.Exists(open.Id))
return Started(open);
if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size) return Outcome<StartUploadResponse>.Fail(507, "insufficient_storage");
var now = time.GetUtcNow();
var upload = await db.Uploads.CreateAsync(new Upload
{
Id = Guid.CreateVersion7(),
UserId = caller.Id,
SequenceId = job.TargetIds.Count == 1 ? job.TargetIds[0] : null,
FileName = fileName,
Size = request.Size,
Sha256 = sha,
ChunkSize = options.ChunkSizeBytes,
State = UploadState.Receiving,
Created = now,
UpdatedAt = now,
JobId = jobId,
RenderJson = JsonSerializer.Serialize(request with { FileName = fileName, Sha256 = sha }, ProtocolJson.Default.RenderResultRequest),
}, cancellationToken);
store.Create(upload.Id, upload.Size);
return Started(upload);
}
/// <summary>Verifies the uploaded output, files it as a render of the sequence and marks the job done.</summary>
public async Task<Outcome<RenderDto>> CompleteResultAsync(Caller caller, Guid nodeId, Guid jobId, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<RenderDto>.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<RenderDto>.Fail(409, "job_not_assigned");
var upload = await db.Uploads.FindByJobAsync(jobId, cancellationToken);
if (upload is not { State: UploadState.Receiving } || upload.RenderJson is null) return Outcome<RenderDto>.Fail(409, "upload_state");
if (!upload.IsComplete)
return Outcome<RenderDto>.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture));
if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken))
return Outcome<RenderDto>.Fail(409, "upload_state");
if (await uploads.StageVerifiedAsync(upload, cancellationToken) is not { } staged) return Outcome<RenderDto>.Fail(422, "hash_mismatch");
var output = JsonSerializer.Deserialize(upload.RenderJson, ProtocolJson.Default.RenderResultRequest)!;
var sequence = await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken);
var render = sequence is null ? null : await renders.CommitAsync(sequence, staged, output, node.ClientName, cancellationToken);
if (render is null)
{
UploadStore.TryDelete(staged);
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None);
store.Delete(upload.Id);
await FinishAsync(node, job, JobState.Failed, "The sequence has no folder in the archive.", CancellationToken.None);
return Outcome<RenderDto>.Fail(409, "upload_state", "sequence");
}
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, cancellationToken: CancellationToken.None);
store.Delete(upload.Id);
await FinishAsync(node, job, JobState.Done, null, CancellationToken.None);
if (await db.Sequences.GetAsync(sequence!.Id, CancellationToken.None) is { } updated)
events.Publish(new SequenceChangedEvent(updated.Id, "renders", updated.FramesVersion, updated.SettingsVersion));
return Dto.Render(render);
}
/// <summary>Running-job owner key stored in <c>jobs.node_id</c>.</summary>
public static string NodeKey(RenderNode node) => node.Id.ToString();
/// <summary>
/// Gives the jobs running on <paramref name="node"/> back to the queue with <paramref name="reason"/> in their log
/// (checkpoint and progress kept); a job that already lost its node <see cref="RenderNodeMonitor.MaxAttempts"/> times
/// fails instead. Returns how many went back.
/// </summary>
internal static async Task<int> ReleaseJobsAsync(ChronosDatabase db, JobEngineHost jobs, RenderNode node, string reason, CancellationToken cancellationToken)
{
var key = NodeKey(node);
var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken);
foreach (var job in lost)
{
var line = AppendLog(job.LogTail, reason);
if (job.Attempts >= RenderNodeMonitor.MaxAttempts)
await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken);
else
await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken);
}
var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken);
foreach (var job in lost)
{
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current);
}
return requeued;
}
private async Task<RenderNode?> OwnedNodeAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) =>
await db.RenderNodes.GetAsync(nodeId, cancellationToken) is { } node && node.UserId == caller.Id ? node : null;
private async Task<Job?> AssignedJobAsync(RenderNode node, Guid jobId, CancellationToken cancellationToken) =>
await db.Jobs.GetAsync(jobId, cancellationToken) is { State: JobState.Running } job && job.NodeId == NodeKey(node) ? job : null;
private async Task<bool> FinishAsync(RenderNode node, Job job, JobState state, string? error, CancellationToken cancellationToken)
{
var tail = error is null ? null : AppendLog(job.LogTail, error);
if (!await db.Jobs.FinishNodeJobAsync(job.Id, NodeKey(node), state, error, tail, cancellationToken)) return false;
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } finished) jobs.Publish(finished);
await TouchAsync(node, cancellationToken);
return true;
}
/// <summary>Records a sign of life and publishes the node's state when it changed.</summary>
private async Task TouchAsync(RenderNode node, CancellationToken cancellationToken)
{
var state = (await db.Jobs.ListRunningByNodeAsync(NodeKey(node), cancellationToken)).Count > 0 ? Busy : Idle;
await db.RenderNodes.HeartbeatAsync(node.Id, state, time.GetUtcNow(), cancellationToken);
if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state));
}
/// <summary>
/// The node has an encoder for the job's codec that handles its resolution: the encoder's own limit, else the node's
/// (0 = no announced limit).
/// </summary>
internal static bool CanEncode(RegisterNodeRequest node, RenderJobParameters job) =>
node.Encoders.Any(e => string.Equals(e.Codec, job.Codec, StringComparison.OrdinalIgnoreCase)
&& Fits(job.Width, e.MaxWidth > 0 ? e.MaxWidth : node.MaxWidth)
&& Fits(job.Height, e.MaxHeight > 0 ? e.MaxHeight : node.MaxHeight));
private static bool Fits(int size, int limit) => limit <= 0 || size <= limit;
internal static string? AppendLog(string? tail, string? line)
{
if (string.IsNullOrWhiteSpace(line)) return tail;
var lines = (tail ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries).ToList();
lines.Add(string.Create(CultureInfo.InvariantCulture, $"{DateTime.UtcNow:HH:mm:ss} {line.Trim()[..Math.Min(line.Trim().Length, 300)]}"));
return string.Join('\n', lines.TakeLast(LogTailLines));
}
private static StartUploadResponse Started(Upload upload) =>
new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null);
}