Files
Chronos/src/Chronos.Server/Jobs/RenderNodeService.cs
T
Alby96andClaude Opus 5.5 d560b41b81 feat(server): show the Unraid array's free space and check the archive volume before filing uploads
On Unraid the archive is a user share (FUSE shfs), which reports the share's
primary storage, often the cache, while the mover sends the photos to the
array: the admin panel showed the cache's free space (owner review 2, R2).

- New option SpacePath (server.json) / CHRONOS_SPACE_PATH: the admin panel
  shows that folder's volume; empty, or a folder that cannot be measured, falls
  back to the archive's volume (one warning, at start, not per request).
- VolumeSpace measures folders (DriveInfo; the path root on Windows) and reads
  the filesystem type and mount from /proc/self/mountinfo on Linux (longest
  mount point on segment boundaries, octal escapes, optional fields, the top of
  stacked mounts), since DriveFormat calls an shfs share only "fuse". It is an
  injectable service, so tests simulate a full volume; the /data checks use it.
- ArchiveStatsDto keeps FreeBytes/CapacityBytes for the shown volume and gains
  optional trailing members: the write volume's figures (when a space path is
  in use and they differ), the filesystem type and UsesSpacePath.
- Admin panel (dashboard and archive): a tooltip on the storage figures names
  the volume and its filesystem and, on an shfs share without a space path,
  says they are the primary storage and to map the array view to /space; a
  "Write volume" line shows the archive's own volume when it differs.
- Completing an upload, a render upload or a render node's result checks the
  archive volume for its size plus the 64 MiB reserve before the staging copy
  and answers 507 insufficient_storage, the upload still receiving.
- Diagnostics list the archive and space volumes with filesystem and mount.
- Unraid template: optional read-only "Array view" /space (default
  /mnt/user0/Timelapse) and "Space path" CHRONOS_SPACE_PATH=/space; compose and
  the releases README describe the same as an option.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 21:58:27 +01:00

268 lines
16 KiB
C#

using System.Globalization;
using System.Text.Json;
using Chronos.Core.Model;
using Chronos.Data;
using Chronos.Data.Jobs;
using Chronos.Data.Repositories;
using Chronos.Protocol;
using Chronos.Server.Api;
using Chronos.Server.Configuration;
using Chronos.Server.Events;
using Chronos.Server.Security;
using Chronos.Server.Storage;
using UploadState = Chronos.Data.Repositories.UploadState;
namespace Chronos.Server.Jobs;
/// <summary>
/// Render nodes (BRIEF §14.7): clients with the render-node right register their encoders, heartbeat, claim queued
/// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of
/// the sequence. A claimed job lists every frame with its flags (a node renders none missing, corrupt or excluded) and
/// the proxies the server asks for (<see cref="ServerProxies"/>), which the node sends from the originals it fetched.
/// A node that stops heart-beating loses its jobs to the queue (<see cref="RenderNodeMonitor"/>).
/// </summary>
public sealed class RenderNodeService(
ChronosDatabase db,
ServerOptions options,
ServerArchive archive,
UploadService uploads,
UploadStore store,
RenderStore renders,
JobEngineHost jobs,
EventHub events,
VolumeSpace volumes,
TimeProvider time)
{
public const string Idle = "idle";
public const string Busy = "busy";
public const string Offline = "offline";
private const int LogTailLines = 40;
public async Task<Outcome<RenderNodeDto>> RegisterAsync(Caller caller, RegisterNodeRequest request, CancellationToken cancellationToken)
{
var name = request.ClientName?.Trim();
if (string.IsNullOrEmpty(name) || name.Length > 100) return Outcome<RenderNodeDto>.Fail(400, "bad_request", "clientName");
if (request.Encoders is null || request.Encoders.Count > 64 || request.Encoders.Any(e => string.IsNullOrWhiteSpace(e?.Codec)))
return Outcome<RenderNodeDto>.Fail(400, "bad_request", "encoders");
var existing = (await db.RenderNodes.ListAsync(cancellationToken)).FirstOrDefault(n => n.UserId == caller.Id && n.ClientName == name);
// A node registers when it starts, holding no job: what the server still lists as running on it was abandoned
// by its previous run (stopped, restarted or crashed within the heartbeat timeout) and goes back to the queue.
if (existing is not null) await ReleaseJobsAsync(db, jobs, existing, $"Render node {name} restarted.", cancellationToken);
var node = new RenderNode
{
Id = existing?.Id ?? Guid.CreateVersion7(),
UserId = caller.Id,
ClientName = name,
CapabilitiesJson = JsonSerializer.Serialize(request with { ClientName = name }, ProtocolJson.Default.RegisterNodeRequest),
LastHeartbeat = time.GetUtcNow(),
State = Idle,
};
await db.RenderNodes.RegisterAsync(node, cancellationToken);
events.Publish(new NodeStateEvent(node.Id, node.ClientName, node.State));
return Dto.Node(node, caller.User.Username);
}
public async Task<Outcome<bool>> HeartbeatAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
await TouchAsync(node, cancellationToken);
return true;
}
/// <summary>The next queued render job this node can encode (highest priority, then oldest), or null when none fits.</summary>
public async Task<Outcome<ClaimedJob?>> ClaimAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<ClaimedJob?>.Fail(404, "not_found", "node");
var capabilities = Dto.Capabilities(node);
foreach (var queued in await db.Jobs.ListQueuedAsync(JobKind.Render, 50, cancellationToken))
{
// A render the server's own engine started (it left a checkpoint) resumes there: only it knows that attempt's
// leftovers and the batch entries it recorded.
if (queued.Checkpoint is not null) continue;
if (JobEndpoints.ParseRender(queued.ParametersJson) is not { } parameters || capabilities is null || !CanEncode(capabilities, parameters)) continue;
if (await db.Jobs.TryClaimAsync(queued.Id, NodeKey(node), cancellationToken) is not { } job) continue;
jobs.Publish(job);
if (job.TargetIds.Count != 1 || await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken) is not { } sequence)
{
await FinishAsync(node, job, JobState.Failed, "The sequence no longer exists.", cancellationToken);
continue;
}
await TouchAsync(node, cancellationToken);
var spec = ServerProxies.Spec(archive.Current);
return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson,
parameters.UseProxies, await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, cancellationToken), spec);
}
await TouchAsync(node, cancellationToken);
return (ClaimedJob?)null;
}
public async Task<Outcome<bool>> ProgressAsync(Caller caller, Guid nodeId, Guid jobId, NodeProgress progress, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<bool>.Fail(409, "job_not_assigned");
// A node reports any rate (a tiny one when it stalls): only a usable time left is stored and broadcast.
var rate = progress.ItemsPerSecond is { } r && double.IsFinite(r) && r > 0 ? r : (double?)null;
var eta = JobEta.Of(progress.Total - progress.Done, rate);
var tail = AppendLog(job.LogTail, progress.LogLine);
if (!await db.Jobs.UpdateNodeProgressAsync(jobId, NodeKey(node), progress.Done, progress.Total, rate, eta, tail, cancellationToken))
return Outcome<bool>.Fail(409, "job_not_assigned");
await TouchAsync(node, cancellationToken);
events.Publish(new JobProgressEvent(jobId, progress.Done, progress.Total, rate, eta?.TotalSeconds));
return true;
}
public async Task<Outcome<bool>> FailAsync(Caller caller, Guid nodeId, Guid jobId, NodeJobFailure failure, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<bool>.Fail(409, "job_not_assigned");
var error = string.IsNullOrWhiteSpace(failure.Error) ? "Render failed on the node." : failure.Error.Trim()[..Math.Min(failure.Error.Trim().Length, 500)];
return await FinishAsync(node, job, JobState.Failed, error, cancellationToken) ? true : Outcome<bool>.Fail(409, "job_not_assigned");
}
/// <summary>Starts (or resumes) the chunked upload of a job's output.</summary>
public async Task<Outcome<StartUploadResponse>> StartResultAsync(Caller caller, Guid nodeId, Guid jobId, RenderResultRequest request, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<StartUploadResponse>.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<StartUploadResponse>.Fail(409, "job_not_assigned");
if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Outcome<StartUploadResponse>.Fail(400, "invalid_file_name");
if (!UploadRules.IsSha256(request.Sha256)) return Outcome<StartUploadResponse>.Fail(400, "invalid_hash");
if (request.Size <= 0) return Outcome<StartUploadResponse>.Fail(400, "bad_request", "size");
if (request.Size > options.MaxUploadBytes)
return Outcome<StartUploadResponse>.Fail(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture));
if (string.IsNullOrWhiteSpace(request.PresetName)) return Outcome<StartUploadResponse>.Fail(400, "bad_request", "presetName");
var sha = request.Sha256.ToLowerInvariant();
if (await db.Uploads.FindByJobAsync(jobId, cancellationToken) is { State: UploadState.Receiving } open && open.Sha256 == sha && open.Size == request.Size
&& store.Exists(open.Id))
return Started(open);
if (volumes.Measure(options.UploadsPath) is { } uploadVolume && uploadVolume.FreeBytes < request.Size)
return Outcome<StartUploadResponse>.Fail(507, "insufficient_storage");
var now = time.GetUtcNow();
var upload = await db.Uploads.CreateAsync(new Upload
{
Id = Guid.CreateVersion7(),
UserId = caller.Id,
SequenceId = job.TargetIds.Count == 1 ? job.TargetIds[0] : null,
FileName = fileName,
Size = request.Size,
Sha256 = sha,
ChunkSize = options.ChunkSizeBytes,
State = UploadState.Receiving,
Created = now,
UpdatedAt = now,
JobId = jobId,
RenderJson = JsonSerializer.Serialize(request with { FileName = fileName, Sha256 = sha }, ProtocolJson.Default.RenderResultRequest),
}, cancellationToken);
store.Create(upload.Id, upload.Size);
return Started(upload);
}
/// <summary>Verifies the uploaded output, files it as a render of the sequence and marks the job done.</summary>
public async Task<Outcome<RenderDto>> CompleteResultAsync(Caller caller, Guid nodeId, Guid jobId, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<RenderDto>.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<RenderDto>.Fail(409, "job_not_assigned");
var upload = await db.Uploads.FindByJobAsync(jobId, cancellationToken);
if (upload is not { State: UploadState.Receiving } || upload.RenderJson is null) return Outcome<RenderDto>.Fail(409, "upload_state");
if (!upload.IsComplete)
return Outcome<RenderDto>.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture));
if (!uploads.ArchiveHasRoom(upload)) return Outcome<RenderDto>.Fail(507, "insufficient_storage");
if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken))
return Outcome<RenderDto>.Fail(409, "upload_state");
if (await uploads.StageVerifiedAsync(upload, cancellationToken) is not { } staged) return Outcome<RenderDto>.Fail(422, "hash_mismatch");
var output = JsonSerializer.Deserialize(upload.RenderJson, ProtocolJson.Default.RenderResultRequest)!;
var sequence = await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken);
var render = sequence is null ? null : await renders.CommitAsync(sequence, staged, output, node.ClientName, cancellationToken);
if (render is null)
{
UploadStore.TryDelete(staged);
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None);
store.Delete(upload.Id);
await FinishAsync(node, job, JobState.Failed, "The sequence has no folder in the archive.", CancellationToken.None);
return Outcome<RenderDto>.Fail(409, "upload_state", "sequence");
}
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, cancellationToken: CancellationToken.None);
store.Delete(upload.Id);
await FinishAsync(node, job, JobState.Done, null, CancellationToken.None);
if (await db.Sequences.GetAsync(sequence!.Id, CancellationToken.None) is { } updated)
events.Publish(new SequenceChangedEvent(updated.Id, "renders", updated.FramesVersion, updated.SettingsVersion));
return Dto.Render(render);
}
/// <summary>Running-job owner key stored in <c>jobs.node_id</c>.</summary>
public static string NodeKey(RenderNode node) => node.Id.ToString();
/// <summary>
/// Gives the jobs running on <paramref name="node"/> back to the queue with <paramref name="reason"/> in their log
/// (checkpoint and progress kept); a job that already lost its node <see cref="RenderNodeMonitor.MaxAttempts"/> times
/// fails instead. Returns how many went back.
/// </summary>
internal static async Task<int> ReleaseJobsAsync(ChronosDatabase db, JobEngineHost jobs, RenderNode node, string reason, CancellationToken cancellationToken)
{
var key = NodeKey(node);
var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken);
foreach (var job in lost)
{
var line = AppendLog(job.LogTail, reason);
if (job.Attempts >= RenderNodeMonitor.MaxAttempts)
await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken);
else
await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken);
}
var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken);
foreach (var job in lost)
{
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current);
}
return requeued;
}
private async Task<RenderNode?> OwnedNodeAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) =>
await db.RenderNodes.GetAsync(nodeId, cancellationToken) is { } node && node.UserId == caller.Id ? node : null;
private async Task<Job?> AssignedJobAsync(RenderNode node, Guid jobId, CancellationToken cancellationToken) =>
await db.Jobs.GetAsync(jobId, cancellationToken) is { State: JobState.Running } job && job.NodeId == NodeKey(node) ? job : null;
private async Task<bool> FinishAsync(RenderNode node, Job job, JobState state, string? error, CancellationToken cancellationToken)
{
var tail = error is null ? null : AppendLog(job.LogTail, error);
if (!await db.Jobs.FinishNodeJobAsync(job.Id, NodeKey(node), state, error, tail, cancellationToken)) return false;
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } finished) jobs.Publish(finished);
await TouchAsync(node, cancellationToken);
return true;
}
/// <summary>Records a sign of life and publishes the node's state when it changed.</summary>
private async Task TouchAsync(RenderNode node, CancellationToken cancellationToken)
{
var state = (await db.Jobs.ListRunningByNodeAsync(NodeKey(node), cancellationToken)).Count > 0 ? Busy : Idle;
await db.RenderNodes.HeartbeatAsync(node.Id, state, time.GetUtcNow(), cancellationToken);
if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state));
}
/// <summary>
/// The node has an encoder for the job's codec that handles its resolution: the encoder's own limit, else the node's
/// (0 = no announced limit).
/// </summary>
internal static bool CanEncode(RegisterNodeRequest node, RenderJobParameters job) =>
node.Encoders.Any(e => string.Equals(e.Codec, job.Codec, StringComparison.OrdinalIgnoreCase)
&& Fits(job.Width, e.MaxWidth > 0 ? e.MaxWidth : node.MaxWidth)
&& Fits(job.Height, e.MaxHeight > 0 ? e.MaxHeight : node.MaxHeight));
private static bool Fits(int size, int limit) => limit <= 0 || size <= limit;
internal static string? AppendLog(string? tail, string? line)
{
if (string.IsNullOrWhiteSpace(line)) return tail;
var lines = (tail ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries).ToList();
lines.Add(string.Create(CultureInfo.InvariantCulture, $"{DateTime.UtcNow:HH:mm:ss} {line.Trim()[..Math.Min(line.Trim().Length, 300)]}"));
return string.Join('\n', lines.TakeLast(LogTailLines));
}
private static StartUploadResponse Started(Upload upload) =>
new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null);
}