A node announced one size limit, the largest of all its encoders (managed MJPEG at 65535 px), so it claimed H.264 jobs larger than its H.264 encoder takes. Each offered encoder now carries its own limit and the server matches codec and size per encoder; the node-wide limit is the largest offered one, for encoders that announce none. A hand-written batch with null or blank entries no longer throws when a job is posted or examined. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
275 lines
17 KiB
C#
275 lines
17 KiB
C#
using System.Globalization;
|
|
using System.Text.Json;
|
|
using Titano.Core.Model;
|
|
using Titano.Data;
|
|
using Titano.Data.Repositories;
|
|
using Titano.Protocol;
|
|
using Titano.Server.Api;
|
|
using Titano.Server.Configuration;
|
|
using Titano.Server.Events;
|
|
using Titano.Server.Security;
|
|
using Titano.Server.Storage;
|
|
using UploadState = Titano.Data.Repositories.UploadState;
|
|
|
|
namespace Titano.Server.Jobs;
|
|
|
|
/// <summary>
|
|
/// Render nodes (BRIEF §14.7): clients with the render-node right register their encoders, heartbeat, claim queued
|
|
/// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of
|
|
/// the sequence. A node that stops heart-beating loses its jobs to the queue (<see cref="RenderNodeMonitor"/>).
|
|
/// </summary>
|
|
public sealed class RenderNodeService(
|
|
TitanoDatabase db,
|
|
ServerOptions options,
|
|
UploadService uploads,
|
|
UploadStore store,
|
|
RenderStore renders,
|
|
JobEngineHost jobs,
|
|
EventHub events,
|
|
TimeProvider time)
|
|
{
|
|
public const string Idle = "idle";
|
|
public const string Busy = "busy";
|
|
public const string Offline = "offline";
|
|
private const int LogTailLines = 40;
|
|
|
|
public async Task<Outcome<RenderNodeDto>> RegisterAsync(Caller caller, RegisterNodeRequest request, CancellationToken cancellationToken)
|
|
{
|
|
var name = request.ClientName?.Trim();
|
|
if (string.IsNullOrEmpty(name) || name.Length > 100) return Outcome<RenderNodeDto>.Fail(400, "bad_request", "clientName");
|
|
if (request.Encoders is null || request.Encoders.Count > 64 || request.Encoders.Any(e => string.IsNullOrWhiteSpace(e?.Codec)))
|
|
return Outcome<RenderNodeDto>.Fail(400, "bad_request", "encoders");
|
|
|
|
var existing = (await db.RenderNodes.ListAsync(cancellationToken)).FirstOrDefault(n => n.UserId == caller.Id && n.ClientName == name);
|
|
// A node registers when it starts, holding no job: what the server still lists as running on it was abandoned
|
|
// by its previous run (stopped, restarted or crashed within the heartbeat timeout) and goes back to the queue.
|
|
if (existing is not null) await ReleaseJobsAsync(db, jobs, existing, $"Render node {name} restarted.", cancellationToken);
|
|
var node = new RenderNode
|
|
{
|
|
Id = existing?.Id ?? Guid.CreateVersion7(),
|
|
UserId = caller.Id,
|
|
ClientName = name,
|
|
CapabilitiesJson = JsonSerializer.Serialize(request with { ClientName = name }, ProtocolJson.Default.RegisterNodeRequest),
|
|
LastHeartbeat = time.GetUtcNow(),
|
|
State = Idle,
|
|
};
|
|
await db.RenderNodes.RegisterAsync(node, cancellationToken);
|
|
events.Publish(new NodeStateEvent(node.Id, node.ClientName, node.State));
|
|
return Dto.Node(node, caller.User.Username);
|
|
}
|
|
|
|
public async Task<Outcome<bool>> HeartbeatAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
|
|
await TouchAsync(node, cancellationToken);
|
|
return true;
|
|
}
|
|
|
|
/// <summary>The next queued render job this node can encode (highest priority, then oldest), or null when none fits.</summary>
|
|
public async Task<Outcome<ClaimedJob?>> ClaimAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<ClaimedJob?>.Fail(404, "not_found", "node");
|
|
var capabilities = Dto.Capabilities(node);
|
|
foreach (var queued in await db.Jobs.ListQueuedAsync(JobKind.Render, 50, cancellationToken))
|
|
{
|
|
// A render the server's own engine started (it left a checkpoint) resumes there: only it knows that attempt's
|
|
// leftovers and the batch entries it recorded.
|
|
if (queued.Checkpoint is not null) continue;
|
|
if (JobEndpoints.ParseRender(queued.ParametersJson) is not { } parameters || capabilities is null || !CanEncode(capabilities, parameters)) continue;
|
|
if (await db.Jobs.TryClaimAsync(queued.Id, NodeKey(node), cancellationToken) is not { } job) continue;
|
|
jobs.Publish(job);
|
|
if (job.TargetIds.Count != 1 || await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken) is not { } sequence)
|
|
{
|
|
await FinishAsync(node, job, JobState.Failed, "The sequence no longer exists.", cancellationToken);
|
|
continue;
|
|
}
|
|
await TouchAsync(node, cancellationToken);
|
|
return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson,
|
|
parameters.UseProxies, await ManifestFramesAsync(sequence, cancellationToken));
|
|
}
|
|
await TouchAsync(node, cancellationToken);
|
|
return (ClaimedJob?)null;
|
|
}
|
|
|
|
public async Task<Outcome<bool>> ProgressAsync(Caller caller, Guid nodeId, Guid jobId, NodeProgress progress, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
|
|
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<bool>.Fail(409, "job_not_assigned");
|
|
double? eta = progress.ItemsPerSecond is > 0 && progress.Total > progress.Done ? (progress.Total - progress.Done) / progress.ItemsPerSecond.Value : null;
|
|
var tail = AppendLog(job.LogTail, progress.LogLine);
|
|
if (!await db.Jobs.UpdateNodeProgressAsync(jobId, NodeKey(node), progress.Done, progress.Total, progress.ItemsPerSecond,
|
|
eta is { } seconds ? TimeSpan.FromSeconds(seconds) : null, tail, cancellationToken))
|
|
return Outcome<bool>.Fail(409, "job_not_assigned");
|
|
await TouchAsync(node, cancellationToken);
|
|
events.Publish(new JobProgressEvent(jobId, progress.Done, progress.Total, progress.ItemsPerSecond, eta));
|
|
return true;
|
|
}
|
|
|
|
public async Task<Outcome<bool>> FailAsync(Caller caller, Guid nodeId, Guid jobId, NodeJobFailure failure, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
|
|
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<bool>.Fail(409, "job_not_assigned");
|
|
var error = string.IsNullOrWhiteSpace(failure.Error) ? "Render failed on the node." : failure.Error.Trim()[..Math.Min(failure.Error.Trim().Length, 500)];
|
|
return await FinishAsync(node, job, JobState.Failed, error, cancellationToken) ? true : Outcome<bool>.Fail(409, "job_not_assigned");
|
|
}
|
|
|
|
/// <summary>Starts (or resumes) the chunked upload of a job's output.</summary>
|
|
public async Task<Outcome<StartUploadResponse>> StartResultAsync(Caller caller, Guid nodeId, Guid jobId, RenderResultRequest request, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<StartUploadResponse>.Fail(404, "not_found", "node");
|
|
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<StartUploadResponse>.Fail(409, "job_not_assigned");
|
|
if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Outcome<StartUploadResponse>.Fail(400, "invalid_file_name");
|
|
if (!UploadRules.IsSha256(request.Sha256)) return Outcome<StartUploadResponse>.Fail(400, "invalid_hash");
|
|
if (request.Size <= 0) return Outcome<StartUploadResponse>.Fail(400, "bad_request", "size");
|
|
if (request.Size > options.MaxUploadBytes)
|
|
return Outcome<StartUploadResponse>.Fail(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture));
|
|
if (string.IsNullOrWhiteSpace(request.PresetName)) return Outcome<StartUploadResponse>.Fail(400, "bad_request", "presetName");
|
|
|
|
var sha = request.Sha256.ToLowerInvariant();
|
|
if (await db.Uploads.FindByJobAsync(jobId, cancellationToken) is { State: UploadState.Receiving } open && open.Sha256 == sha && open.Size == request.Size
|
|
&& store.Exists(open.Id))
|
|
return Started(open);
|
|
if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size) return Outcome<StartUploadResponse>.Fail(507, "insufficient_storage");
|
|
|
|
var now = time.GetUtcNow();
|
|
var upload = await db.Uploads.CreateAsync(new Upload
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
UserId = caller.Id,
|
|
SequenceId = job.TargetIds.Count == 1 ? job.TargetIds[0] : null,
|
|
FileName = fileName,
|
|
Size = request.Size,
|
|
Sha256 = sha,
|
|
ChunkSize = options.ChunkSizeBytes,
|
|
State = UploadState.Receiving,
|
|
Created = now,
|
|
UpdatedAt = now,
|
|
JobId = jobId,
|
|
RenderJson = JsonSerializer.Serialize(request with { FileName = fileName, Sha256 = sha }, ProtocolJson.Default.RenderResultRequest),
|
|
}, cancellationToken);
|
|
store.Create(upload.Id, upload.Size);
|
|
return Started(upload);
|
|
}
|
|
|
|
/// <summary>Verifies the uploaded output, files it as a render of the sequence and marks the job done.</summary>
|
|
public async Task<Outcome<RenderDto>> CompleteResultAsync(Caller caller, Guid nodeId, Guid jobId, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<RenderDto>.Fail(404, "not_found", "node");
|
|
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<RenderDto>.Fail(409, "job_not_assigned");
|
|
var upload = await db.Uploads.FindByJobAsync(jobId, cancellationToken);
|
|
if (upload is not { State: UploadState.Receiving } || upload.RenderJson is null) return Outcome<RenderDto>.Fail(409, "upload_state");
|
|
if (!upload.IsComplete)
|
|
return Outcome<RenderDto>.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture));
|
|
if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken))
|
|
return Outcome<RenderDto>.Fail(409, "upload_state");
|
|
|
|
if (await uploads.StageVerifiedAsync(upload, cancellationToken) is not { } staged) return Outcome<RenderDto>.Fail(422, "hash_mismatch");
|
|
var output = JsonSerializer.Deserialize(upload.RenderJson, ProtocolJson.Default.RenderResultRequest)!;
|
|
var sequence = await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken);
|
|
var render = sequence is null ? null : await renders.CommitAsync(sequence, staged, output, node.ClientName, cancellationToken);
|
|
if (render is null)
|
|
{
|
|
UploadStore.TryDelete(staged);
|
|
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None);
|
|
store.Delete(upload.Id);
|
|
await FinishAsync(node, job, JobState.Failed, "The sequence has no folder in the archive.", CancellationToken.None);
|
|
return Outcome<RenderDto>.Fail(409, "upload_state", "sequence");
|
|
}
|
|
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, cancellationToken: CancellationToken.None);
|
|
store.Delete(upload.Id);
|
|
await FinishAsync(node, job, JobState.Done, null, CancellationToken.None);
|
|
if (await db.Sequences.GetAsync(sequence!.Id, CancellationToken.None) is { } updated)
|
|
events.Publish(new SequenceChangedEvent(updated.Id, "renders", updated.FramesVersion, updated.SettingsVersion));
|
|
return Dto.Render(render);
|
|
}
|
|
|
|
/// <summary>Running-job owner key stored in <c>jobs.node_id</c>.</summary>
|
|
public static string NodeKey(RenderNode node) => node.Id.ToString();
|
|
|
|
/// <summary>
|
|
/// Gives the jobs running on <paramref name="node"/> back to the queue with <paramref name="reason"/> in their log
|
|
/// (checkpoint and progress kept); a job that already lost its node <see cref="RenderNodeMonitor.MaxAttempts"/> times
|
|
/// fails instead. Returns how many went back.
|
|
/// </summary>
|
|
internal static async Task<int> ReleaseJobsAsync(TitanoDatabase db, JobEngineHost jobs, RenderNode node, string reason, CancellationToken cancellationToken)
|
|
{
|
|
var key = NodeKey(node);
|
|
var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken);
|
|
foreach (var job in lost)
|
|
{
|
|
var line = AppendLog(job.LogTail, reason);
|
|
if (job.Attempts >= RenderNodeMonitor.MaxAttempts)
|
|
await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken);
|
|
else
|
|
await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken);
|
|
}
|
|
var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken);
|
|
foreach (var job in lost)
|
|
{
|
|
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current);
|
|
}
|
|
return requeued;
|
|
}
|
|
|
|
private async Task<RenderNode?> OwnedNodeAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) =>
|
|
await db.RenderNodes.GetAsync(nodeId, cancellationToken) is { } node && node.UserId == caller.Id ? node : null;
|
|
|
|
private async Task<Job?> AssignedJobAsync(RenderNode node, Guid jobId, CancellationToken cancellationToken) =>
|
|
await db.Jobs.GetAsync(jobId, cancellationToken) is { State: JobState.Running } job && job.NodeId == NodeKey(node) ? job : null;
|
|
|
|
private async Task<bool> FinishAsync(RenderNode node, Job job, JobState state, string? error, CancellationToken cancellationToken)
|
|
{
|
|
var tail = error is null ? null : AppendLog(job.LogTail, error);
|
|
if (!await db.Jobs.FinishNodeJobAsync(job.Id, NodeKey(node), state, error, tail, cancellationToken)) return false;
|
|
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } finished) jobs.Publish(finished);
|
|
await TouchAsync(node, cancellationToken);
|
|
return true;
|
|
}
|
|
|
|
/// <summary>Records a sign of life and publishes the node's state when it changed.</summary>
|
|
private async Task TouchAsync(RenderNode node, CancellationToken cancellationToken)
|
|
{
|
|
var state = (await db.Jobs.ListRunningByNodeAsync(NodeKey(node), cancellationToken)).Count > 0 ? Busy : Idle;
|
|
await db.RenderNodes.HeartbeatAsync(node.Id, state, time.GetUtcNow(), cancellationToken);
|
|
if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state));
|
|
}
|
|
|
|
private async Task<List<ManifestFrame>> ManifestFramesAsync(Sequence sequence, CancellationToken cancellationToken)
|
|
{
|
|
var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
|
|
var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt);
|
|
var frames = new List<ManifestFrame>(sequence.FrameCount);
|
|
var after = -1;
|
|
while (true)
|
|
{
|
|
var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, 1000, cancellationToken);
|
|
frames.AddRange(batch.Where(f => !f.Flags.HasFlag(FrameFlags.Excluded))
|
|
.Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb)));
|
|
if (batch.Count < 1000) return frames;
|
|
after = batch[^1].Index;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The node has an encoder for the job's codec that handles its resolution: the encoder's own limit, else the node's
|
|
/// (0 = no announced limit).
|
|
/// </summary>
|
|
internal static bool CanEncode(RegisterNodeRequest node, RenderJobParameters job) =>
|
|
node.Encoders.Any(e => string.Equals(e.Codec, job.Codec, StringComparison.OrdinalIgnoreCase)
|
|
&& Fits(job.Width, e.MaxWidth > 0 ? e.MaxWidth : node.MaxWidth)
|
|
&& Fits(job.Height, e.MaxHeight > 0 ? e.MaxHeight : node.MaxHeight));
|
|
|
|
private static bool Fits(int size, int limit) => limit <= 0 || size <= limit;
|
|
|
|
internal static string? AppendLog(string? tail, string? line)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(line)) return tail;
|
|
var lines = (tail ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries).ToList();
|
|
lines.Add(string.Create(CultureInfo.InvariantCulture, $"{DateTime.UtcNow:HH:mm:ss} {line.Trim()[..Math.Min(line.Trim().Length, 300)]}"));
|
|
return string.Join('\n', lines.TakeLast(LogTailLines));
|
|
}
|
|
|
|
private static StartUploadResponse Started(Upload upload) =>
|
|
new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null);
|
|
}
|