using System.Globalization;
using System.Text.Json;
using Titano.Core.Model;
using Titano.Data;
using Titano.Data.Repositories;
using Titano.Protocol;
using Titano.Server.Api;
using Titano.Server.Configuration;
using Titano.Server.Events;
using Titano.Server.Security;
using Titano.Server.Storage;
using UploadState = Titano.Data.Repositories.UploadState;
namespace Titano.Server.Jobs;
///
/// Render nodes (BRIEF ยง14.7): clients with the render-node right register their encoders, heartbeat, claim queued
/// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of
/// the sequence. A node that stops heart-beating loses its jobs to the queue ().
///
public sealed class RenderNodeService(
TitanoDatabase db,
ServerOptions options,
UploadService uploads,
UploadStore store,
RenderStore renders,
JobEngineHost jobs,
EventHub events,
TimeProvider time)
{
public const string Idle = "idle";
public const string Busy = "busy";
public const string Offline = "offline";
private const int LogTailLines = 40;
public async Task> RegisterAsync(Caller caller, RegisterNodeRequest request, CancellationToken cancellationToken)
{
var name = request.ClientName?.Trim();
if (string.IsNullOrEmpty(name) || name.Length > 100) return Outcome.Fail(400, "bad_request", "clientName");
if (request.Encoders is null || request.Encoders.Count > 64 || request.Encoders.Any(e => string.IsNullOrWhiteSpace(e?.Codec)))
return Outcome.Fail(400, "bad_request", "encoders");
var existing = (await db.RenderNodes.ListAsync(cancellationToken)).FirstOrDefault(n => n.UserId == caller.Id && n.ClientName == name);
// A node registers when it starts, holding no job: what the server still lists as running on it was abandoned
// by its previous run (stopped, restarted or crashed within the heartbeat timeout) and goes back to the queue.
if (existing is not null) await ReleaseJobsAsync(db, jobs, existing, $"Render node {name} restarted.", cancellationToken);
var node = new RenderNode
{
Id = existing?.Id ?? Guid.CreateVersion7(),
UserId = caller.Id,
ClientName = name,
CapabilitiesJson = JsonSerializer.Serialize(request with { ClientName = name }, ProtocolJson.Default.RegisterNodeRequest),
LastHeartbeat = time.GetUtcNow(),
State = Idle,
};
await db.RenderNodes.RegisterAsync(node, cancellationToken);
events.Publish(new NodeStateEvent(node.Id, node.ClientName, node.State));
return Dto.Node(node, caller.User.Username);
}
public async Task> HeartbeatAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node");
await TouchAsync(node, cancellationToken);
return true;
}
/// The next queued render job this node can encode (highest priority, then oldest), or null when none fits.
public async Task> ClaimAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node");
var capabilities = Dto.Capabilities(node);
foreach (var queued in await db.Jobs.ListQueuedAsync(JobKind.Render, 50, cancellationToken))
{
// A render the server's own engine started (it left a checkpoint) resumes there: only it knows that attempt's
// leftovers and the batch entries it recorded.
if (queued.Checkpoint is not null) continue;
if (JobEndpoints.ParseRender(queued.ParametersJson) is not { } parameters || capabilities is null || !CanEncode(capabilities, parameters)) continue;
if (await db.Jobs.TryClaimAsync(queued.Id, NodeKey(node), cancellationToken) is not { } job) continue;
jobs.Publish(job);
if (job.TargetIds.Count != 1 || await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken) is not { } sequence)
{
await FinishAsync(node, job, JobState.Failed, "The sequence no longer exists.", cancellationToken);
continue;
}
await TouchAsync(node, cancellationToken);
return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson,
parameters.UseProxies, await ManifestFramesAsync(sequence, cancellationToken));
}
await TouchAsync(node, cancellationToken);
return (ClaimedJob?)null;
}
public async Task> ProgressAsync(Caller caller, Guid nodeId, Guid jobId, NodeProgress progress, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned");
double? eta = progress.ItemsPerSecond is > 0 && progress.Total > progress.Done ? (progress.Total - progress.Done) / progress.ItemsPerSecond.Value : null;
var tail = AppendLog(job.LogTail, progress.LogLine);
if (!await db.Jobs.UpdateNodeProgressAsync(jobId, NodeKey(node), progress.Done, progress.Total, progress.ItemsPerSecond,
eta is { } seconds ? TimeSpan.FromSeconds(seconds) : null, tail, cancellationToken))
return Outcome.Fail(409, "job_not_assigned");
await TouchAsync(node, cancellationToken);
events.Publish(new JobProgressEvent(jobId, progress.Done, progress.Total, progress.ItemsPerSecond, eta));
return true;
}
public async Task> FailAsync(Caller caller, Guid nodeId, Guid jobId, NodeJobFailure failure, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned");
var error = string.IsNullOrWhiteSpace(failure.Error) ? "Render failed on the node." : failure.Error.Trim()[..Math.Min(failure.Error.Trim().Length, 500)];
return await FinishAsync(node, job, JobState.Failed, error, cancellationToken) ? true : Outcome.Fail(409, "job_not_assigned");
}
/// Starts (or resumes) the chunked upload of a job's output.
public async Task> StartResultAsync(Caller caller, Guid nodeId, Guid jobId, RenderResultRequest request, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned");
if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Outcome.Fail(400, "invalid_file_name");
if (!UploadRules.IsSha256(request.Sha256)) return Outcome.Fail(400, "invalid_hash");
if (request.Size <= 0) return Outcome.Fail(400, "bad_request", "size");
if (request.Size > options.MaxUploadBytes)
return Outcome.Fail(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture));
if (string.IsNullOrWhiteSpace(request.PresetName)) return Outcome.Fail(400, "bad_request", "presetName");
var sha = request.Sha256.ToLowerInvariant();
if (await db.Uploads.FindByJobAsync(jobId, cancellationToken) is { State: UploadState.Receiving } open && open.Sha256 == sha && open.Size == request.Size
&& store.Exists(open.Id))
return Started(open);
if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size) return Outcome.Fail(507, "insufficient_storage");
var now = time.GetUtcNow();
var upload = await db.Uploads.CreateAsync(new Upload
{
Id = Guid.CreateVersion7(),
UserId = caller.Id,
SequenceId = job.TargetIds.Count == 1 ? job.TargetIds[0] : null,
FileName = fileName,
Size = request.Size,
Sha256 = sha,
ChunkSize = options.ChunkSizeBytes,
State = UploadState.Receiving,
Created = now,
UpdatedAt = now,
JobId = jobId,
RenderJson = JsonSerializer.Serialize(request with { FileName = fileName, Sha256 = sha }, ProtocolJson.Default.RenderResultRequest),
}, cancellationToken);
store.Create(upload.Id, upload.Size);
return Started(upload);
}
/// Verifies the uploaded output, files it as a render of the sequence and marks the job done.
public async Task> CompleteResultAsync(Caller caller, Guid nodeId, Guid jobId, CancellationToken cancellationToken)
{
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node");
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned");
var upload = await db.Uploads.FindByJobAsync(jobId, cancellationToken);
if (upload is not { State: UploadState.Receiving } || upload.RenderJson is null) return Outcome.Fail(409, "upload_state");
if (!upload.IsComplete)
return Outcome.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture));
if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken))
return Outcome.Fail(409, "upload_state");
if (await uploads.StageVerifiedAsync(upload, cancellationToken) is not { } staged) return Outcome.Fail(422, "hash_mismatch");
var output = JsonSerializer.Deserialize(upload.RenderJson, ProtocolJson.Default.RenderResultRequest)!;
var sequence = await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken);
var render = sequence is null ? null : await renders.CommitAsync(sequence, staged, output, node.ClientName, cancellationToken);
if (render is null)
{
UploadStore.TryDelete(staged);
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None);
store.Delete(upload.Id);
await FinishAsync(node, job, JobState.Failed, "The sequence has no folder in the archive.", CancellationToken.None);
return Outcome.Fail(409, "upload_state", "sequence");
}
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, cancellationToken: CancellationToken.None);
store.Delete(upload.Id);
await FinishAsync(node, job, JobState.Done, null, CancellationToken.None);
if (await db.Sequences.GetAsync(sequence!.Id, CancellationToken.None) is { } updated)
events.Publish(new SequenceChangedEvent(updated.Id, "renders", updated.FramesVersion, updated.SettingsVersion));
return Dto.Render(render);
}
/// Running-job owner key stored in jobs.node_id.
public static string NodeKey(RenderNode node) => node.Id.ToString();
///
/// Gives the jobs running on back to the queue with in their log
/// (checkpoint and progress kept); a job that already lost its node times
/// fails instead. Returns how many went back.
///
internal static async Task ReleaseJobsAsync(TitanoDatabase db, JobEngineHost jobs, RenderNode node, string reason, CancellationToken cancellationToken)
{
var key = NodeKey(node);
var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken);
foreach (var job in lost)
{
var line = AppendLog(job.LogTail, reason);
if (job.Attempts >= RenderNodeMonitor.MaxAttempts)
await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken);
else
await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken);
}
var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken);
foreach (var job in lost)
{
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current);
}
return requeued;
}
private async Task OwnedNodeAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) =>
await db.RenderNodes.GetAsync(nodeId, cancellationToken) is { } node && node.UserId == caller.Id ? node : null;
private async Task AssignedJobAsync(RenderNode node, Guid jobId, CancellationToken cancellationToken) =>
await db.Jobs.GetAsync(jobId, cancellationToken) is { State: JobState.Running } job && job.NodeId == NodeKey(node) ? job : null;
private async Task FinishAsync(RenderNode node, Job job, JobState state, string? error, CancellationToken cancellationToken)
{
var tail = error is null ? null : AppendLog(job.LogTail, error);
if (!await db.Jobs.FinishNodeJobAsync(job.Id, NodeKey(node), state, error, tail, cancellationToken)) return false;
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } finished) jobs.Publish(finished);
await TouchAsync(node, cancellationToken);
return true;
}
/// Records a sign of life and publishes the node's state when it changed.
private async Task TouchAsync(RenderNode node, CancellationToken cancellationToken)
{
var state = (await db.Jobs.ListRunningByNodeAsync(NodeKey(node), cancellationToken)).Count > 0 ? Busy : Idle;
await db.RenderNodes.HeartbeatAsync(node.Id, state, time.GetUtcNow(), cancellationToken);
if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state));
}
private async Task> ManifestFramesAsync(Sequence sequence, CancellationToken cancellationToken)
{
var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt);
var frames = new List(sequence.FrameCount);
var after = -1;
while (true)
{
var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, 1000, cancellationToken);
frames.AddRange(batch.Where(f => !f.Flags.HasFlag(FrameFlags.Excluded))
.Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb)));
if (batch.Count < 1000) return frames;
after = batch[^1].Index;
}
}
///
/// The node has an encoder for the job's codec that handles its resolution: the encoder's own limit, else the node's
/// (0 = no announced limit).
///
internal static bool CanEncode(RegisterNodeRequest node, RenderJobParameters job) =>
node.Encoders.Any(e => string.Equals(e.Codec, job.Codec, StringComparison.OrdinalIgnoreCase)
&& Fits(job.Width, e.MaxWidth > 0 ? e.MaxWidth : node.MaxWidth)
&& Fits(job.Height, e.MaxHeight > 0 ? e.MaxHeight : node.MaxHeight));
private static bool Fits(int size, int limit) => limit <= 0 || size <= limit;
internal static string? AppendLog(string? tail, string? line)
{
if (string.IsNullOrWhiteSpace(line)) return tail;
var lines = (tail ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries).ToList();
lines.Add(string.Create(CultureInfo.InvariantCulture, $"{DateTime.UtcNow:HH:mm:ss} {line.Trim()[..Math.Min(line.Trim().Length, 300)]}"));
return string.Join('\n', lines.TakeLast(LogTailLines));
}
private static StartUploadResponse Started(Upload upload) =>
new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null);
}