using System.Globalization; using System.Text.Json; using Titano.Core.Model; using Titano.Data; using Titano.Data.Repositories; using Titano.Protocol; using Titano.Server.Api; using Titano.Server.Configuration; using Titano.Server.Events; using Titano.Server.Security; using Titano.Server.Storage; using UploadState = Titano.Data.Repositories.UploadState; namespace Titano.Server.Jobs; /// /// Render nodes (BRIEF ยง14.7): clients with the render-node right register their encoders, heartbeat, claim queued /// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of /// the sequence. A node that stops heart-beating loses its jobs to the queue (). /// public sealed class RenderNodeService( TitanoDatabase db, ServerOptions options, UploadService uploads, UploadStore store, RenderStore renders, JobEngineHost jobs, EventHub events, TimeProvider time) { public const string Idle = "idle"; public const string Busy = "busy"; public const string Offline = "offline"; private const int LogTailLines = 40; public async Task> RegisterAsync(Caller caller, RegisterNodeRequest request, CancellationToken cancellationToken) { var name = request.ClientName?.Trim(); if (string.IsNullOrEmpty(name) || name.Length > 100) return Outcome.Fail(400, "bad_request", "clientName"); if (request.Encoders is null || request.Encoders.Count > 64 || request.Encoders.Any(e => string.IsNullOrWhiteSpace(e?.Codec))) return Outcome.Fail(400, "bad_request", "encoders"); var existing = (await db.RenderNodes.ListAsync(cancellationToken)).FirstOrDefault(n => n.UserId == caller.Id && n.ClientName == name); // A node registers when it starts, holding no job: what the server still lists as running on it was abandoned // by its previous run (stopped, restarted or crashed within the heartbeat timeout) and goes back to the queue. if (existing is not null) await ReleaseJobsAsync(db, jobs, existing, $"Render node {name} restarted.", cancellationToken); var node = new RenderNode { Id = existing?.Id ?? Guid.CreateVersion7(), UserId = caller.Id, ClientName = name, CapabilitiesJson = JsonSerializer.Serialize(request with { ClientName = name }, ProtocolJson.Default.RegisterNodeRequest), LastHeartbeat = time.GetUtcNow(), State = Idle, }; await db.RenderNodes.RegisterAsync(node, cancellationToken); events.Publish(new NodeStateEvent(node.Id, node.ClientName, node.State)); return Dto.Node(node, caller.User.Username); } public async Task> HeartbeatAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) { if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); await TouchAsync(node, cancellationToken); return true; } /// The next queued render job this node can encode (highest priority, then oldest), or null when none fits. public async Task> ClaimAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) { if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); var capabilities = Dto.Capabilities(node); foreach (var queued in await db.Jobs.ListQueuedAsync(JobKind.Render, 50, cancellationToken)) { // A render the server's own engine started (it left a checkpoint) resumes there: only it knows that attempt's // leftovers and the batch entries it recorded. if (queued.Checkpoint is not null) continue; if (JobEndpoints.ParseRender(queued.ParametersJson) is not { } parameters || capabilities is null || !CanEncode(capabilities, parameters)) continue; if (await db.Jobs.TryClaimAsync(queued.Id, NodeKey(node), cancellationToken) is not { } job) continue; jobs.Publish(job); if (job.TargetIds.Count != 1 || await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken) is not { } sequence) { await FinishAsync(node, job, JobState.Failed, "The sequence no longer exists.", cancellationToken); continue; } await TouchAsync(node, cancellationToken); return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson, parameters.UseProxies, await ManifestFramesAsync(sequence, cancellationToken)); } await TouchAsync(node, cancellationToken); return (ClaimedJob?)null; } public async Task> ProgressAsync(Caller caller, Guid nodeId, Guid jobId, NodeProgress progress, CancellationToken cancellationToken) { if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned"); double? eta = progress.ItemsPerSecond is > 0 && progress.Total > progress.Done ? (progress.Total - progress.Done) / progress.ItemsPerSecond.Value : null; var tail = AppendLog(job.LogTail, progress.LogLine); if (!await db.Jobs.UpdateNodeProgressAsync(jobId, NodeKey(node), progress.Done, progress.Total, progress.ItemsPerSecond, eta is { } seconds ? TimeSpan.FromSeconds(seconds) : null, tail, cancellationToken)) return Outcome.Fail(409, "job_not_assigned"); await TouchAsync(node, cancellationToken); events.Publish(new JobProgressEvent(jobId, progress.Done, progress.Total, progress.ItemsPerSecond, eta)); return true; } public async Task> FailAsync(Caller caller, Guid nodeId, Guid jobId, NodeJobFailure failure, CancellationToken cancellationToken) { if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned"); var error = string.IsNullOrWhiteSpace(failure.Error) ? "Render failed on the node." : failure.Error.Trim()[..Math.Min(failure.Error.Trim().Length, 500)]; return await FinishAsync(node, job, JobState.Failed, error, cancellationToken) ? true : Outcome.Fail(409, "job_not_assigned"); } /// Starts (or resumes) the chunked upload of a job's output. public async Task> StartResultAsync(Caller caller, Guid nodeId, Guid jobId, RenderResultRequest request, CancellationToken cancellationToken) { if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned"); if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Outcome.Fail(400, "invalid_file_name"); if (!UploadRules.IsSha256(request.Sha256)) return Outcome.Fail(400, "invalid_hash"); if (request.Size <= 0) return Outcome.Fail(400, "bad_request", "size"); if (request.Size > options.MaxUploadBytes) return Outcome.Fail(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture)); if (string.IsNullOrWhiteSpace(request.PresetName)) return Outcome.Fail(400, "bad_request", "presetName"); var sha = request.Sha256.ToLowerInvariant(); if (await db.Uploads.FindByJobAsync(jobId, cancellationToken) is { State: UploadState.Receiving } open && open.Sha256 == sha && open.Size == request.Size && store.Exists(open.Id)) return Started(open); if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size) return Outcome.Fail(507, "insufficient_storage"); var now = time.GetUtcNow(); var upload = await db.Uploads.CreateAsync(new Upload { Id = Guid.CreateVersion7(), UserId = caller.Id, SequenceId = job.TargetIds.Count == 1 ? job.TargetIds[0] : null, FileName = fileName, Size = request.Size, Sha256 = sha, ChunkSize = options.ChunkSizeBytes, State = UploadState.Receiving, Created = now, UpdatedAt = now, JobId = jobId, RenderJson = JsonSerializer.Serialize(request with { FileName = fileName, Sha256 = sha }, ProtocolJson.Default.RenderResultRequest), }, cancellationToken); store.Create(upload.Id, upload.Size); return Started(upload); } /// Verifies the uploaded output, files it as a render of the sequence and marks the job done. public async Task> CompleteResultAsync(Caller caller, Guid nodeId, Guid jobId, CancellationToken cancellationToken) { if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned"); var upload = await db.Uploads.FindByJobAsync(jobId, cancellationToken); if (upload is not { State: UploadState.Receiving } || upload.RenderJson is null) return Outcome.Fail(409, "upload_state"); if (!upload.IsComplete) return Outcome.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture)); if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken)) return Outcome.Fail(409, "upload_state"); if (await uploads.StageVerifiedAsync(upload, cancellationToken) is not { } staged) return Outcome.Fail(422, "hash_mismatch"); var output = JsonSerializer.Deserialize(upload.RenderJson, ProtocolJson.Default.RenderResultRequest)!; var sequence = await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken); var render = sequence is null ? null : await renders.CommitAsync(sequence, staged, output, node.ClientName, cancellationToken); if (render is null) { UploadStore.TryDelete(staged); await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None); store.Delete(upload.Id); await FinishAsync(node, job, JobState.Failed, "The sequence has no folder in the archive.", CancellationToken.None); return Outcome.Fail(409, "upload_state", "sequence"); } await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, cancellationToken: CancellationToken.None); store.Delete(upload.Id); await FinishAsync(node, job, JobState.Done, null, CancellationToken.None); if (await db.Sequences.GetAsync(sequence!.Id, CancellationToken.None) is { } updated) events.Publish(new SequenceChangedEvent(updated.Id, "renders", updated.FramesVersion, updated.SettingsVersion)); return Dto.Render(render); } /// Running-job owner key stored in jobs.node_id. public static string NodeKey(RenderNode node) => node.Id.ToString(); /// /// Gives the jobs running on back to the queue with in their log /// (checkpoint and progress kept); a job that already lost its node times /// fails instead. Returns how many went back. /// internal static async Task ReleaseJobsAsync(TitanoDatabase db, JobEngineHost jobs, RenderNode node, string reason, CancellationToken cancellationToken) { var key = NodeKey(node); var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken); foreach (var job in lost) { var line = AppendLog(job.LogTail, reason); if (job.Attempts >= RenderNodeMonitor.MaxAttempts) await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken); else await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken); } var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken); foreach (var job in lost) { if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current); } return requeued; } private async Task OwnedNodeAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) => await db.RenderNodes.GetAsync(nodeId, cancellationToken) is { } node && node.UserId == caller.Id ? node : null; private async Task AssignedJobAsync(RenderNode node, Guid jobId, CancellationToken cancellationToken) => await db.Jobs.GetAsync(jobId, cancellationToken) is { State: JobState.Running } job && job.NodeId == NodeKey(node) ? job : null; private async Task FinishAsync(RenderNode node, Job job, JobState state, string? error, CancellationToken cancellationToken) { var tail = error is null ? null : AppendLog(job.LogTail, error); if (!await db.Jobs.FinishNodeJobAsync(job.Id, NodeKey(node), state, error, tail, cancellationToken)) return false; if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } finished) jobs.Publish(finished); await TouchAsync(node, cancellationToken); return true; } /// Records a sign of life and publishes the node's state when it changed. private async Task TouchAsync(RenderNode node, CancellationToken cancellationToken) { var state = (await db.Jobs.ListRunningByNodeAsync(NodeKey(node), cancellationToken)).Count > 0 ? Busy : Idle; await db.RenderNodes.HeartbeatAsync(node.Id, state, time.GetUtcNow(), cancellationToken); if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state)); } private async Task> ManifestFramesAsync(Sequence sequence, CancellationToken cancellationToken) { var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt); var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt); var frames = new List(sequence.FrameCount); var after = -1; while (true) { var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, 1000, cancellationToken); frames.AddRange(batch.Where(f => !f.Flags.HasFlag(FrameFlags.Excluded)) .Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb))); if (batch.Count < 1000) return frames; after = batch[^1].Index; } } /// /// The node has an encoder for the job's codec that handles its resolution: the encoder's own limit, else the node's /// (0 = no announced limit). /// internal static bool CanEncode(RegisterNodeRequest node, RenderJobParameters job) => node.Encoders.Any(e => string.Equals(e.Codec, job.Codec, StringComparison.OrdinalIgnoreCase) && Fits(job.Width, e.MaxWidth > 0 ? e.MaxWidth : node.MaxWidth) && Fits(job.Height, e.MaxHeight > 0 ? e.MaxHeight : node.MaxHeight)); private static bool Fits(int size, int limit) => limit <= 0 || size <= limit; internal static string? AppendLog(string? tail, string? line) { if (string.IsNullOrWhiteSpace(line)) return tail; var lines = (tail ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries).ToList(); lines.Add(string.Create(CultureInfo.InvariantCulture, $"{DateTime.UtcNow:HH:mm:ss} {line.Trim()[..Math.Min(line.Trim().Length, 300)]}")); return string.Join('\n', lines.TakeLast(LogTailLines)); } private static StartUploadResponse Started(Upload upload) => new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null); }