Files
Chronos/src/Titano.Server/Jobs/RenderNodeMonitor.cs
T
Claude 23c40ece6f fix(render-node): a restarted node frees its job; downloads resume; batches split on arrival
A node registering again gives back the job its previous run abandoned, so
a restart within the heartbeat timeout no longer leaves it running forever.
Frame downloads retry dropped connections from the partial file; a 404 at
registration waits for the next poll; render remarks reach the job log; a
render the server engine started stays with it. POST /jobs splits a batch
into one job per preset, and a node fails a batch it cannot send back
instead of reporting it done. Renders honour Settings > Rendering: hardware
encoders only where the preference allows them, also for what a node offers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-02 23:16:23 +00:00

55 lines
2.3 KiB
C#

using Titano.Core.Model;
using Titano.Data;
using Titano.Protocol;
using Titano.Server.Configuration;
using Titano.Server.Events;
namespace Titano.Server.Jobs;
/// <summary>
/// Marks render nodes offline when they miss heartbeats for <see cref="ServerOptions.NodeTimeoutSeconds"/> and gives
/// their running jobs back to the queue for another node; a job that lost its node <see cref="MaxAttempts"/> times fails.
/// </summary>
public sealed class RenderNodeMonitor(TitanoDatabase db, ServerOptions options, JobEngineHost jobs, EventHub events, TimeProvider time,
ILogger<RenderNodeMonitor> logger) : BackgroundService
{
public const int MaxAttempts = 5;
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var interval = TimeSpan.FromSeconds(Math.Clamp(options.NodeTimeoutSeconds / 3.0, 1, 30));
while (!stoppingToken.IsCancellationRequested)
{
try
{
await SweepAsync(stoppingToken);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogWarning(ex, "Render node check failed");
}
try
{
await Task.Delay(interval, time, stoppingToken);
}
catch (OperationCanceledException)
{
return;
}
}
}
/// <summary>One pass: silent nodes go offline and their jobs are requeued (or failed after too many attempts).</summary>
public async Task SweepAsync(CancellationToken cancellationToken)
{
var silentSince = time.GetUtcNow() - TimeSpan.FromSeconds(options.NodeTimeoutSeconds);
foreach (var node in await db.RenderNodes.ListSilentAsync(silentSince, RenderNodeService.Offline, cancellationToken))
{
await db.RenderNodes.SetStateAsync(node.Id, RenderNodeService.Offline, cancellationToken);
events.Publish(new NodeStateEvent(node.Id, node.ClientName, RenderNodeService.Offline));
var requeued = await RenderNodeService.ReleaseJobsAsync(db, jobs, node, $"Render node {node.ClientName} stopped responding.", cancellationToken);
logger.LogWarning("Render node {Node} is offline; {Count} job(s) back in the queue", node.ClientName, requeued);
}
}
}