A node registering again gives back the job its previous run abandoned, so a restart within the heartbeat timeout no longer leaves it running forever. Frame downloads retry dropped connections from the partial file; a 404 at registration waits for the next poll; render remarks reach the job log; a render the server engine started stays with it. POST /jobs splits a batch into one job per preset, and a node fails a batch it cannot send back instead of reporting it done. Renders honour Settings > Rendering: hardware encoders only where the preference allows them, also for what a node offers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
55 lines
2.3 KiB
C#
55 lines
2.3 KiB
C#
using Titano.Core.Model;
|
|
using Titano.Data;
|
|
using Titano.Protocol;
|
|
using Titano.Server.Configuration;
|
|
using Titano.Server.Events;
|
|
|
|
namespace Titano.Server.Jobs;
|
|
|
|
/// <summary>
|
|
/// Marks render nodes offline when they miss heartbeats for <see cref="ServerOptions.NodeTimeoutSeconds"/> and gives
|
|
/// their running jobs back to the queue for another node; a job that lost its node <see cref="MaxAttempts"/> times fails.
|
|
/// </summary>
|
|
public sealed class RenderNodeMonitor(TitanoDatabase db, ServerOptions options, JobEngineHost jobs, EventHub events, TimeProvider time,
|
|
ILogger<RenderNodeMonitor> logger) : BackgroundService
|
|
{
|
|
public const int MaxAttempts = 5;
|
|
|
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
{
|
|
var interval = TimeSpan.FromSeconds(Math.Clamp(options.NodeTimeoutSeconds / 3.0, 1, 30));
|
|
while (!stoppingToken.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
await SweepAsync(stoppingToken);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
logger.LogWarning(ex, "Render node check failed");
|
|
}
|
|
try
|
|
{
|
|
await Task.Delay(interval, time, stoppingToken);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>One pass: silent nodes go offline and their jobs are requeued (or failed after too many attempts).</summary>
|
|
public async Task SweepAsync(CancellationToken cancellationToken)
|
|
{
|
|
var silentSince = time.GetUtcNow() - TimeSpan.FromSeconds(options.NodeTimeoutSeconds);
|
|
foreach (var node in await db.RenderNodes.ListSilentAsync(silentSince, RenderNodeService.Offline, cancellationToken))
|
|
{
|
|
await db.RenderNodes.SetStateAsync(node.Id, RenderNodeService.Offline, cancellationToken);
|
|
events.Publish(new NodeStateEvent(node.Id, node.ClientName, RenderNodeService.Offline));
|
|
var requeued = await RenderNodeService.ReleaseJobsAsync(db, jobs, node, $"Render node {node.ClientName} stopped responding.", cancellationToken);
|
|
logger.LogWarning("Render node {Node} is offline; {Count} job(s) back in the queue", node.ClientName, requeued);
|
|
}
|
|
}
|
|
}
|