using Titano.Core.Model;
using Titano.Data;
using Titano.Protocol;
using Titano.Server.Configuration;
using Titano.Server.Events;
namespace Titano.Server.Jobs;
///
/// Marks render nodes offline when they miss heartbeats for and gives
/// their running jobs back to the queue for another node; a job that lost its node times fails.
///
public sealed class RenderNodeMonitor(TitanoDatabase db, ServerOptions options, JobEngineHost jobs, EventHub events, TimeProvider time,
ILogger logger) : BackgroundService
{
public const int MaxAttempts = 5;
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var interval = TimeSpan.FromSeconds(Math.Clamp(options.NodeTimeoutSeconds / 3.0, 1, 30));
while (!stoppingToken.IsCancellationRequested)
{
try
{
await SweepAsync(stoppingToken);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogWarning(ex, "Render node check failed");
}
try
{
await Task.Delay(interval, time, stoppingToken);
}
catch (OperationCanceledException)
{
return;
}
}
}
/// One pass: silent nodes go offline and their jobs are requeued (or failed after too many attempts).
public async Task SweepAsync(CancellationToken cancellationToken)
{
var silentSince = time.GetUtcNow() - TimeSpan.FromSeconds(options.NodeTimeoutSeconds);
foreach (var node in await db.RenderNodes.ListSilentAsync(silentSince, RenderNodeService.Offline, cancellationToken))
{
await db.RenderNodes.SetStateAsync(node.Id, RenderNodeService.Offline, cancellationToken);
events.Publish(new NodeStateEvent(node.Id, node.ClientName, RenderNodeService.Offline));
var requeued = await RenderNodeService.ReleaseJobsAsync(db, jobs, node, $"Render node {node.ClientName} stopped responding.", cancellationToken);
logger.LogWarning("Render node {Node} is offline; {Count} job(s) back in the queue", node.ClientName, requeued);
}
}
}