using Titano.Core.Model; using Titano.Data; using Titano.Protocol; using Titano.Server.Configuration; using Titano.Server.Events; namespace Titano.Server.Jobs; /// /// Marks render nodes offline when they miss heartbeats for and gives /// their running jobs back to the queue for another node; a job that lost its node times fails. /// public sealed class RenderNodeMonitor(TitanoDatabase db, ServerOptions options, JobEngineHost jobs, EventHub events, TimeProvider time, ILogger logger) : BackgroundService { public const int MaxAttempts = 5; protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var interval = TimeSpan.FromSeconds(Math.Clamp(options.NodeTimeoutSeconds / 3.0, 1, 30)); while (!stoppingToken.IsCancellationRequested) { try { await SweepAsync(stoppingToken); } catch (Exception ex) when (ex is not OperationCanceledException) { logger.LogWarning(ex, "Render node check failed"); } try { await Task.Delay(interval, time, stoppingToken); } catch (OperationCanceledException) { return; } } } /// One pass: silent nodes go offline and their jobs are requeued (or failed after too many attempts). public async Task SweepAsync(CancellationToken cancellationToken) { var silentSince = time.GetUtcNow() - TimeSpan.FromSeconds(options.NodeTimeoutSeconds); foreach (var node in await db.RenderNodes.ListSilentAsync(silentSince, RenderNodeService.Offline, cancellationToken)) { await db.RenderNodes.SetStateAsync(node.Id, RenderNodeService.Offline, cancellationToken); events.Publish(new NodeStateEvent(node.Id, node.ClientName, RenderNodeService.Offline)); var requeued = await RenderNodeService.ReleaseJobsAsync(db, jobs, node, $"Render node {node.ClientName} stopped responding.", cancellationToken); logger.LogWarning("Render node {Node} is offline; {Count} job(s) back in the queue", node.ClientName, requeued); } } }