using System.Diagnostics; using System.Net; using System.Text.Json; using Titano.Core.Diagnostics; using Titano.Core.Export; using Titano.Core.Imaging; using Titano.Core.Localization; using Titano.Core.Model; using Titano.Core.Render; using Titano.Core.Video; using Titano.Data.Processing.Jobs; using Titano.Protocol; using Titano.Protocol.Client; namespace Titano.Data.Processing; /// Settings of a . public sealed record RenderNodeOptions { /// Name the server lists the node under; one node per user and name. public required string ClientName { get; init; } /// Folder for downloaded originals and for outputs until they are uploaded (inside the client's cache). public required string CacheDirectory { get; init; } /// Downloaded originals beyond this size are deleted, least recently used first, after every job. public long CacheLimitBytes { get; init; } = 20L << 30; /// Wait between claims while the queue has nothing for this node. public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(10); /// Sign of life; the server gives the jobs of a node silent for 90 s to others. public TimeSpan HeartbeatInterval { get; init; } = TimeSpan.FromSeconds(30); /// Shortest interval between progress reports to the server. public TimeSpan ProgressInterval { get; init; } = TimeSpan.FromSeconds(2); /// Concurrent frame downloads. public int Downloads { get; init; } = 4; } /// /// "Offer this PC as render node" (BRIEF ยง14.7): registers this machine's single-file encoders with the server, /// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals /// hash-verified into a content-addressed cache, or proxies when the job asks), renders the job's preset with the /// host's encoders and uploads the output, which the server files as a render of the sequence. Progress goes to the /// server; a job cancelled or reassigned there stops here; a failure is reported with its reason. Stopping abandons /// the job in flight without reporting it, so the server requeues it once the heartbeats stop. Image sequences are /// not offered: the protocol uploads one file per job. /// public sealed class RenderNodeClient : IAsyncDisposable { private const string LogCategory = "render-node"; private readonly TitanoClient _client; private readonly ProcessingOptions _processing; private readonly RenderNodeOptions _options; private readonly CancellationTokenSource _stopping = new(); private Task _running = Task.CompletedTask; private Guid? _nodeId; public RenderNodeClient(TitanoClient client, ProcessingOptions processing, RenderNodeOptions options) { _client = client; _processing = processing; _options = options; } /// The encoders this node offers: every single-file capability of the host's providers. public IReadOnlyList Encoders => [ .. _processing.Encoders.SelectMany(p => p.Capabilities) .Where(c => VideoCodecInfo.DefaultContainer(c.Codec) != ContainerFormat.ImageSequence) .Select(c => new NodeEncoder(ProcessingJobs.CodecName(c.Codec), c.Name, c.Hardware, c.SupportsTenBit)), ]; /// Starts registering, heart-beating and claiming in the background. public void Start() => _running = Task.Run(() => RunAsync(_stopping.Token)); /// Stops claiming, abandons the job in flight and waits for the background work to end. public async Task StopAsync() { await _stopping.CancelAsync().ConfigureAwait(false); await _running.ConfigureAwait(false); } public async ValueTask DisposeAsync() { await StopAsync().ConfigureAwait(false); _stopping.Dispose(); } /// Registers this node (again) and returns its id. public async Task RegisterAsync(CancellationToken cancellationToken) { var capabilities = _processing.Encoders.SelectMany(p => p.Capabilities).ToList(); var request = new RegisterNodeRequest(_options.ClientName, Encoders, capabilities.Count > 0 ? capabilities.Max(c => c.MaxWidth) : 0, capabilities.Count > 0 ? capabilities.Max(c => c.MaxHeight) : 0); var node = await _client.RegisterNodeAsync(request, cancellationToken).ConfigureAwait(false); _nodeId = node.Id; return node.Id; } /// Claims one job this node can encode and runs it to the end; false when the queue had none. public async Task RunOnceAsync(CancellationToken cancellationToken) { var nodeId = _nodeId ?? await RegisterAsync(cancellationToken).ConfigureAwait(false); if (await _client.ClaimAsync(nodeId, cancellationToken).ConfigureAwait(false) is not { } claimed) return false; var work = Path.Combine(_options.CacheDirectory, "jobs", claimed.Job.Id.ToString("N")); using var job = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var reporter = new NodeReporter(_client, nodeId, claimed.Job.Id, _options.ProgressInterval, job); try { Directory.CreateDirectory(work); await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false); } catch (Exception ex) when (job.IsCancellationRequested) { // Stopping, or the server cancelled or reassigned the job: either way it is no longer ours to report. Log.Info(LogCategory, $"Render job {claimed.Job.Id} abandoned: {ex.Message}"); } catch (Exception ex) { // Whatever went wrong belongs to this job: report it and stay available for the next one. Log.Warning(LogCategory, $"Render job {claimed.Job.Id} failed", ex); await ReportFailureAsync(nodeId, claimed.Job.Id, SequenceRender.Reason(ex), cancellationToken).ConfigureAwait(false); } finally { TryDelete(work); TrimCache(); } return true; } private async Task RunAsync(CancellationToken stopping) { using var heartbeat = CancellationTokenSource.CreateLinkedTokenSource(stopping); Task? beating = null; while (!stopping.IsCancellationRequested) { try { var nodeId = _nodeId ?? await RegisterAsync(stopping).ConfigureAwait(false); beating ??= HeartbeatAsync(heartbeat.Token); if (await RunOnceAsync(stopping).ConfigureAwait(false)) continue; } catch (OperationCanceledException) when (stopping.IsCancellationRequested) { break; } catch (Exception ex) when (ex is TitanoApiException { Status: HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden } or UntrustedCertificateException) { // Signed out, no render-node right, or a changed certificate: nothing to retry until the user acts. Log.Warning(LogCategory, "The server refused this render node", ex); break; } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) { _nodeId = null; // removed on the server: register again continue; } catch (Exception ex) { // Server unreachable or busy: the node keeps trying until it is stopped. Log.Warning(LogCategory, "Render node cycle failed; retrying", ex); } try { await Task.Delay(_options.PollInterval, stopping).ConfigureAwait(false); } catch (OperationCanceledException) { break; } } await heartbeat.CancelAsync().ConfigureAwait(false); if (beating is not null) await beating.ConfigureAwait(false); } private async Task HeartbeatAsync(CancellationToken cancellationToken) { using var timer = new PeriodicTimer(_options.HeartbeatInterval); try { while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) { if (_nodeId is not { } nodeId) continue; try { await _client.HeartbeatAsync(nodeId, cancellationToken).ConfigureAwait(false); } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) { _nodeId = null; } catch (Exception ex) when (!cancellationToken.IsCancellationRequested) { // Unreachable or refusing: the claim loop decides whether to go on; beating must not stop the node. Log.Warning(LogCategory, "Heartbeat failed", ex); } } } catch (OperationCanceledException) { } } private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken) { var export = ProcessingJobs.ReadExport(claimed.PresetJson); var resolved = ExportResolver.Resolve(export); var target = new ExportTarget(resolved, ProcessingJobs.PresetName(resolved)); if (target.Export.Container == ContainerFormat.ImageSequence) throw new InvalidOperationException(Loc.L("Errors.Render.NodeSequence")); var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export }; // One output per job: batch presets were meant for separate jobs (ProcessingJobs.RenderEach). if (RenderJobHandler.Targets(export, target.PresetName).Skip(1).Select(t => t.PresetName).ToList() is { Count: > 0 } skipped) await reporter.ReportAsync(0, claimed.Frames.Count, Loc.L("Export.Log.NodeBatch", string.Join(", ", skipped))).ConfigureAwait(false); var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false)); var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false); var files = await DownloadAsync(claimed, work, reporter, cancellationToken).ConfigureAwait(false); var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id))]); IReadOnlyList outcomes; await using (var render = new SequenceRender(input, project, null, _processing)) { outcomes = await render.RenderAllAsync( [(0, target)], (t, setup) => ExportNaming.OutputPath(work, project.Export.NameTemplate, SequenceRender.NameContext(sequence, t, setup)), (_, _) => ValueTask.CompletedTask, (_, t, result) => new ValueTask(UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken)), reporter.ReportAsync, cancellationToken).ConfigureAwait(false); } if (outcomes[0].Error is { } error) throw new InvalidOperationException(error); } private async Task UploadAsync(Guid nodeId, Guid jobId, ExportTarget target, RenderResult result, CancellationToken cancellationToken) { var size = new FileInfo(result.OutputPath).Length; var sha = await TitanoClient.HashFileAsync(result.OutputPath, cancellationToken).ConfigureAwait(false); var request = new RenderResultRequest(Path.GetFileName(result.OutputPath), size, sha, target.PresetName, ProcessingJobs.CodecName(result.Codec), result.Width, result.Height, result.FrameRate.Value, result.DurationSeconds); await _client.UploadRenderResultAsync(nodeId, jobId, result.OutputPath, request, null, cancellationToken).ConfigureAwait(false); } private async Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken) { var frames = new List(); string? cursor = null; do { var page = await _client.ListFramesAsync(sequenceId, cursor, 1000, cancellationToken).ConfigureAwait(false); frames.AddRange(page.Items.Select(ToFrame)); cursor = page.NextCursor; } while (cursor is not null); frames.Sort((a, b) => a.Index.CompareTo(b.Index)); return frames; } /// /// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused /// across jobs, fetched once per content), or proxies into the job folder when the job asks for them and the server /// has them. /// private async Task> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken) { var files = new Dictionary(); var done = 0; var total = claimed.Frames.Count; await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false); var downloads = claimed.Frames.GroupBy(f => claimed.UseProxies && f.HasProxy ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList(); await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken }, async (frames, ct) => { var frame = frames.First(); var path = claimed.UseProxies && frame.HasProxy ? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false) : await DownloadOriginalAsync(frame, ct).ConfigureAwait(false); lock (files) { foreach (var f in frames) files[f.Id] = path; } await reporter.ReportAsync(Interlocked.Add(ref done, frames.Count()), total, null).ConfigureAwait(false); }).ConfigureAwait(false); return files; } /// A proxy into the job folder (regenerable, so not cached); the original when the server has none for the frame. private async Task DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) { var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg"); Directory.CreateDirectory(Path.GetDirectoryName(path)!); try { await using (var file = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) await _client.DownloadFrameAsync(frame.Id, "proxy", file, null, cancellationToken).ConfigureAwait(false); return path; } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) { File.Delete(path); return await DownloadOriginalAsync(frame, cancellationToken).ConfigureAwait(false); } } /// A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256. private async Task DownloadOriginalAsync(ManifestFrame frame, CancellationToken cancellationToken) { var sha = frame.Sha256.ToLowerInvariant(); if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName)); var path = Path.Combine(_options.CacheDirectory, "originals", sha[..2], sha + Path.GetExtension(frame.FileName).ToLowerInvariant()); if (File.Exists(path)) { File.SetLastWriteTimeUtc(path, DateTime.UtcNow); // most recently used: kept by the cache trim return path; } Directory.CreateDirectory(Path.GetDirectoryName(path)!); var part = path + ".part"; for (var attempt = 1; ; attempt++) { // A part as long as the file was complete when the last attempt stopped: only its hash is left to check. if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) { await using var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); await _client.DownloadFrameAsync(frame.Id, "original", file, null, cancellationToken).ConfigureAwait(false); } if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha) { File.Move(part, path, overwrite: true); return path; } File.Delete(part); if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName)); } } private async Task ReportFailureAsync(Guid nodeId, Guid jobId, string error, CancellationToken cancellationToken) { try { await _client.ReportNodeFailureAsync(nodeId, jobId, error, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException or OperationCanceledException) { // Cancelled or reassigned meanwhile, or the server is gone: it requeues the job when the heartbeats stop. Log.Warning(LogCategory, $"The failure of job {jobId} could not be reported", ex); } } /// Deletes the least recently used originals until the cache fits its limit. private void TrimCache() { try { var root = new DirectoryInfo(Path.Combine(_options.CacheDirectory, "originals")); if (!root.Exists) return; var files = root.EnumerateFiles("*", SearchOption.AllDirectories).OrderBy(f => f.LastWriteTimeUtc).ToList(); var size = files.Sum(f => f.Length); foreach (var file in files) { if (size <= _options.CacheLimitBytes) break; size -= file.Length; file.Delete(); } } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { Log.Warning(LogCategory, "The render node cache could not be trimmed", ex); } } private static void TryDelete(string folder) { try { if (Directory.Exists(folder)) Directory.Delete(folder, recursive: true); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { Log.Warning(LogCategory, $"Could not remove {folder}", ex); } } private static Sequence ToSequence(SequenceDto s) => new() { Id = s.Id, ArchiveId = Guid.Empty, Folder = s.Folder, Title = s.Title, CameraMake = s.CameraMake, CameraModel = s.CameraModel, Lens = s.Lens, FirstCapture = s.FirstCapture, LastCapture = s.LastCapture, FrameCount = s.FrameCount, }; private static Frame ToFrame(FrameDto f) => new() { Id = f.Id, SequenceId = f.SequenceId, Index = f.Index, RelativePath = f.FileName, FileName = f.FileName, Size = f.Size, Sha256 = f.Sha256, CapturedAt = f.CapturedAt, UtcOffsetMinutes = f.UtcOffsetMinutes, ExposureTime = f.ExposureTime, FNumber = f.FNumber, Iso = f.Iso, FocalLength = f.FocalLength, Orientation = f.Orientation, Width = f.Width, Height = f.Height, Format = Enum.TryParse(f.Format, ignoreCase: true, out var format) ? format : default, Flags = (FrameFlags)f.Flags, }; /// /// Progress of the job in flight to the server, at most every interval (and at the end). A refusal (409) means the /// job was cancelled or given to another node: the job's token is cancelled. /// private sealed class NodeReporter(TitanoClient client, Guid nodeId, Guid jobId, TimeSpan interval, CancellationTokenSource job) { private readonly Stopwatch _clock = Stopwatch.StartNew(); private readonly SemaphoreSlim _gate = new(1, 1); private TimeSpan _last = TimeSpan.MinValue; private long _lastDone; private string? _lastMessage; public Guid NodeId => nodeId; public async ValueTask ReportAsync(long done, long total, string? message) { await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false); try { var now = _clock.Elapsed; var line = message is not null && message != _lastMessage ? message : null; if (line is null && done < total && now - _last < interval) return; double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3); (_last, _lastDone) = (now, done); if (line is not null) _lastMessage = line; await client.ReportNodeProgressAsync(nodeId, jobId, new NodeProgress(done, total, rate, line), job.Token).ConfigureAwait(false); } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict) { await job.CancelAsync().ConfigureAwait(false); } catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException) { Log.Warning(LogCategory, "Progress could not be reported", ex); } catch (OperationCanceledException) { // The job is stopping; its outcome is handled by the caller. } finally { _gate.Release(); } } } }