using System.Diagnostics;
using System.Net;
using System.Text.Json;
using Titano.Core.Diagnostics;
using Titano.Core.Export;
using Titano.Core.Imaging;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Render;
using Titano.Core.Video;
using Titano.Data.Processing.Jobs;
using Titano.Protocol;
using Titano.Protocol.Client;
namespace Titano.Data.Processing;
/// Settings of a .
public sealed record RenderNodeOptions
{
/// Name the server lists the node under; one node per user and name.
public required string ClientName { get; init; }
/// Folder for downloaded originals and for outputs until they are uploaded (inside the client's cache).
public required string CacheDirectory { get; init; }
/// Downloaded originals beyond this size are deleted, least recently used first, after every job.
public long CacheLimitBytes { get; init; } = 20L << 30;
/// Wait between claims while the queue has nothing for this node.
public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(10);
/// Sign of life; the server gives the jobs of a node silent for 90 s to others.
public TimeSpan HeartbeatInterval { get; init; } = TimeSpan.FromSeconds(30);
/// Shortest interval between progress reports to the server.
public TimeSpan ProgressInterval { get; init; } = TimeSpan.FromSeconds(2);
/// Concurrent frame downloads.
public int Downloads { get; init; } = 4;
}
///
/// "Offer this PC as render node" (BRIEF ยง14.7): registers this machine's single-file encoders with the server,
/// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals
/// hash-verified into a content-addressed cache, or proxies when the job asks), renders the job's preset with the
/// host's encoders and uploads the output, which the server files as a render of the sequence. Progress goes to the
/// server; a job cancelled or reassigned there stops here; a failure is reported with its reason. Stopping abandons
/// the job in flight without reporting it, so the server requeues it once the heartbeats stop. Image sequences are
/// not offered: the protocol uploads one file per job.
///
public sealed class RenderNodeClient : IAsyncDisposable
{
private const string LogCategory = "render-node";
private readonly TitanoClient _client;
private readonly ProcessingOptions _processing;
private readonly RenderNodeOptions _options;
private readonly CancellationTokenSource _stopping = new();
private Task _running = Task.CompletedTask;
private Guid? _nodeId;
public RenderNodeClient(TitanoClient client, ProcessingOptions processing, RenderNodeOptions options)
{
_client = client;
_processing = processing;
_options = options;
}
/// The encoders this node offers: every single-file capability of the host's providers.
public IReadOnlyList Encoders =>
[
.. _processing.Encoders.SelectMany(p => p.Capabilities)
.Where(c => VideoCodecInfo.DefaultContainer(c.Codec) != ContainerFormat.ImageSequence)
.Select(c => new NodeEncoder(ProcessingJobs.CodecName(c.Codec), c.Name, c.Hardware, c.SupportsTenBit)),
];
/// Starts registering, heart-beating and claiming in the background.
public void Start() => _running = Task.Run(() => RunAsync(_stopping.Token));
/// Stops claiming, abandons the job in flight and waits for the background work to end.
public async Task StopAsync()
{
await _stopping.CancelAsync().ConfigureAwait(false);
await _running.ConfigureAwait(false);
}
public async ValueTask DisposeAsync()
{
await StopAsync().ConfigureAwait(false);
_stopping.Dispose();
}
/// Registers this node (again) and returns its id.
public async Task RegisterAsync(CancellationToken cancellationToken)
{
var capabilities = _processing.Encoders.SelectMany(p => p.Capabilities).ToList();
var request = new RegisterNodeRequest(_options.ClientName, Encoders, capabilities.Count > 0 ? capabilities.Max(c => c.MaxWidth) : 0,
capabilities.Count > 0 ? capabilities.Max(c => c.MaxHeight) : 0);
var node = await _client.RegisterNodeAsync(request, cancellationToken).ConfigureAwait(false);
_nodeId = node.Id;
return node.Id;
}
/// Claims one job this node can encode and runs it to the end; false when the queue had none.
public async Task RunOnceAsync(CancellationToken cancellationToken)
{
var nodeId = _nodeId ?? await RegisterAsync(cancellationToken).ConfigureAwait(false);
if (await _client.ClaimAsync(nodeId, cancellationToken).ConfigureAwait(false) is not { } claimed) return false;
var work = Path.Combine(_options.CacheDirectory, "jobs", claimed.Job.Id.ToString("N"));
using var job = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var reporter = new NodeReporter(_client, nodeId, claimed.Job.Id, _options.ProgressInterval, job);
try
{
Directory.CreateDirectory(work);
await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false);
}
catch (Exception ex) when (job.IsCancellationRequested)
{
// Stopping, or the server cancelled or reassigned the job: either way it is no longer ours to report.
Log.Info(LogCategory, $"Render job {claimed.Job.Id} abandoned: {ex.Message}");
}
catch (Exception ex)
{
// Whatever went wrong belongs to this job: report it and stay available for the next one.
Log.Warning(LogCategory, $"Render job {claimed.Job.Id} failed", ex);
await ReportFailureAsync(nodeId, claimed.Job.Id, SequenceRender.Reason(ex), cancellationToken).ConfigureAwait(false);
}
finally
{
TryDelete(work);
TrimCache();
}
return true;
}
private async Task RunAsync(CancellationToken stopping)
{
using var heartbeat = CancellationTokenSource.CreateLinkedTokenSource(stopping);
Task? beating = null;
while (!stopping.IsCancellationRequested)
{
try
{
var nodeId = _nodeId ?? await RegisterAsync(stopping).ConfigureAwait(false);
beating ??= HeartbeatAsync(heartbeat.Token);
if (await RunOnceAsync(stopping).ConfigureAwait(false)) continue;
}
catch (OperationCanceledException) when (stopping.IsCancellationRequested)
{
break;
}
catch (Exception ex) when (ex is TitanoApiException { Status: HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden } or UntrustedCertificateException)
{
// Signed out, no render-node right, or a changed certificate: nothing to retry until the user acts.
Log.Warning(LogCategory, "The server refused this render node", ex);
break;
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
{
_nodeId = null; // removed on the server: register again
continue;
}
catch (Exception ex)
{
// Server unreachable or busy: the node keeps trying until it is stopped.
Log.Warning(LogCategory, "Render node cycle failed; retrying", ex);
}
try
{
await Task.Delay(_options.PollInterval, stopping).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
break;
}
}
await heartbeat.CancelAsync().ConfigureAwait(false);
if (beating is not null) await beating.ConfigureAwait(false);
}
private async Task HeartbeatAsync(CancellationToken cancellationToken)
{
using var timer = new PeriodicTimer(_options.HeartbeatInterval);
try
{
while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false))
{
if (_nodeId is not { } nodeId) continue;
try
{
await _client.HeartbeatAsync(nodeId, cancellationToken).ConfigureAwait(false);
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
{
_nodeId = null;
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
// Unreachable or refusing: the claim loop decides whether to go on; beating must not stop the node.
Log.Warning(LogCategory, "Heartbeat failed", ex);
}
}
}
catch (OperationCanceledException)
{
}
}
private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
{
var export = ProcessingJobs.ReadExport(claimed.PresetJson);
var resolved = ExportResolver.Resolve(export);
var target = new ExportTarget(resolved, ProcessingJobs.PresetName(resolved));
if (target.Export.Container == ContainerFormat.ImageSequence) throw new InvalidOperationException(Loc.L("Errors.Render.NodeSequence"));
var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export };
// One output per job: batch presets were meant for separate jobs (ProcessingJobs.RenderEach).
if (RenderJobHandler.Targets(export, target.PresetName).Skip(1).Select(t => t.PresetName).ToList() is { Count: > 0 } skipped)
await reporter.ReportAsync(0, claimed.Frames.Count, Loc.L("Export.Log.NodeBatch", string.Join(", ", skipped))).ConfigureAwait(false);
var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false));
var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false);
var files = await DownloadAsync(claimed, work, reporter, cancellationToken).ConfigureAwait(false);
var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id))]);
IReadOnlyList outcomes;
await using (var render = new SequenceRender(input, project, null, _processing))
{
outcomes = await render.RenderAllAsync(
[(0, target)],
(t, setup) => ExportNaming.OutputPath(work, project.Export.NameTemplate, SequenceRender.NameContext(sequence, t, setup)),
(_, _) => ValueTask.CompletedTask,
(_, t, result) => new ValueTask(UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken)),
reporter.ReportAsync,
cancellationToken).ConfigureAwait(false);
}
if (outcomes[0].Error is { } error) throw new InvalidOperationException(error);
}
private async Task UploadAsync(Guid nodeId, Guid jobId, ExportTarget target, RenderResult result, CancellationToken cancellationToken)
{
var size = new FileInfo(result.OutputPath).Length;
var sha = await TitanoClient.HashFileAsync(result.OutputPath, cancellationToken).ConfigureAwait(false);
var request = new RenderResultRequest(Path.GetFileName(result.OutputPath), size, sha, target.PresetName, ProcessingJobs.CodecName(result.Codec),
result.Width, result.Height, result.FrameRate.Value, result.DurationSeconds);
await _client.UploadRenderResultAsync(nodeId, jobId, result.OutputPath, request, null, cancellationToken).ConfigureAwait(false);
}
private async Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken)
{
var frames = new List();
string? cursor = null;
do
{
var page = await _client.ListFramesAsync(sequenceId, cursor, 1000, cancellationToken).ConfigureAwait(false);
frames.AddRange(page.Items.Select(ToFrame));
cursor = page.NextCursor;
}
while (cursor is not null);
frames.Sort((a, b) => a.Index.CompareTo(b.Index));
return frames;
}
///
/// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused
/// across jobs, fetched once per content), or proxies into the job folder when the job asks for them and the server
/// has them.
///
private async Task> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
{
var files = new Dictionary();
var done = 0;
var total = claimed.Frames.Count;
await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false);
var downloads = claimed.Frames.GroupBy(f => claimed.UseProxies && f.HasProxy ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken },
async (frames, ct) =>
{
var frame = frames.First();
var path = claimed.UseProxies && frame.HasProxy
? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false)
: await DownloadOriginalAsync(frame, ct).ConfigureAwait(false);
lock (files)
{
foreach (var f in frames) files[f.Id] = path;
}
await reporter.ReportAsync(Interlocked.Add(ref done, frames.Count()), total, null).ConfigureAwait(false);
}).ConfigureAwait(false);
return files;
}
/// A proxy into the job folder (regenerable, so not cached); the original when the server has none for the frame.
private async Task DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
{
var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg");
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
try
{
await using (var file = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
await _client.DownloadFrameAsync(frame.Id, "proxy", file, null, cancellationToken).ConfigureAwait(false);
return path;
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
{
File.Delete(path);
return await DownloadOriginalAsync(frame, cancellationToken).ConfigureAwait(false);
}
}
/// A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256.
private async Task DownloadOriginalAsync(ManifestFrame frame, CancellationToken cancellationToken)
{
var sha = frame.Sha256.ToLowerInvariant();
if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
var path = Path.Combine(_options.CacheDirectory, "originals", sha[..2], sha + Path.GetExtension(frame.FileName).ToLowerInvariant());
if (File.Exists(path))
{
File.SetLastWriteTimeUtc(path, DateTime.UtcNow); // most recently used: kept by the cache trim
return path;
}
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
var part = path + ".part";
for (var attempt = 1; ; attempt++)
{
// A part as long as the file was complete when the last attempt stopped: only its hash is left to check.
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size)
{
await using var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
await _client.DownloadFrameAsync(frame.Id, "original", file, null, cancellationToken).ConfigureAwait(false);
}
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
{
File.Move(part, path, overwrite: true);
return path;
}
File.Delete(part);
if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
}
}
private async Task ReportFailureAsync(Guid nodeId, Guid jobId, string error, CancellationToken cancellationToken)
{
try
{
await _client.ReportNodeFailureAsync(nodeId, jobId, error, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException or OperationCanceledException)
{
// Cancelled or reassigned meanwhile, or the server is gone: it requeues the job when the heartbeats stop.
Log.Warning(LogCategory, $"The failure of job {jobId} could not be reported", ex);
}
}
/// Deletes the least recently used originals until the cache fits its limit.
private void TrimCache()
{
try
{
var root = new DirectoryInfo(Path.Combine(_options.CacheDirectory, "originals"));
if (!root.Exists) return;
var files = root.EnumerateFiles("*", SearchOption.AllDirectories).OrderBy(f => f.LastWriteTimeUtc).ToList();
var size = files.Sum(f => f.Length);
foreach (var file in files)
{
if (size <= _options.CacheLimitBytes) break;
size -= file.Length;
file.Delete();
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning(LogCategory, "The render node cache could not be trimmed", ex);
}
}
private static void TryDelete(string folder)
{
try
{
if (Directory.Exists(folder)) Directory.Delete(folder, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning(LogCategory, $"Could not remove {folder}", ex);
}
}
private static Sequence ToSequence(SequenceDto s) => new()
{
Id = s.Id,
ArchiveId = Guid.Empty,
Folder = s.Folder,
Title = s.Title,
CameraMake = s.CameraMake,
CameraModel = s.CameraModel,
Lens = s.Lens,
FirstCapture = s.FirstCapture,
LastCapture = s.LastCapture,
FrameCount = s.FrameCount,
};
private static Frame ToFrame(FrameDto f) => new()
{
Id = f.Id,
SequenceId = f.SequenceId,
Index = f.Index,
RelativePath = f.FileName,
FileName = f.FileName,
Size = f.Size,
Sha256 = f.Sha256,
CapturedAt = f.CapturedAt,
UtcOffsetMinutes = f.UtcOffsetMinutes,
ExposureTime = f.ExposureTime,
FNumber = f.FNumber,
Iso = f.Iso,
FocalLength = f.FocalLength,
Orientation = f.Orientation,
Width = f.Width,
Height = f.Height,
Format = Enum.TryParse(f.Format, ignoreCase: true, out var format) ? format : default,
Flags = (FrameFlags)f.Flags,
};
///
/// Progress of the job in flight to the server, at most every interval (and at the end). A refusal (409) means the
/// job was cancelled or given to another node: the job's token is cancelled.
///
private sealed class NodeReporter(TitanoClient client, Guid nodeId, Guid jobId, TimeSpan interval, CancellationTokenSource job)
{
private readonly Stopwatch _clock = Stopwatch.StartNew();
private readonly SemaphoreSlim _gate = new(1, 1);
private TimeSpan _last = TimeSpan.MinValue;
private long _lastDone;
private string? _lastMessage;
public Guid NodeId => nodeId;
public async ValueTask ReportAsync(long done, long total, string? message)
{
await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
try
{
var now = _clock.Elapsed;
var line = message is not null && message != _lastMessage ? message : null;
if (line is null && done < total && now - _last < interval) return;
double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3);
(_last, _lastDone) = (now, done);
if (line is not null) _lastMessage = line;
await client.ReportNodeProgressAsync(nodeId, jobId, new NodeProgress(done, total, rate, line), job.Token).ConfigureAwait(false);
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict)
{
await job.CancelAsync().ConfigureAwait(false);
}
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException)
{
Log.Warning(LogCategory, "Progress could not be reported", ex);
}
catch (OperationCanceledException)
{
// The job is stopping; its outcome is handled by the caller.
}
finally
{
_gate.Release();
}
}
}
}