Originals are resumed from their partial file because their SHA-256 checks the result. Proxies have no hash on the server and can be rebuilt between two attempts, so appending to what a dropped attempt left could mix two versions; a retry now fetches the proxy whole. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
531 lines
25 KiB
C#
531 lines
25 KiB
C#
using System.Diagnostics;
|
|
using System.Net;
|
|
using System.Text.Json;
|
|
using Titano.Core.Diagnostics;
|
|
using Titano.Core.Export;
|
|
using Titano.Core.Imaging;
|
|
using Titano.Core.Localization;
|
|
using Titano.Core.Model;
|
|
using Titano.Core.Render;
|
|
using Titano.Core.Video;
|
|
using Titano.Protocol;
|
|
using Titano.Protocol.Client;
|
|
|
|
namespace Titano.Data.Processing;
|
|
|
|
/// <summary>Settings of a <see cref="RenderNodeClient"/>.</summary>
|
|
public sealed record RenderNodeOptions
|
|
{
|
|
/// <summary>Name the server lists the node under; one node per user and name.</summary>
|
|
public required string ClientName { get; init; }
|
|
|
|
/// <summary>Folder for downloaded originals and for outputs until they are uploaded (inside the client's cache).</summary>
|
|
public required string CacheDirectory { get; init; }
|
|
|
|
/// <summary>Downloaded originals beyond this size are deleted, least recently used first, after every job.</summary>
|
|
public long CacheLimitBytes { get; init; } = 20L << 30;
|
|
|
|
/// <summary>Wait between claims while the queue has nothing for this node.</summary>
|
|
public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(10);
|
|
|
|
/// <summary>Sign of life; the server gives the jobs of a node silent for 90 s to others.</summary>
|
|
public TimeSpan HeartbeatInterval { get; init; } = TimeSpan.FromSeconds(30);
|
|
|
|
/// <summary>Shortest interval between progress reports to the server.</summary>
|
|
public TimeSpan ProgressInterval { get; init; } = TimeSpan.FromSeconds(2);
|
|
|
|
/// <summary>Concurrent frame downloads.</summary>
|
|
public int Downloads { get; init; } = 4;
|
|
|
|
/// <summary>Retries of a frame download cut by the network or a busy server, each resuming the partial file.</summary>
|
|
public int DownloadRetries { get; init; } = 5;
|
|
|
|
/// <summary>Wait before the first retry; it doubles with each further one.</summary>
|
|
public TimeSpan RetryDelay { get; init; } = TimeSpan.FromSeconds(1);
|
|
}
|
|
|
|
/// <summary>
|
|
/// "Offer this PC as render node" (BRIEF §14.7): registers this machine's single-file encoders with the server,
|
|
/// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals
|
|
/// hash-verified into a content-addressed cache, or proxies when the job asks; a dropped download resumes), renders the
|
|
/// job's preset with the host's encoders and uploads the output, which the server files as a render of the sequence.
|
|
/// Progress and render remarks go to the server; a job cancelled or reassigned there stops here; a failure is reported
|
|
/// with its reason. Stopping abandons the job in flight without reporting it: the server requeues it when the node
|
|
/// registers again or its heartbeats stop. Image sequences are not offered and batches are refused: the protocol uploads
|
|
/// one file per job.
|
|
/// </summary>
|
|
public sealed class RenderNodeClient : IAsyncDisposable
|
|
{
|
|
private const string LogCategory = "render-node";
|
|
|
|
private readonly TitanoClient _client;
|
|
private readonly ProcessingOptions _processing;
|
|
private readonly RenderNodeOptions _options;
|
|
private readonly CancellationTokenSource _stopping = new();
|
|
private Task _running = Task.CompletedTask;
|
|
private Guid? _nodeId;
|
|
|
|
public RenderNodeClient(TitanoClient client, ProcessingOptions processing, RenderNodeOptions options)
|
|
{
|
|
_client = client;
|
|
_processing = processing;
|
|
_options = options;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The encoders this node offers, each with its largest output: every single-file capability of the host's
|
|
/// providers, hardware ones only when the user lets renders use them (<see cref="ProcessingOptions.AllowsHardware"/>).
|
|
/// </summary>
|
|
public IReadOnlyList<NodeEncoder> Encoders =>
|
|
[
|
|
.. _processing.Encoders.SelectMany(p => p.Capabilities)
|
|
.Where(c => VideoCodecInfo.DefaultContainer(c.Codec) != ContainerFormat.ImageSequence
|
|
&& (!c.Hardware || _processing.AllowsHardware(ExportResolver.CustomId)))
|
|
.Select(c => new NodeEncoder(ProcessingJobs.CodecName(c.Codec), c.Name, c.Hardware, c.SupportsTenBit, c.MaxWidth, c.MaxHeight)),
|
|
];
|
|
|
|
/// <summary>Completes when the node has stopped: asked to, or refused by the server (signed out, no render-node right,
|
|
/// changed certificate), which only the user can change.</summary>
|
|
public Task Completion => _running;
|
|
|
|
/// <summary>Starts registering, heart-beating and claiming in the background.</summary>
|
|
public void Start() => _running = Task.Run(() => RunAsync(_stopping.Token));
|
|
|
|
/// <summary>Stops claiming, abandons the job in flight and waits for the background work to end.</summary>
|
|
public async Task StopAsync()
|
|
{
|
|
await _stopping.CancelAsync().ConfigureAwait(false);
|
|
await _running.ConfigureAwait(false);
|
|
}
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
await StopAsync().ConfigureAwait(false);
|
|
_stopping.Dispose();
|
|
}
|
|
|
|
/// <summary>Registers this node (again) and returns its id.</summary>
|
|
public async Task<Guid> RegisterAsync(CancellationToken cancellationToken)
|
|
{
|
|
// Each encoder carries its own limit; the node-wide one (for servers that read only that) is the largest offered.
|
|
var encoders = Encoders;
|
|
var request = new RegisterNodeRequest(_options.ClientName, encoders, encoders.Select(e => e.MaxWidth).DefaultIfEmpty().Max(),
|
|
encoders.Select(e => e.MaxHeight).DefaultIfEmpty().Max());
|
|
var node = await _client.RegisterNodeAsync(request, cancellationToken).ConfigureAwait(false);
|
|
_nodeId = node.Id;
|
|
return node.Id;
|
|
}
|
|
|
|
/// <summary>Claims one job this node can encode and runs it to the end; false when the queue had none.</summary>
|
|
public async Task<bool> RunOnceAsync(CancellationToken cancellationToken)
|
|
{
|
|
var nodeId = _nodeId ?? await RegisterAsync(cancellationToken).ConfigureAwait(false);
|
|
if (await _client.ClaimAsync(nodeId, cancellationToken).ConfigureAwait(false) is not { } claimed) return false;
|
|
var work = Path.Combine(_options.CacheDirectory, "jobs", claimed.Job.Id.ToString("N"));
|
|
using var job = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
|
var reporter = new NodeReporter(_client, nodeId, claimed.Job.Id, _options.ProgressInterval, job);
|
|
try
|
|
{
|
|
Directory.CreateDirectory(work);
|
|
await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (job.IsCancellationRequested)
|
|
{
|
|
// Stopping, or the server cancelled or reassigned the job: either way it is no longer ours to report.
|
|
Log.Info(LogCategory, $"Render job {claimed.Job.Id} abandoned: {ex.Message}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Whatever went wrong belongs to this job: report it and stay available for the next one.
|
|
Log.Warning(LogCategory, $"Render job {claimed.Job.Id} failed", ex);
|
|
await ReportFailureAsync(nodeId, claimed.Job.Id, SequenceRender.Reason(ex), cancellationToken).ConfigureAwait(false);
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(work);
|
|
TrimCache();
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private async Task RunAsync(CancellationToken stopping)
|
|
{
|
|
using var heartbeat = CancellationTokenSource.CreateLinkedTokenSource(stopping);
|
|
Task? beating = null;
|
|
while (!stopping.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
var nodeId = _nodeId ?? await RegisterAsync(stopping).ConfigureAwait(false);
|
|
beating ??= HeartbeatAsync(heartbeat.Token);
|
|
if (await RunOnceAsync(stopping).ConfigureAwait(false)) continue;
|
|
}
|
|
catch (OperationCanceledException) when (stopping.IsCancellationRequested)
|
|
{
|
|
break;
|
|
}
|
|
catch (Exception ex) when (ex is TitanoApiException { Status: HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden } or UntrustedCertificateException)
|
|
{
|
|
// Signed out, no render-node right, or a changed certificate: nothing to retry until the user acts.
|
|
Log.Warning(LogCategory, "The server refused this render node", ex);
|
|
break;
|
|
}
|
|
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound && _nodeId is not null)
|
|
{
|
|
_nodeId = null; // removed on the server: register again at once
|
|
continue;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Server unreachable, busy or without the render-node routes: the node keeps trying until it is stopped.
|
|
Log.Warning(LogCategory, "Render node cycle failed; retrying", ex);
|
|
}
|
|
try
|
|
{
|
|
await Task.Delay(_options.PollInterval, stopping).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
await heartbeat.CancelAsync().ConfigureAwait(false);
|
|
if (beating is not null) await beating.ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task HeartbeatAsync(CancellationToken cancellationToken)
|
|
{
|
|
using var timer = new PeriodicTimer(_options.HeartbeatInterval);
|
|
try
|
|
{
|
|
while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
if (_nodeId is not { } nodeId) continue;
|
|
try
|
|
{
|
|
await _client.HeartbeatAsync(nodeId, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
|
{
|
|
_nodeId = null;
|
|
}
|
|
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
// Unreachable or refusing: the claim loop decides whether to go on; beating must not stop the node.
|
|
Log.Warning(LogCategory, "Heartbeat failed", ex);
|
|
}
|
|
}
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
}
|
|
}
|
|
|
|
private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
|
|
{
|
|
var export = ProcessingJobs.ReadExport(claimed.PresetJson);
|
|
var resolved = ExportResolver.Resolve(export);
|
|
var target = new ExportTarget(resolved, ProcessingJobs.PresetName(resolved));
|
|
if (target.Export.Container == ContainerFormat.ImageSequence) throw new InvalidOperationException(Loc.L("Errors.Render.NodeSequence"));
|
|
// One output per job: a server splits batches into one job per preset when they are posted.
|
|
if (ProcessingJobs.BatchPresets(export).Any()) throw new InvalidOperationException(Loc.L("Errors.Render.NodeBatch"));
|
|
var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export };
|
|
|
|
var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false));
|
|
var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false);
|
|
var files = await DownloadAsync(claimed, work, reporter, cancellationToken).ConfigureAwait(false);
|
|
var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id))]);
|
|
|
|
IReadOnlyList<TargetOutcome> outcomes;
|
|
await using (var render = new SequenceRender(input, project, null, _processing))
|
|
{
|
|
outcomes = await render.RenderAllAsync(
|
|
[(0, target)],
|
|
(t, setup) => ExportNaming.OutputPath(work, project.Export.NameTemplate, SequenceRender.NameContext(sequence, t, setup)),
|
|
(_, _) => ValueTask.CompletedTask,
|
|
async (_, t, result) =>
|
|
{
|
|
// Remarks first: completing the upload ends the job on the server.
|
|
foreach (var remark in SequenceRender.Remarks(result)) await reporter.LogAsync(remark).ConfigureAwait(false);
|
|
await UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken).ConfigureAwait(false);
|
|
},
|
|
reporter.ReportAsync,
|
|
cancellationToken).ConfigureAwait(false);
|
|
}
|
|
if (outcomes[0].Error is { } error) throw new InvalidOperationException(error);
|
|
}
|
|
|
|
private async Task UploadAsync(Guid nodeId, Guid jobId, ExportTarget target, RenderResult result, CancellationToken cancellationToken)
|
|
{
|
|
var size = new FileInfo(result.OutputPath).Length;
|
|
var sha = await TitanoClient.HashFileAsync(result.OutputPath, cancellationToken).ConfigureAwait(false);
|
|
var request = new RenderResultRequest(Path.GetFileName(result.OutputPath), size, sha, target.PresetName, ProcessingJobs.CodecName(result.Codec),
|
|
result.Width, result.Height, result.FrameRate.Value, result.DurationSeconds);
|
|
await _client.UploadRenderResultAsync(nodeId, jobId, result.OutputPath, request, null, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task<List<Frame>> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken)
|
|
{
|
|
var frames = new List<Frame>();
|
|
string? cursor = null;
|
|
do
|
|
{
|
|
var page = await _client.ListFramesAsync(sequenceId, cursor, 1000, cancellationToken).ConfigureAwait(false);
|
|
frames.AddRange(page.Items.Select(ToFrame));
|
|
cursor = page.NextCursor;
|
|
}
|
|
while (cursor is not null);
|
|
frames.Sort((a, b) => a.Index.CompareTo(b.Index));
|
|
return frames;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused
|
|
/// across jobs, fetched once per content), or proxies into the job folder when the job asks for them and the server
|
|
/// has them.
|
|
/// </summary>
|
|
private async Task<Dictionary<Guid, string>> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
|
|
{
|
|
var files = new Dictionary<Guid, string>();
|
|
var done = 0;
|
|
var total = claimed.Frames.Count;
|
|
await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false);
|
|
var downloads = claimed.Frames.GroupBy(f => claimed.UseProxies && f.HasProxy ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
|
|
await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken },
|
|
async (frames, ct) =>
|
|
{
|
|
var frame = frames.First();
|
|
var path = claimed.UseProxies && frame.HasProxy
|
|
? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false)
|
|
: await DownloadOriginalAsync(frame, ct).ConfigureAwait(false);
|
|
lock (files)
|
|
{
|
|
foreach (var f in frames) files[f.Id] = path;
|
|
}
|
|
await reporter.ReportAsync(Interlocked.Add(ref done, frames.Count()), total, null).ConfigureAwait(false);
|
|
}).ConfigureAwait(false);
|
|
return files;
|
|
}
|
|
|
|
/// <summary>
|
|
/// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it, and a retry
|
|
/// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server
|
|
/// has none for the frame.
|
|
/// </summary>
|
|
private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
|
{
|
|
var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg");
|
|
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
|
try
|
|
{
|
|
await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false);
|
|
return path;
|
|
}
|
|
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
|
{
|
|
File.Delete(path);
|
|
return await DownloadOriginalAsync(frame, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <summary>A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256.</summary>
|
|
private async Task<string> DownloadOriginalAsync(ManifestFrame frame, CancellationToken cancellationToken)
|
|
{
|
|
var sha = frame.Sha256.ToLowerInvariant();
|
|
if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
|
|
var path = Path.Combine(_options.CacheDirectory, "originals", sha[..2], sha + Path.GetExtension(frame.FileName).ToLowerInvariant());
|
|
if (File.Exists(path))
|
|
{
|
|
File.SetLastWriteTimeUtc(path, DateTime.UtcNow); // most recently used: kept by the cache trim
|
|
return path;
|
|
}
|
|
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
|
var part = path + ".part";
|
|
for (var attempt = 1; ; attempt++)
|
|
{
|
|
// A part as long as the file was complete when the last attempt stopped: only its hash is left to check.
|
|
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, resume: true, cancellationToken).ConfigureAwait(false);
|
|
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
|
|
{
|
|
File.Move(part, path, overwrite: true);
|
|
return path;
|
|
}
|
|
File.Delete(part);
|
|
if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Fetches a frame resource into <paramref name="path"/>. A dropped connection or a busy server is retried (up to
|
|
/// <see cref="RenderNodeOptions.DownloadRetries"/> times, the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>);
|
|
/// with <paramref name="resume"/> (a download its hash verifies) each attempt appends to what the last one left,
|
|
/// otherwise it starts over.
|
|
/// </summary>
|
|
private async Task FetchAsync(ManifestFrame frame, string kind, string path, bool resume, CancellationToken cancellationToken)
|
|
{
|
|
for (var attempt = 0; ; attempt++)
|
|
{
|
|
try
|
|
{
|
|
await using var file = new FileStream(path, resume ? FileMode.OpenOrCreate : FileMode.Create, FileAccess.ReadWrite, FileShare.None);
|
|
await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false);
|
|
return;
|
|
}
|
|
catch (Exception ex) when (attempt < _options.DownloadRetries && !cancellationToken.IsCancellationRequested && IsTransient(ex))
|
|
{
|
|
Log.Info(LogCategory, $"Download of {frame.FileName} ({kind}) interrupted, resuming: {ex.Message}");
|
|
await Task.Delay(_options.RetryDelay * (1 << attempt), cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>Failures a later attempt can get past: the network, a busy server, a timeout (not a cancellation of ours).</summary>
|
|
private static bool IsTransient(Exception ex) =>
|
|
ex is HttpRequestException or IOException or OperationCanceledException or TitanoApiException { IsTransient: true };
|
|
|
|
private async Task ReportFailureAsync(Guid nodeId, Guid jobId, string error, CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
await _client.ReportNodeFailureAsync(nodeId, jobId, error, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException or OperationCanceledException)
|
|
{
|
|
// Cancelled or reassigned meanwhile, or the server is gone: it requeues the job when the heartbeats stop.
|
|
Log.Warning(LogCategory, $"The failure of job {jobId} could not be reported", ex);
|
|
}
|
|
}
|
|
|
|
/// <summary>Deletes the least recently used originals until the cache fits its limit.</summary>
|
|
private void TrimCache()
|
|
{
|
|
try
|
|
{
|
|
var root = new DirectoryInfo(Path.Combine(_options.CacheDirectory, "originals"));
|
|
if (!root.Exists) return;
|
|
var files = root.EnumerateFiles("*", SearchOption.AllDirectories).OrderBy(f => f.LastWriteTimeUtc).ToList();
|
|
var size = files.Sum(f => f.Length);
|
|
foreach (var file in files)
|
|
{
|
|
if (size <= _options.CacheLimitBytes) break;
|
|
size -= file.Length;
|
|
file.Delete();
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
Log.Warning(LogCategory, "The render node cache could not be trimmed", ex);
|
|
}
|
|
}
|
|
|
|
private static void TryDelete(string folder)
|
|
{
|
|
try
|
|
{
|
|
if (Directory.Exists(folder)) Directory.Delete(folder, recursive: true);
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
Log.Warning(LogCategory, $"Could not remove {folder}", ex);
|
|
}
|
|
}
|
|
|
|
private static Sequence ToSequence(SequenceDto s) => new()
|
|
{
|
|
Id = s.Id,
|
|
ArchiveId = Guid.Empty,
|
|
Folder = s.Folder,
|
|
Title = s.Title,
|
|
CameraMake = s.CameraMake,
|
|
CameraModel = s.CameraModel,
|
|
Lens = s.Lens,
|
|
FirstCapture = s.FirstCapture,
|
|
LastCapture = s.LastCapture,
|
|
FrameCount = s.FrameCount,
|
|
};
|
|
|
|
private static Frame ToFrame(FrameDto f) => new()
|
|
{
|
|
Id = f.Id,
|
|
SequenceId = f.SequenceId,
|
|
Index = f.Index,
|
|
RelativePath = f.FileName,
|
|
FileName = f.FileName,
|
|
Size = f.Size,
|
|
Sha256 = f.Sha256,
|
|
CapturedAt = f.CapturedAt,
|
|
UtcOffsetMinutes = f.UtcOffsetMinutes,
|
|
ExposureTime = f.ExposureTime,
|
|
FNumber = f.FNumber,
|
|
Iso = f.Iso,
|
|
FocalLength = f.FocalLength,
|
|
Orientation = f.Orientation,
|
|
Width = f.Width,
|
|
Height = f.Height,
|
|
Format = Enum.TryParse<ImageFileFormat>(f.Format, ignoreCase: true, out var format) ? format : default,
|
|
Flags = (FrameFlags)f.Flags,
|
|
};
|
|
|
|
/// <summary>
|
|
/// Progress of the job in flight to the server, at most every interval (and at the end). A refusal (409) means the
|
|
/// job was cancelled or given to another node: the job's token is cancelled.
|
|
/// </summary>
|
|
private sealed class NodeReporter(TitanoClient client, Guid nodeId, Guid jobId, TimeSpan interval, CancellationTokenSource job)
|
|
{
|
|
private readonly Stopwatch _clock = Stopwatch.StartNew();
|
|
private readonly SemaphoreSlim _gate = new(1, 1);
|
|
private TimeSpan _last = TimeSpan.MinValue;
|
|
private long _lastDone;
|
|
private long _lastTotal;
|
|
private string? _lastMessage;
|
|
|
|
public Guid NodeId => nodeId;
|
|
|
|
/// <summary>Progress, sent at most every interval; a stage message is sent when it changes.</summary>
|
|
public ValueTask ReportAsync(long done, long total, string? message) => SendAsync(done, total, message, remark: false);
|
|
|
|
/// <summary>A line for the job's log on the server, sent with the progress last reported.</summary>
|
|
public ValueTask LogAsync(string line) => SendAsync(0, 0, line, remark: true);
|
|
|
|
private async ValueTask SendAsync(long done, long total, string? message, bool remark)
|
|
{
|
|
await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
|
|
try
|
|
{
|
|
NodeProgress progress;
|
|
if (remark)
|
|
{
|
|
progress = new NodeProgress(_lastDone, _lastTotal, null, message);
|
|
}
|
|
else
|
|
{
|
|
var now = _clock.Elapsed;
|
|
var line = message is not null && message != _lastMessage ? message : null;
|
|
if (line is null && done < total && now - _last < interval) return;
|
|
double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3);
|
|
(_last, _lastDone, _lastTotal) = (now, done, total);
|
|
if (line is not null) _lastMessage = line;
|
|
progress = new NodeProgress(done, total, rate, line);
|
|
}
|
|
await client.ReportNodeProgressAsync(nodeId, jobId, progress, job.Token).ConfigureAwait(false);
|
|
}
|
|
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict)
|
|
{
|
|
await job.CancelAsync().ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException)
|
|
{
|
|
Log.Warning(LogCategory, "Progress could not be reported", ex);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
// The job is stopping; its outcome is handled by the caller.
|
|
}
|
|
finally
|
|
{
|
|
_gate.Release();
|
|
}
|
|
}
|
|
}
|
|
}
|