feat(app): render node follows the opt-in and the server session
RenderNodeHost starts, restarts or stops this PC's render node in order and off the UI thread whenever the opt-in, the session or the index changes; a node that cannot start is retried by the next change and exit stops it. The server test covers managed renders on the server and MP4 jobs left queued for render nodes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
4 files changed
+276
-51
No files matched your search
@@ -41,9 +41,8 @@ public sealed class AppServices : IAsyncDisposable
|
||||
private readonly CancellationTokenSource _stopping = new();
|
||||
private readonly Dictionary<Guid, Guid> _scans = [];
|
||||
private Task _polling = Task.CompletedTask;
|
||||
private readonly RenderNodeHost _renderNode;
|
||||
private ServerSession _server = null!;
|
||||
private Task _nodeSwitch = Task.CompletedTask;
|
||||
private (RenderNodeClient Node, TitanoClient Client, Uri Address, string Token)? _node;
|
||||
|
||||
public AppServices(AppEnvironment environment, AppSettings settings, Action saveSettings, IAppDialogs dialogs)
|
||||
{
|
||||
@@ -51,6 +50,7 @@ public sealed class AppServices : IAsyncDisposable
|
||||
Settings = settings;
|
||||
SaveSettingsAction = saveSettings;
|
||||
Dialogs = dialogs;
|
||||
_renderNode = new RenderNodeHost(StartRenderNode);
|
||||
ArchiveSetup = new EngineArchiveSetup(this);
|
||||
Server = new ServerSession(settings, saveSettings, new TitanoServerApiFactory(), new DpapiTokenProtector());
|
||||
Encoders = new MediaFoundationInventory();
|
||||
@@ -136,8 +136,11 @@ public sealed class AppServices : IAsyncDisposable
|
||||
UpdateRenderNode();
|
||||
}
|
||||
|
||||
/// <summary>Starts or stops the render node to match the opt-in and the server session (BRIEF §14.7); switches run in order.</summary>
|
||||
public void UpdateRenderNode() => _nodeSwitch = SwitchRenderNodeAsync(_nodeSwitch);
|
||||
/// <summary>Starts or stops the render node to match the opt-in and the server session (BRIEF §14.7).</summary>
|
||||
public void UpdateRenderNode() =>
|
||||
_renderNode.Update(Settings.RenderNodeOptIn && _server.IsConnected && Processing is not null && _server.Flow is { Address: { } address, Token: { } token }
|
||||
? new RenderNodeSession(address, token)
|
||||
: null);
|
||||
|
||||
/// <summary>Opens (or creates) the index at <paramref name="databasePath"/>, migrates it and starts the job engine.</summary>
|
||||
public async Task OpenAsync(string databasePath, CancellationToken ct = default)
|
||||
@@ -172,8 +175,7 @@ public sealed class AppServices : IAsyncDisposable
|
||||
await _polling;
|
||||
_stopping.Dispose();
|
||||
_server.PropertyChanged -= OnServerChanged;
|
||||
await _nodeSwitch;
|
||||
await StopRenderNodeAsync();
|
||||
await _renderNode.DisposeAsync();
|
||||
(Jobs as IDisposable)?.Dispose();
|
||||
if (Engine is not null) await Engine.DisposeAsync();
|
||||
Thumbnails?.Dispose();
|
||||
@@ -237,45 +239,36 @@ public sealed class AppServices : IAsyncDisposable
|
||||
|
||||
private void OnServerChanged(object? sender, PropertyChangedEventArgs e) => UpdateRenderNode();
|
||||
|
||||
/// <summary>Waits for the previous switch, then runs the node for the current session, or stops it.</summary>
|
||||
private async Task SwitchRenderNodeAsync(Task previous)
|
||||
/// <summary>
|
||||
/// The render node of <paramref name="session"/>: this PC's encoders (Media Foundation, then managed), downloads and
|
||||
/// outputs in the cache folder within its size limit.
|
||||
/// </summary>
|
||||
private IAsyncDisposable StartRenderNode(RenderNodeSession session)
|
||||
{
|
||||
await previous;
|
||||
var flow = _server.Flow;
|
||||
(ProcessingService Engine, Uri Address, string Token)? wanted =
|
||||
Settings.RenderNodeOptIn && _server.IsConnected && Processing is { } processing && flow.Address is { } address && flow.Token is { } token
|
||||
? (processing, address, token)
|
||||
: null;
|
||||
if (_node is { } running && wanted is { } same && running.Address == same.Address && running.Token == same.Token) return;
|
||||
await StopRenderNodeAsync();
|
||||
if (wanted is not { } start) return;
|
||||
try
|
||||
var processing = Processing ?? throw new InvalidOperationException("The index is not open.");
|
||||
var cache = CacheStore.Folder(Environment, Settings);
|
||||
CacheStore.Prepare(cache);
|
||||
var client = new TitanoClient(session.Address, new CertificatePinning(_server.Fingerprints), $"Titano ({System.Environment.MachineName})")
|
||||
{
|
||||
var cache = CacheStore.Folder(Environment, Settings);
|
||||
CacheStore.Prepare(cache);
|
||||
var client = new TitanoClient(start.Address, new CertificatePinning(_server.Fingerprints), $"Titano ({System.Environment.MachineName})") { Token = start.Token };
|
||||
var node = new RenderNodeClient(client, start.Engine.Options, new RenderNodeOptions
|
||||
{
|
||||
ClientName = System.Environment.MachineName,
|
||||
CacheDirectory = Path.Combine(cache, "render-node"),
|
||||
CacheLimitBytes = Settings.CacheLimitGb * (1L << 30),
|
||||
});
|
||||
node.Start();
|
||||
_node = (node, client, start.Address, start.Token);
|
||||
Log.Info("render-node", $"Offering this PC as render node to {start.Address}");
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
Token = session.Token,
|
||||
};
|
||||
var node = new RenderNodeClient(client, processing.Options, new RenderNodeOptions
|
||||
{
|
||||
Log.Warning("render-node", "The render node could not start", ex);
|
||||
}
|
||||
ClientName = System.Environment.MachineName,
|
||||
CacheDirectory = Path.Combine(cache, "render-node"),
|
||||
CacheLimitBytes = Settings.CacheLimitGb * (1L << 30),
|
||||
});
|
||||
node.Start();
|
||||
return new RunningNode(node, client);
|
||||
}
|
||||
|
||||
private async Task StopRenderNodeAsync()
|
||||
/// <summary>A started node and the client it owns.</summary>
|
||||
private sealed class RunningNode(RenderNodeClient node, TitanoClient client) : IAsyncDisposable
|
||||
{
|
||||
if (_node is not { } running) return;
|
||||
_node = null;
|
||||
await running.Node.DisposeAsync();
|
||||
running.Client.Dispose();
|
||||
Log.Info("render-node", "Render node stopped");
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await node.DisposeAsync().ConfigureAwait(false);
|
||||
client.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
using Titano.Core.Diagnostics;
|
||||
|
||||
namespace Titano.App.Infrastructure;
|
||||
|
||||
/// <summary>The server session a render node works for: a change of either value means another node.</summary>
|
||||
public sealed record RenderNodeSession(Uri Address, string Token);
|
||||
|
||||
/// <summary>
|
||||
/// Keeps this PC's render node (BRIEF §14.7) in line with "Offer this PC as render node" and the server session:
|
||||
/// <see cref="Update"/> starts the node for the wanted session, restarts it for another session and stops it when none
|
||||
/// is wanted. Switches run one after the other, off the caller's thread, and never fail: a node that cannot start is
|
||||
/// logged and stays off until the next update.
|
||||
/// </summary>
|
||||
public sealed class RenderNodeHost(Func<RenderNodeSession, IAsyncDisposable> start) : IAsyncDisposable
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
private Task _switching = Task.CompletedTask;
|
||||
private (RenderNodeSession Session, IAsyncDisposable Node)? _running;
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>The session the node runs for; null when it is off.</summary>
|
||||
public RenderNodeSession? Session
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate) return _running?.Session;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Completes when every switch requested so far has run.</summary>
|
||||
public Task Switching
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate) return _switching;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Runs the node for <paramref name="wanted"/>, or stops it when null.</summary>
|
||||
public void Update(RenderNodeSession? wanted)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_disposed) return;
|
||||
var previous = _switching;
|
||||
_switching = Task.Run(async () =>
|
||||
{
|
||||
await previous.ConfigureAwait(false);
|
||||
await SwitchAsync(wanted).ConfigureAwait(false);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Stops the node after the pending switches; later updates are ignored.</summary>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
Task pending;
|
||||
lock (_gate)
|
||||
{
|
||||
_disposed = true;
|
||||
pending = _switching;
|
||||
}
|
||||
await pending.ConfigureAwait(false);
|
||||
await SwitchAsync(null).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task SwitchAsync(RenderNodeSession? wanted)
|
||||
{
|
||||
(RenderNodeSession Session, IAsyncDisposable Node)? running;
|
||||
lock (_gate) running = _running;
|
||||
if (running?.Session == wanted) return;
|
||||
if (running is { } old)
|
||||
{
|
||||
lock (_gate) _running = null;
|
||||
try
|
||||
{
|
||||
await old.Node.DisposeAsync().ConfigureAwait(false);
|
||||
Log.Info("render-node", "Render node stopped");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A node that fails to stop is gone anyway; the next one must still be able to start.
|
||||
Log.Warning("render-node", "The render node did not stop cleanly", ex);
|
||||
}
|
||||
}
|
||||
if (wanted is null) return;
|
||||
try
|
||||
{
|
||||
var node = start(wanted);
|
||||
lock (_gate) _running = (wanted, node);
|
||||
Log.Info("render-node", $"Offering this PC as render node to {wanted.Address}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning("render-node", "The render node could not start", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using System.IO;
|
||||
using Titano.App.Infrastructure;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.App.Tests.Server;
|
||||
|
||||
public sealed class RenderNodeHostTests
|
||||
{
|
||||
private static readonly RenderNodeSession Nas = new(new Uri("https://nas.local:8443/"), "token-a");
|
||||
|
||||
/// <summary>Records the nodes it starts; fails the first <see cref="Failures"/> starts.</summary>
|
||||
private sealed class Nodes
|
||||
{
|
||||
public List<(RenderNodeSession Session, Node Node)> Started { get; } = [];
|
||||
public int Failures { get; set; }
|
||||
|
||||
public IAsyncDisposable Start(RenderNodeSession session)
|
||||
{
|
||||
if (Failures-- > 0) throw new IOException("cache folder unavailable");
|
||||
var node = new Node();
|
||||
Started.Add((session, node));
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Node : IAsyncDisposable
|
||||
{
|
||||
public bool Stopped { get; private set; }
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
Stopped = true;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheNodeFollowsTheWantedSession()
|
||||
{
|
||||
var nodes = new Nodes();
|
||||
await using var host = new RenderNodeHost(nodes.Start);
|
||||
|
||||
host.Update(Nas);
|
||||
host.Update(Nas);
|
||||
await host.Switching;
|
||||
Assert.Equal(Nas, host.Session);
|
||||
Assert.Count(1, nodes.Started);
|
||||
|
||||
// A new token (signed in again) is another session: the old node stops before the new one starts.
|
||||
var renewed = Nas with { Token = "token-b" };
|
||||
host.Update(renewed);
|
||||
await host.Switching;
|
||||
Assert.True(nodes.Started[0].Node.Stopped, "old node stopped");
|
||||
Assert.Equal((renewed, false), (nodes.Started[1].Session, nodes.Started[1].Node.Stopped));
|
||||
|
||||
host.Update(null);
|
||||
await host.Switching;
|
||||
Assert.Null(host.Session);
|
||||
Assert.True(nodes.Started[1].Node.Stopped, "stopped when no longer wanted");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANodeThatCannotStartIsRetriedByTheNextUpdate()
|
||||
{
|
||||
var nodes = new Nodes { Failures = 1 };
|
||||
await using var host = new RenderNodeHost(nodes.Start);
|
||||
|
||||
host.Update(Nas);
|
||||
await host.Switching;
|
||||
Assert.Null(host.Session);
|
||||
|
||||
host.Update(Nas);
|
||||
await host.Switching;
|
||||
Assert.Equal(Nas, host.Session);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DisposingStopsTheNodeAndIgnoresLaterUpdates()
|
||||
{
|
||||
var nodes = new Nodes();
|
||||
var host = new RenderNodeHost(nodes.Start);
|
||||
host.Update(Nas);
|
||||
await host.DisposeAsync();
|
||||
|
||||
Assert.True(nodes.Started.Single().Node.Stopped, "stopped on exit");
|
||||
host.Update(Nas);
|
||||
await host.Switching;
|
||||
Assert.Count(1, nodes.Started);
|
||||
}
|
||||
}
|
||||
@@ -5,15 +5,23 @@ using Titano.Core.Diagnostics.Synthetic;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Data.Processing;
|
||||
using Titano.Server.Jobs;
|
||||
using Titano.Server.Storage;
|
||||
using Titano.Server.Tests.Jobs;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Server.Tests.Storage;
|
||||
|
||||
/// <summary>The production composition: frames committed by the archive engine are light-analysed by the processing handlers.</summary>
|
||||
/// <summary>
|
||||
/// The production composition: frames committed by the archive engine are light-analysed by the processing handlers,
|
||||
/// and the server renders the managed formats itself.
|
||||
/// </summary>
|
||||
[Category("processing")]
|
||||
public sealed class ProcessingModuleTests(TestContext context)
|
||||
{
|
||||
private const int Frames = 10;
|
||||
|
||||
/// <summary>The registrations of Program.cs.</summary>
|
||||
private static void Production(IServiceCollection services)
|
||||
{
|
||||
@@ -26,15 +34,7 @@ public sealed class ProcessingModuleTests(TestContext context)
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: Production);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 10, StartEv = 0.5, EndEv = -0.5 });
|
||||
Guid sequenceId = default;
|
||||
for (var i = 0; i < scene.Count; i++)
|
||||
{
|
||||
using var image = scene.Render(i);
|
||||
var exif = ExifWriter.BuildJpegApp1(scene.Metadata[i]);
|
||||
var photo = JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90, App1Segments = [exif[4..]] });
|
||||
sequenceId = (await Uploads.UploadAsync(editor, photo, $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
}
|
||||
var sequenceId = await UploadSceneAsync(editor);
|
||||
|
||||
// The commits queue a light analysis; its handler comes from the processing module.
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
@@ -42,7 +42,7 @@ public sealed class ProcessingModuleTests(TestContext context)
|
||||
while (true)
|
||||
{
|
||||
sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
|
||||
if (sequence.Status.HasFlag(SequenceStatus.AnalyzedLight) && sequence.FrameCount == scene.Count
|
||||
if (sequence.Status.HasFlag(SequenceStatus.AnalyzedLight) && sequence.FrameCount == Frames
|
||||
&& !await server.Database.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequenceId)) break;
|
||||
Assert.True(DateTime.UtcNow < until, $"not analysed: {sequence.Status}, {sequence.FrameCount} frames");
|
||||
await Task.Delay(100);
|
||||
@@ -53,4 +53,48 @@ public sealed class ProcessingModuleTests(TestContext context)
|
||||
Assert.FileExists(folder.ProjectPath);
|
||||
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheServerRendersManagedFormatsAndLeavesMp4ToRenderNodes()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: Production);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var sequence = (await server.Database.Sequences.GetAsync(await UploadSceneAsync(editor)))!;
|
||||
// A client's export folder means nothing on the server: the output still goes to the sequence's exports.
|
||||
var avi = new ExportSettings
|
||||
{
|
||||
Preset = new Auto<string>().WithManual("mjpeg-avi"),
|
||||
Overrides = new ExportOverrides { Width = 96, Height = 64 },
|
||||
NameTemplate = "clip",
|
||||
Folder = context.CreateTempDirectory("elsewhere"),
|
||||
};
|
||||
var hevc = new ExportSettings { Preset = new Auto<string>().WithManual("4k-hevc") };
|
||||
|
||||
var mp4 = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, hevc, 96, 64).ParametersJson);
|
||||
var managed = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, avi, 96, 64).ParametersJson);
|
||||
var done = await JobApiTests.WaitForAsync(editor, managed.Id, "done");
|
||||
|
||||
Assert.Equal(JobEngineHost.WorkerId, done.NodeId?.Split('#')[0]);
|
||||
var render = (await server.Database.Renders.ListBySequenceAsync(sequence.Id)).Single();
|
||||
Assert.Equal(($"{sequence.Folder}/exports/clip.avi", server.Options.ServerName, "Mjpeg"), (render.OutputPath, render.ProducedBy, render.Codec));
|
||||
Assert.FileExists(Path.Combine(server.Options.ArchivePath, render.OutputPath));
|
||||
Assert.Empty(Directory.EnumerateFileSystemEntries(avi.Folder));
|
||||
var waiting = await JobApiTests.GetAsync(editor, mp4.Id);
|
||||
Assert.Equal(("queued", (string?)null), (waiting.State, waiting.NodeId));
|
||||
}
|
||||
|
||||
/// <summary>Uploads a synthetic scene with Exif into one new sequence; returns its id.</summary>
|
||||
private static async Task<Guid> UploadSceneAsync(HttpClient editor)
|
||||
{
|
||||
var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
|
||||
Guid sequenceId = default;
|
||||
for (var i = 0; i < scene.Count; i++)
|
||||
{
|
||||
using var image = scene.Render(i);
|
||||
var exif = ExifWriter.BuildJpegApp1(scene.Metadata[i]);
|
||||
var photo = JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90, App1Segments = [exif[4..]] });
|
||||
sequenceId = (await Uploads.UploadAsync(editor, photo, $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
}
|
||||
return sequenceId;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user