merge: processing work in progress (analysis/render handlers, render node client)
This commit is contained in:
30 files changed
+2262
-21
No files matched your search
@@ -31,6 +31,11 @@
|
||||
<data name="Advisor.Finding.UnreadableFrames" xml:space="preserve"><value>{0} frames could not be decoded.</value><comment>{0} frame count</comment></data>
|
||||
<data name="Advisor.Finding.WhiteBalanceJitter" xml:space="preserve"><value>Colour balance jumps by {0:0.#}% between frames.</value><comment>{0} RMS colour-ratio jitter in percent</comment></data>
|
||||
<data name="Advisor.Finding.WhiteFrames" xml:space="preserve"><value>{0} frames are blown out to white.</value><comment>{0} frame count</comment></data>
|
||||
<data name="Advisor.Log.AutoUpdated" xml:space="preserve"><value>Automatic values of {0} updated.</value><comment>job log line; {0} sequence title</comment></data>
|
||||
<data name="Advisor.Log.DeepDone" xml:space="preserve"><value>Deep analysis of {0}: {1} frames, {2} warnings.</value><comment>job log line; {0} sequence title; {1} frame count; {2} warning count</comment></data>
|
||||
<data name="Advisor.Log.LightDone" xml:space="preserve"><value>Light analysis of {0}: {1} frames, {2} warnings.</value><comment>job log line; {0} sequence title; {1} frame count; {2} warning count</comment></data>
|
||||
<data name="Advisor.Log.Presets" xml:space="preserve"><value>Suggested presets: {0}</value><comment>job log line; {0} preset names separated by dots</comment></data>
|
||||
<data name="Advisor.Log.ProjectUnreadable" xml:space="preserve"><value>The project settings of {0} cannot be read; their automatic values were left as they were.</value><comment>job log line; {0} sequence title</comment></data>
|
||||
<data name="Advisor.Missing.Deep" xml:space="preserve"><value>Needs the deep analysis of the pixels, which has not run yet; nothing was changed.</value><comment>no placeholders</comment></data>
|
||||
<data name="Advisor.Missing.Exposure" xml:space="preserve"><value>Shutter time or aperture is missing from the metadata, so this cannot be decided; nothing was changed.</value><comment>no placeholders</comment></data>
|
||||
<data name="Advisor.Missing.Motion" xml:space="preserve"><value>Too few frame pairs to measure motion; nothing was changed.</value><comment>no placeholders</comment></data>
|
||||
|
||||
@@ -11,6 +11,11 @@
|
||||
<data name="Errors.Path.IsFile" xml:space="preserve"><value>This path is a file, not a folder.</value></data>
|
||||
<data name="Errors.Path.NoDrive" xml:space="preserve"><value>The drive of this folder is not available.</value></data>
|
||||
<data name="Errors.Path.NotAbsolute" xml:space="preserve"><value>Enter a full path that starts with a drive or a network share.</value></data>
|
||||
<data name="Errors.Render.Download" xml:space="preserve"><value>{0} could not be downloaded intact.</value><comment>{0} file name</comment></data>
|
||||
<data name="Errors.Render.Failed" xml:space="preserve"><value>{0} of {1} exports failed; see the job log.</value><comment>{0} failed count; {1} total count</comment></data>
|
||||
<data name="Errors.Render.NodeSequence" xml:space="preserve"><value>Render nodes send back one file per job; image sequences render where the archive is.</value></data>
|
||||
<data name="Errors.Render.Parameters" xml:space="preserve"><value>The render job has no valid export settings.</value></data>
|
||||
<data name="Errors.Render.Sequence" xml:space="preserve"><value>The sequence to render is no longer in the archive.</value></data>
|
||||
<data name="Errors.Server.Address" xml:space="preserve"><value>Enter a server address such as nas.local:8443 or https://titano.example.com.</value></data>
|
||||
<data name="Errors.Settings.Import" xml:space="preserve"><value>This file does not contain Titano settings.</value></data>
|
||||
<data name="Errors.Template.Empty" xml:space="preserve"><value>The template is empty.</value></data>
|
||||
|
||||
@@ -13,6 +13,10 @@
|
||||
<data name="Export.Encoder.ResolutionCappedByLevel" xml:space="preserve"><value>Resolution reduced to {1}, the maximum of level {0}.</value><comment>long; {0} codec level name; {1} size as WIDTHxHEIGHT</comment></data>
|
||||
<data name="Export.Encoder.SoftwareOnly" xml:space="preserve"><value>No hardware encoder: {0} encodes in software, which is slower.</value><comment>long; {0} encoder name</comment></data>
|
||||
<data name="Export.Encoder.TenBitUnavailable" xml:space="preserve"><value>{0} cannot encode 10-bit; 8-bit is used.</value><comment>long; {0} encoder name</comment></data>
|
||||
<data name="Export.Job.Downloading" xml:space="preserve"><value>Downloading frames</value><comment>render node progress stage</comment></data>
|
||||
<data name="Export.Job.Preparing" xml:space="preserve"><value>Measuring frames</value><comment>job progress stage</comment></data>
|
||||
<data name="Export.Log.Failed" xml:space="preserve"><value>{0} failed: {1}</value><comment>long: job log line; {0} preset name; {1} reason</comment></data>
|
||||
<data name="Export.Log.Rendered" xml:space="preserve"><value>{0} saved as {1}</value><comment>long: job log line; {0} preset name; {1} output file name</comment></data>
|
||||
<data name="Export.Plan.FrameRateIgnored" xml:space="preserve"><value>Playback rate not understood; the export frame rate is used.</value><comment>long</comment></data>
|
||||
<data name="Export.Plan.NoCaptureTimes" xml:space="preserve"><value>No capture times: frames are spaced evenly.</value><comment>long</comment></data>
|
||||
<data name="Export.Plan.NoFrames" xml:space="preserve"><value>Every frame is excluded.</value><comment>long</comment></data>
|
||||
@@ -28,6 +32,7 @@
|
||||
<data name="Export.Preset.8kArchive" xml:space="preserve"><value>8K archive</value></data>
|
||||
<data name="Export.Preset.Apng" xml:space="preserve"><value>Animated PNG</value></data>
|
||||
<data name="Export.Preset.Cinema" xml:space="preserve"><value>Cinema 21:9</value><comment>invariant</comment></data>
|
||||
<data name="Export.Preset.Custom" xml:space="preserve"><value>Custom</value><comment>preset name when the values match no preset</comment></data>
|
||||
<data name="Export.Preset.Dci4k" xml:space="preserve"><value>DCI 4K</value><comment>invariant</comment></data>
|
||||
<data name="Export.Preset.Gif" xml:space="preserve"><value>Animated GIF</value></data>
|
||||
<data name="Export.Preset.Jpeg" xml:space="preserve"><value>JPEG frames</value></data>
|
||||
@@ -44,6 +49,7 @@
|
||||
<data name="Export.Render.BlurOffWhileStacking" xml:space="preserve"><value>Motion blur is off while frames are stacked.</value><comment>long</comment></data>
|
||||
<data name="Export.Render.CropIgnored" xml:space="preserve"><value>The crop lies outside the frame and is ignored.</value><comment>long</comment></data>
|
||||
<data name="Export.Render.EasingIgnored" xml:space="preserve"><value>Easing “{0}” not understood; linear is used.</value><comment>long; {0} easing name as entered</comment></data>
|
||||
<data name="Export.Render.FrameUnreadable" xml:space="preserve"><value>Frame {0} ({1}) cannot be decoded; exclude it or check the file.</value><comment>long; {0} frame number from 1; {1} file name</comment></data>
|
||||
<data name="Export.Render.KeyframeIgnored" xml:space="preserve"><value>The keyframe at frame {0} is invalid and was skipped.</value><comment>long; {0} source frame number</comment></data>
|
||||
<data name="Export.Render.KneeIgnored" xml:space="preserve"><value>Highlight knee {0} is out of range and is ignored.</value><comment>long; {0} knee value as entered</comment></data>
|
||||
<data name="Export.Render.NoFrames" xml:space="preserve"><value>Every frame is excluded; there is nothing to render.</value><comment>long</comment></data>
|
||||
|
||||
@@ -31,6 +31,11 @@
|
||||
<data name="Advisor.Finding.UnreadableFrames" xml:space="preserve"><value>{0} fotogrammi non sono decodificabili.</value></data>
|
||||
<data name="Advisor.Finding.WhiteBalanceJitter" xml:space="preserve"><value>Il bilanciamento del colore varia del {0:0.#}% tra un fotogramma e l'altro.</value></data>
|
||||
<data name="Advisor.Finding.WhiteFrames" xml:space="preserve"><value>{0} fotogrammi sono completamente bruciati.</value></data>
|
||||
<data name="Advisor.Log.AutoUpdated" xml:space="preserve"><value>Valori automatici di {0} aggiornati.</value></data>
|
||||
<data name="Advisor.Log.DeepDone" xml:space="preserve"><value>Analisi approfondita di {0}: {1} fotogrammi, {2} avvisi.</value></data>
|
||||
<data name="Advisor.Log.LightDone" xml:space="preserve"><value>Analisi rapida di {0}: {1} fotogrammi, {2} avvisi.</value></data>
|
||||
<data name="Advisor.Log.Presets" xml:space="preserve"><value>Preset suggeriti: {0}</value></data>
|
||||
<data name="Advisor.Log.ProjectUnreadable" xml:space="preserve"><value>Le impostazioni del progetto di {0} non sono leggibili; i valori automatici sono rimasti invariati.</value></data>
|
||||
<data name="Advisor.Missing.Deep" xml:space="preserve"><value>Serve l'analisi approfondita dei pixel, non ancora eseguita; nulla è stato cambiato.</value></data>
|
||||
<data name="Advisor.Missing.Exposure" xml:space="preserve"><value>Tempo di scatto o diaframma mancano nei metadati, quindi non si può decidere; nulla è stato cambiato.</value></data>
|
||||
<data name="Advisor.Missing.Motion" xml:space="preserve"><value>Troppo poche coppie di fotogrammi per misurare il movimento; nulla è stato cambiato.</value></data>
|
||||
|
||||
@@ -11,6 +11,11 @@
|
||||
<data name="Errors.Path.IsFile" xml:space="preserve"><value>Questo percorso è un file, non una cartella.</value></data>
|
||||
<data name="Errors.Path.NoDrive" xml:space="preserve"><value>L'unità di questa cartella non è disponibile.</value></data>
|
||||
<data name="Errors.Path.NotAbsolute" xml:space="preserve"><value>Inserisci un percorso completo che inizi con un'unità o una condivisione di rete.</value></data>
|
||||
<data name="Errors.Render.Download" xml:space="preserve"><value>{0} non è stato scaricato integro.</value></data>
|
||||
<data name="Errors.Render.Failed" xml:space="preserve"><value>{0} esportazioni su {1} non riuscite; vedi il registro del job.</value></data>
|
||||
<data name="Errors.Render.NodeSequence" xml:space="preserve"><value>I nodi di rendering restituiscono un file per job; le sequenze di immagini si renderizzano dove si trova l'archivio.</value></data>
|
||||
<data name="Errors.Render.Parameters" xml:space="preserve"><value>Il job di rendering non ha impostazioni di esportazione valide.</value></data>
|
||||
<data name="Errors.Render.Sequence" xml:space="preserve"><value>La sequenza da renderizzare non è più nell'archivio.</value></data>
|
||||
<data name="Errors.Server.Address" xml:space="preserve"><value>Inserisci l'indirizzo di un server, ad esempio nas.local:8443 o https://titano.example.com.</value></data>
|
||||
<data name="Errors.Settings.Import" xml:space="preserve"><value>Questo file non contiene impostazioni di Titano.</value></data>
|
||||
<data name="Errors.Template.Empty" xml:space="preserve"><value>Il modello è vuoto.</value></data>
|
||||
|
||||
@@ -13,6 +13,10 @@
|
||||
<data name="Export.Encoder.ResolutionCappedByLevel" xml:space="preserve"><value>Risoluzione ridotta a {1}, il massimo del livello {0}.</value></data>
|
||||
<data name="Export.Encoder.SoftwareOnly" xml:space="preserve"><value>Nessun encoder hardware: {0} codifica via software, più lentamente.</value></data>
|
||||
<data name="Export.Encoder.TenBitUnavailable" xml:space="preserve"><value>{0} non codifica a 10 bit; viene usato 8 bit.</value></data>
|
||||
<data name="Export.Job.Downloading" xml:space="preserve"><value>Download dei fotogrammi</value></data>
|
||||
<data name="Export.Job.Preparing" xml:space="preserve"><value>Misura dei fotogrammi</value></data>
|
||||
<data name="Export.Log.Failed" xml:space="preserve"><value>{0} non riuscito: {1}</value></data>
|
||||
<data name="Export.Log.Rendered" xml:space="preserve"><value>{0} salvato come {1}</value></data>
|
||||
<data name="Export.Plan.FrameRateIgnored" xml:space="preserve"><value>Velocità di riproduzione non valida; viene usata la frequenza di esportazione.</value></data>
|
||||
<data name="Export.Plan.NoCaptureTimes" xml:space="preserve"><value>Nessun orario di scatto: fotogrammi distribuiti in modo uniforme.</value></data>
|
||||
<data name="Export.Plan.NoFrames" xml:space="preserve"><value>Tutti i fotogrammi sono esclusi.</value></data>
|
||||
@@ -28,6 +32,7 @@
|
||||
<data name="Export.Preset.8kArchive" xml:space="preserve"><value>Archivio 8K</value></data>
|
||||
<data name="Export.Preset.Apng" xml:space="preserve"><value>PNG animato</value></data>
|
||||
<data name="Export.Preset.Cinema" xml:space="preserve"><value>Cinema 21:9</value></data>
|
||||
<data name="Export.Preset.Custom" xml:space="preserve"><value>Personalizzato</value></data>
|
||||
<data name="Export.Preset.Dci4k" xml:space="preserve"><value>DCI 4K</value></data>
|
||||
<data name="Export.Preset.Gif" xml:space="preserve"><value>GIF animata</value></data>
|
||||
<data name="Export.Preset.Jpeg" xml:space="preserve"><value>Fotogrammi JPEG</value></data>
|
||||
@@ -44,6 +49,7 @@
|
||||
<data name="Export.Render.BlurOffWhileStacking" xml:space="preserve"><value>La sfocatura di movimento è disattivata mentre i fotogrammi sono sovrapposti.</value></data>
|
||||
<data name="Export.Render.CropIgnored" xml:space="preserve"><value>Il ritaglio è fuori dall'immagine e viene ignorato.</value></data>
|
||||
<data name="Export.Render.EasingIgnored" xml:space="preserve"><value>Andamento «{0}» non valido; viene usato quello lineare.</value></data>
|
||||
<data name="Export.Render.FrameUnreadable" xml:space="preserve"><value>Il fotogramma {0} ({1}) non è decodificabile; escludilo o controlla il file.</value></data>
|
||||
<data name="Export.Render.KeyframeIgnored" xml:space="preserve"><value>Il fotogramma chiave {0} non è valido ed è stato saltato.</value></data>
|
||||
<data name="Export.Render.KneeIgnored" xml:space="preserve"><value>Compressione delle alte luci {0} fuori intervallo, ignorata.</value></data>
|
||||
<data name="Export.Render.NoFrames" xml:space="preserve"><value>Tutti i fotogrammi sono esclusi: non c'è nulla da renderizzare.</value></data>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.ComponentModel;
|
||||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using System.Reflection;
|
||||
@@ -8,21 +9,29 @@ using Titano.App.Server;
|
||||
using Titano.App.Settings;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Video.Managed;
|
||||
using Titano.Core.Windows.MediaFoundation;
|
||||
using Titano.Core.Windows.Wic;
|
||||
using Titano.Data;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Jobs;
|
||||
using Titano.Data.Processing;
|
||||
using Titano.Protocol.Client;
|
||||
|
||||
namespace Titano.App.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Composition root of local mode (BRIEF §4.4): the SQLite index under the data folder (migrated on open), the
|
||||
/// archive engine (WIC first, then managed decoders), the one <see cref="JobEngine"/> of this PC with the handlers of
|
||||
/// <see cref="CreateJobHandlers"/>, thumbnails, the server session, the encoder inventory and the update check.
|
||||
/// Started after the window shows, stopped when the app exits. The last archive is reopened at start and rescanned
|
||||
/// every <see cref="ScanInterval"/> (BRIEF §7: polling, no file-system watchers).
|
||||
/// archive engine (WIC first, then managed decoders), the analysis and render engine (the same decoders; Media
|
||||
/// Foundation, then managed encoders), the one <see cref="JobEngine"/> of this PC with the handlers of
|
||||
/// <see cref="CreateJobHandlers"/>, thumbnails, the server session, the render node, the encoder inventory and the
|
||||
/// update check. Started after the window shows, stopped when the app exits. The last archive is reopened at start and
|
||||
/// rescanned every <see cref="ScanInterval"/> (BRIEF §7: polling, no file-system watchers). The render node runs while
|
||||
/// the user offers this PC (<see cref="AppSettings.RenderNodeOptIn"/>) and a server session is open.
|
||||
/// </summary>
|
||||
public sealed class AppServices : IAsyncDisposable
|
||||
{
|
||||
@@ -32,6 +41,9 @@ public sealed class AppServices : IAsyncDisposable
|
||||
private readonly CancellationTokenSource _stopping = new();
|
||||
private readonly Dictionary<Guid, Guid> _scans = [];
|
||||
private Task _polling = Task.CompletedTask;
|
||||
private ServerSession _server = null!;
|
||||
private Task _nodeSwitch = Task.CompletedTask;
|
||||
private (RenderNodeClient Node, TitanoClient Client, Uri Address, string Token)? _node;
|
||||
|
||||
public AppServices(AppEnvironment environment, AppSettings settings, Action saveSettings, IAppDialogs dialogs)
|
||||
{
|
||||
@@ -57,7 +69,20 @@ public sealed class AppServices : IAsyncDisposable
|
||||
/// <summary>The archive engine; null until the index is open.</summary>
|
||||
public ArchiveService? Archives { get; private set; }
|
||||
|
||||
public ServerSession Server { get; set; }
|
||||
/// <summary>Analysis and render engine; null until the index is open.</summary>
|
||||
public ProcessingService? Processing { get; private set; }
|
||||
|
||||
public ServerSession Server
|
||||
{
|
||||
get => _server;
|
||||
set
|
||||
{
|
||||
if (_server is not null) _server.PropertyChanged -= OnServerChanged;
|
||||
_server = value;
|
||||
_server.PropertyChanged += OnServerChanged;
|
||||
UpdateRenderNode();
|
||||
}
|
||||
}
|
||||
|
||||
public IEncoderInventory Encoders { get; set; }
|
||||
|
||||
@@ -86,7 +111,7 @@ public sealed class AppServices : IAsyncDisposable
|
||||
/// <summary>The job handlers of this PC's engine — the single registration list; a kind without a handler stays
|
||||
/// queued (it is never claimed here).</summary>
|
||||
public static IReadOnlyList<IJobHandler> CreateJobHandlers(AppServices services) =>
|
||||
services.Archives?.CreateJobHandlers() ?? [];
|
||||
[.. services.Archives?.CreateJobHandlers() ?? [], .. services.Processing?.CreateJobHandlers() ?? []];
|
||||
|
||||
/// <summary>Queues a scan of <paramref name="archive"/> unless one is already waiting or running.</summary>
|
||||
public async Task QueueScanAsync(Archive archive, bool full, CancellationToken ct = default)
|
||||
@@ -105,7 +130,14 @@ public sealed class AppServices : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
public void SaveSettings() => SaveSettingsAction();
|
||||
public void SaveSettings()
|
||||
{
|
||||
SaveSettingsAction();
|
||||
UpdateRenderNode();
|
||||
}
|
||||
|
||||
/// <summary>Starts or stops the render node to match the opt-in and the server session (BRIEF §14.7); switches run in order.</summary>
|
||||
public void UpdateRenderNode() => _nodeSwitch = SwitchRenderNodeAsync(_nodeSwitch);
|
||||
|
||||
/// <summary>Opens (or creates) the index at <paramref name="databasePath"/>, migrates it and starts the job engine.</summary>
|
||||
public async Task OpenAsync(string databasePath, CancellationToken ct = default)
|
||||
@@ -139,6 +171,9 @@ public sealed class AppServices : IAsyncDisposable
|
||||
await _stopping.CancelAsync();
|
||||
await _polling;
|
||||
_stopping.Dispose();
|
||||
_server.PropertyChanged -= OnServerChanged;
|
||||
await _nodeSwitch;
|
||||
await StopRenderNodeAsync();
|
||||
(Jobs as IDisposable)?.Dispose();
|
||||
if (Engine is not null) await Engine.DisposeAsync();
|
||||
Thumbnails?.Dispose();
|
||||
@@ -171,7 +206,9 @@ public sealed class AppServices : IAsyncDisposable
|
||||
|
||||
private async Task StartJobsAsync(TitanoDatabase database, bool startEngine, CancellationToken ct)
|
||||
{
|
||||
Archives = new ArchiveService(database, new ArchiveServiceOptions { Decoder = DecoderFactory.CreateClient(new WicDecoderProvider()) });
|
||||
var archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateClient(new WicDecoderProvider()) };
|
||||
Archives = new ArchiveService(database, archive);
|
||||
Processing = new ProcessingService(database, ProcessingOptions(archive));
|
||||
var handlers = CreateJobHandlers(this);
|
||||
var kinds = handlers.Select(h => h.Kind).Distinct().ToArray();
|
||||
Engine = new JobEngine(database, handlers, new JobEngineOptions { WorkerId = WorkerId, Kinds = kinds });
|
||||
@@ -180,5 +217,65 @@ public sealed class AppServices : IAsyncDisposable
|
||||
if (started) await Engine.StartAsync(ct);
|
||||
Jobs = new EngineJobQueue(database, Engine, started);
|
||||
Log.Info("jobs", started ? $"Job engine started with {Engine.Options.Workers} worker(s) for {string.Join(", ", kinds)}" : "No job handlers registered; jobs stay queued");
|
||||
UpdateRenderNode();
|
||||
}
|
||||
|
||||
/// <summary>Media Foundation (hardware first, software H.264) then managed encoders; frames spill into the cache folder
|
||||
/// within the memory budget of Settings › Rendering (Auto: a quarter of the installed memory).</summary>
|
||||
private ProcessingOptions ProcessingOptions(ArchiveServiceOptions archive) => new()
|
||||
{
|
||||
Archive = archive,
|
||||
Encoders = [new MediaFoundationEncoderProvider(), new ManagedEncoderProvider()],
|
||||
Host = EncoderHost.Client,
|
||||
ProducedBy = System.Environment.MachineName,
|
||||
FrameWindow = new FrameWindowOptions
|
||||
{
|
||||
MemoryBudget = Settings.MemoryBudgetGb is { } gb ? gb * (1L << 30) : GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 4,
|
||||
SpillDirectory = CacheStore.Folder(Environment, Settings),
|
||||
},
|
||||
};
|
||||
|
||||
private void OnServerChanged(object? sender, PropertyChangedEventArgs e) => UpdateRenderNode();
|
||||
|
||||
/// <summary>Waits for the previous switch, then runs the node for the current session, or stops it.</summary>
|
||||
private async Task SwitchRenderNodeAsync(Task previous)
|
||||
{
|
||||
await previous;
|
||||
var flow = _server.Flow;
|
||||
(ProcessingService Engine, Uri Address, string Token)? wanted =
|
||||
Settings.RenderNodeOptIn && _server.IsConnected && Processing is { } processing && flow.Address is { } address && flow.Token is { } token
|
||||
? (processing, address, token)
|
||||
: null;
|
||||
if (_node is { } running && wanted is { } same && running.Address == same.Address && running.Token == same.Token) return;
|
||||
await StopRenderNodeAsync();
|
||||
if (wanted is not { } start) return;
|
||||
try
|
||||
{
|
||||
var cache = CacheStore.Folder(Environment, Settings);
|
||||
CacheStore.Prepare(cache);
|
||||
var client = new TitanoClient(start.Address, new CertificatePinning(_server.Fingerprints), $"Titano ({System.Environment.MachineName})") { Token = start.Token };
|
||||
var node = new RenderNodeClient(client, start.Engine.Options, new RenderNodeOptions
|
||||
{
|
||||
ClientName = System.Environment.MachineName,
|
||||
CacheDirectory = Path.Combine(cache, "render-node"),
|
||||
CacheLimitBytes = Settings.CacheLimitGb * (1L << 30),
|
||||
});
|
||||
node.Start();
|
||||
_node = (node, client, start.Address, start.Token);
|
||||
Log.Info("render-node", $"Offering this PC as render node to {start.Address}");
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("render-node", "The render node could not start", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task StopRenderNodeAsync()
|
||||
{
|
||||
if (_node is not { } running) return;
|
||||
_node = null;
|
||||
await running.Node.DisposeAsync();
|
||||
running.Client.Dispose();
|
||||
Log.Info("render-node", "Render node stopped");
|
||||
}
|
||||
}
|
||||
@@ -189,16 +189,23 @@ public sealed class LeaseFile : IDisposable
|
||||
|
||||
private static bool TryCreate(string path, LeaseRecord record)
|
||||
{
|
||||
try
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read);
|
||||
fs.Write(JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord));
|
||||
fs.Flush(flushToDisk: true);
|
||||
return true;
|
||||
}
|
||||
catch (IOException) when (File.Exists(path))
|
||||
{
|
||||
return false;
|
||||
try
|
||||
{
|
||||
using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read);
|
||||
fs.Write(JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord));
|
||||
fs.Flush(flushToDisk: true);
|
||||
return true;
|
||||
}
|
||||
catch (IOException) when (File.Exists(path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (IOException) when (attempt < 3)
|
||||
{
|
||||
// The holder released the lease between the failed create and the check: the name is free again.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Titano.Core.Project;
|
||||
@@ -181,13 +182,40 @@ public sealed record ProjectSettings
|
||||
|
||||
public string ToJson() => JsonSerializer.Serialize(this, ProjectJson.Default.ProjectSettings);
|
||||
|
||||
/// <summary>
|
||||
/// Reads a project; members missing from <paramref name="json"/> (an API client's partial settings, an older file)
|
||||
/// keep their defaults. The JSON is laid over the default project first, because generated deserialisation leaves
|
||||
/// missing init-only members null or false rather than at their initial values.
|
||||
/// </summary>
|
||||
/// <exception cref="JsonException">The text is not a JSON object.</exception>
|
||||
/// <exception cref="InvalidDataException">The project was written by a newer Titano.</exception>
|
||||
public static ProjectSettings FromJson(string json)
|
||||
{
|
||||
var settings = JsonSerializer.Deserialize(json, ProjectJson.Default.ProjectSettings) ?? new ProjectSettings();
|
||||
var options = new JsonNodeOptions { PropertyNameCaseInsensitive = true };
|
||||
var merged = JsonNode.Parse(new ProjectSettings().ToJson(), options)!.AsObject();
|
||||
switch (JsonNode.Parse(json, options))
|
||||
{
|
||||
case JsonObject given:
|
||||
Overlay(merged, given);
|
||||
break;
|
||||
case not null:
|
||||
throw new JsonException("A project must be a JSON object.");
|
||||
}
|
||||
var settings = JsonSerializer.Deserialize(merged, ProjectJson.Default.ProjectSettings) ?? new ProjectSettings();
|
||||
if (settings.Version > CurrentVersion)
|
||||
throw new InvalidDataException($"project.json version {settings.Version} is newer than this Titano ({CurrentVersion}).");
|
||||
return settings;
|
||||
}
|
||||
|
||||
/// <summary>Copies <paramref name="source"/> into <paramref name="target"/>, merging nested objects; a null never replaces an object.</summary>
|
||||
private static void Overlay(JsonObject target, JsonObject source)
|
||||
{
|
||||
foreach (var (name, value) in source)
|
||||
{
|
||||
if (value is JsonObject child && target[name] is JsonObject existing) Overlay(existing, child);
|
||||
else if (value is not null || target[name] is not JsonObject) target[name] = value?.DeepClone();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
using System.Globalization;
|
||||
using Titano.Core.Analysis.Deep;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Processing.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.LightAnalyse"/> and <see cref="JobKind.DeepAnalyse"/> over the target sequences, one after the
|
||||
/// other. Progress counts sequences (light) or frames (deep); checkpoint: the number of sequences done. A deep pass
|
||||
/// interrupted inside a sequence resumes from that sequence's <c>analysis.partial</c>.
|
||||
/// </summary>
|
||||
internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser analyser, JobKind kind) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => kind;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var deep = kind == JobKind.DeepAnalyse;
|
||||
var targets = context.Job.TargetIds;
|
||||
var first = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var next) ? next : 0;
|
||||
// Deep progress is in frames across all targets, so the bar and the ETA cover the whole job.
|
||||
var frames = new long[targets.Count];
|
||||
if (deep)
|
||||
{
|
||||
for (var i = 0; i < targets.Count; i++)
|
||||
frames[i] = (await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false))?.FrameCount ?? 0;
|
||||
}
|
||||
var total = deep ? frames.Sum() : targets.Count;
|
||||
for (var i = first; i < targets.Count; i++)
|
||||
{
|
||||
context.Cancellation.ThrowIfCancellationRequested();
|
||||
var before = deep ? frames[..i].Sum() : i;
|
||||
var chain = new ProgressChain(context.ReportAsync);
|
||||
var progress = deep ? chain.For<AnalysisProgress>(p => (before + p.Done, total, null)) : null;
|
||||
var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false);
|
||||
await chain.FlushAsync().ConfigureAwait(false);
|
||||
if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i]));
|
||||
else Report(context, outcome, deep);
|
||||
await context.CheckpointAsync((i + 1).ToString(CultureInfo.InvariantCulture), deep ? before + frames[i] : i + 1, total).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static void Report(JobContext context, AnalysisOutcome outcome, bool deep)
|
||||
{
|
||||
var title = outcome.Sequence.DisplayTitle;
|
||||
context.Log(Loc.L(deep ? "Advisor.Log.DeepDone" : "Advisor.Log.LightDone", title, outcome.Frames, outcome.Warnings));
|
||||
if (outcome.SettingsChanged) context.Log(Loc.L("Advisor.Log.AutoUpdated", title));
|
||||
if (outcome.SettingsUnreadable) context.Log(Loc.L("Advisor.Log.ProjectUnreadable", title));
|
||||
if (outcome.Presets.Count > 0) context.Log(Loc.L("Advisor.Log.Presets", string.Join(" · ", outcome.Presets.Select(p => p.DisplayName()))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Core.Render;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Processing.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Render"/> on this machine (BRIEF §11.5): target = the sequence; parameters = Protocol
|
||||
/// <c>RenderJobParameters</c> whose preset JSON is the export section of a project (preset, overrides, name template,
|
||||
/// folder, batch). The project is the embedded one or the sequence's own; frames flagged missing, corrupt, excluded or
|
||||
/// undecodable are left out. The preset and every batch preset render after one preparation pass into the sequence's
|
||||
/// <c>exports/</c> (or the export folder), named by the template, and each finished output is recorded at once.
|
||||
/// Checkpoint: the batch entries recorded and the output being written, so a crashed attempt's leftovers are removed and
|
||||
/// only unfinished entries render again. A failed entry fails the job after the others ran; retrying renders it alone.
|
||||
/// </summary>
|
||||
internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions options) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => JobKind.Render;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var cancellation = context.Cancellation;
|
||||
var (parameters, export) = ProcessingJobs.ReadRender(context.Job.ParametersJson);
|
||||
if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, cancellation).ConfigureAwait(false) is not { } sequence
|
||||
|| await db.Archives.GetAsync(sequence.ArchiveId, cancellation).ConfigureAwait(false) is not { } archive)
|
||||
throw new InvalidOperationException(Loc.L("Errors.Render.Sequence"));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var project = (parameters.SettingsJson is { } embedded ? ProjectStore.Parse(embedded) : ProjectStore.Read(sequence, folder)) with { Export = export };
|
||||
var settingsJson = project.ToJson();
|
||||
|
||||
var resume = RenderCheckpoint.Parse(context.Checkpoint);
|
||||
var done = resume.Done.ToList();
|
||||
if (resume is { Pending: { } pending, PendingOutput: { } output })
|
||||
{
|
||||
// A crash after the render row was written but before the checkpoint: that entry is finished.
|
||||
var recorded = await db.Renders.ListBySequenceAsync(sequence.Id, cancellation).ConfigureAwait(false);
|
||||
if (recorded.Any(r => r.OutputPath == Stored(paths, output))) done.Add(pending);
|
||||
else Discard(output);
|
||||
}
|
||||
var targets = Targets(export, parameters.PresetName).Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).ToList();
|
||||
if (targets.Count == 0) return;
|
||||
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, cancellation).ConfigureAwait(false);
|
||||
IReadOnlyList<TargetOutcome> outcomes;
|
||||
await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames.Count), options))
|
||||
{
|
||||
outcomes = await render.RenderAllAsync(
|
||||
targets,
|
||||
(target, setup) => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup)),
|
||||
(index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total),
|
||||
async (index, target, result) =>
|
||||
{
|
||||
// The output is complete: record it even if the job is being cancelled, so it is never orphaned.
|
||||
var entity = result.ToEntity(sequence.Id, target.PresetName, settingsJson, options.ProducedBy) with { OutputPath = Stored(paths, result.OutputPath) };
|
||||
await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false);
|
||||
done.Add(index);
|
||||
await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath)));
|
||||
foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark);
|
||||
},
|
||||
context.ReportAsync,
|
||||
cancellation).ConfigureAwait(false);
|
||||
}
|
||||
var failed = outcomes.Where(o => o.Error is not null).ToList();
|
||||
foreach (var outcome in failed) context.Log(Loc.L("Export.Log.Failed", outcome.Target.PresetName, outcome.Error));
|
||||
if (failed.Count > 0)
|
||||
throw new InvalidOperationException(failed.Count == 1 ? failed[0].Error : Loc.L("Errors.Render.Failed", failed.Count, outcomes.Count));
|
||||
}
|
||||
|
||||
/// <summary>The job's preset, then every other preset of the batch with its own values.</summary>
|
||||
internal static List<ExportTarget> Targets(ExportSettings export, string presetName)
|
||||
{
|
||||
var main = ExportResolver.Resolve(export);
|
||||
var targets = new List<ExportTarget> { new(main, presetName) };
|
||||
foreach (var id in export.Batch.Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
if (id.Equals(main.BasePresetId, StringComparison.OrdinalIgnoreCase)) continue;
|
||||
var resolved = ExportResolver.Resolve(new ExportSettings { Preset = new Auto<string>().WithManual(id) });
|
||||
targets.Add(new ExportTarget(resolved, ProcessingJobs.PresetName(resolved)));
|
||||
}
|
||||
return targets;
|
||||
}
|
||||
|
||||
/// <summary>Archive-relative for outputs inside the archive (like the server's renders), absolute otherwise.</summary>
|
||||
private static string Stored(ArchivePaths paths, string output)
|
||||
{
|
||||
var relative = paths.ToRelative(output);
|
||||
return relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative) ? output : relative;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes what an interrupted attempt left of <paramref name="output"/>: the encoder's hidden temporary file or
|
||||
/// folder, and an output that was completed but never recorded (it renders again under the same name).
|
||||
/// </summary>
|
||||
private static void Discard(string output)
|
||||
{
|
||||
var parent = Path.GetDirectoryName(output);
|
||||
if (parent is null || !Directory.Exists(parent)) return;
|
||||
foreach (var leftover in Directory.EnumerateFileSystemEntries(parent, "." + Path.GetFileName(output) + ".*.tmp").Append(output))
|
||||
{
|
||||
if (Directory.Exists(leftover)) Directory.Delete(leftover, recursive: true);
|
||||
else if (File.Exists(leftover)) File.Delete(leftover);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.Json.Serialization;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Core.Video;
|
||||
using Titano.Data.Archiving.Jobs;
|
||||
using Titano.Protocol;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// Builds the analysis and render jobs; enqueue them with <see cref="Archiving.ArchiveJobs.EnqueueAsync"/> (local
|
||||
/// engine) or post the same kind, targets and parameters to <c>/api/v1/jobs</c> (server).
|
||||
/// </summary>
|
||||
public static class ProcessingJobs
|
||||
{
|
||||
/// <summary><see cref="JobKind.LightAnalyse"/>: targets = sequences; no parameters.</summary>
|
||||
public static ArchiveJobRequest LightAnalysis(IReadOnlyList<Guid> sequenceIds, string? label = null) =>
|
||||
new(JobKind.LightAnalyse, sequenceIds, label, null, JobPriority.Background);
|
||||
|
||||
/// <summary><see cref="JobKind.DeepAnalyse"/>: targets = sequences; no parameters.</summary>
|
||||
public static ArchiveJobRequest DeepAnalysis(IReadOnlyList<Guid> sequenceIds, string? label = null) =>
|
||||
new(JobKind.DeepAnalyse, sequenceIds, label, null, JobPriority.User);
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Render"/> of one sequence. <paramref name="export"/> (preset, overrides, name template,
|
||||
/// folder, batch) replaces the project's export section; <paramref name="project"/> embeds the settings to render,
|
||||
/// null renders the sequence's current project. <paramref name="sourceWidth"/>×<paramref name="sourceHeight"/>
|
||||
/// stands in for a preset without a fixed size, so render nodes can match the job by resolution.
|
||||
/// </summary>
|
||||
public static ArchiveJobRequest Render(Sequence sequence, ExportSettings export, int sourceWidth, int sourceHeight, ProjectSettings? project = null,
|
||||
bool useProxies = false)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sequence);
|
||||
ArgumentNullException.ThrowIfNull(export);
|
||||
var resolved = ExportResolver.Resolve(export);
|
||||
var name = PresetName(resolved);
|
||||
var parameters = new RenderJobParameters(name, JsonSerializer.Serialize(export, ProcessingJson.Default.ExportSettings), CodecName(resolved.Codec),
|
||||
resolved.Width > 0 ? resolved.Width : sourceWidth, resolved.Height > 0 ? resolved.Height : sourceHeight, useProxies, project?.ToJson());
|
||||
return new ArchiveJobRequest(JobKind.Render, [sequence.Id], $"{sequence.DisplayTitle} → {name}",
|
||||
JsonSerializer.Serialize(parameters, ProtocolJson.Default.RenderJobParameters), JobPriority.User);
|
||||
}
|
||||
|
||||
/// <summary>Codec name in job parameters and render-node capabilities ("h264", "hevc", "mjpeg", "pngsequence"…).</summary>
|
||||
public static string CodecName(VideoCodec codec) => codec.ToString().ToLowerInvariant();
|
||||
|
||||
/// <summary>Display name of an export's preset in the current language ("Custom" when the values match none).</summary>
|
||||
public static string PresetName(ResolvedExport export)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(export);
|
||||
return Loc.L(export.NameKey ?? "Export.Preset.Custom");
|
||||
}
|
||||
|
||||
/// <summary>The parameters of a render job and the export they select.</summary>
|
||||
/// <exception cref="InvalidDataException">The job has no valid render parameters.</exception>
|
||||
internal static (RenderJobParameters Parameters, ExportSettings Export) ReadRender(string? parametersJson)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (parametersJson is { Length: > 0 } && JsonSerializer.Deserialize(parametersJson, ProtocolJson.Default.RenderJobParameters) is { } parameters)
|
||||
return (parameters, ReadExport(parameters.PresetJson));
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException(Loc.L("Errors.Render.Parameters"), ex);
|
||||
}
|
||||
throw new InvalidDataException(Loc.L("Errors.Render.Parameters"));
|
||||
}
|
||||
|
||||
/// <summary>The export section in a render job's preset JSON; values it leaves out take their defaults.</summary>
|
||||
/// <exception cref="InvalidDataException">The JSON is not an export section.</exception>
|
||||
internal static ExportSettings ReadExport(string? presetJson)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Read as the export section of a project, so missing values keep their defaults (ProjectSettings.FromJson).
|
||||
var project = new JsonObject { ["export"] = JsonNode.Parse(presetJson ?? string.Empty) };
|
||||
return ProjectSettings.FromJson(project.ToJsonString()).Export;
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException(Loc.L("Errors.Render.Parameters"), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resume point of a render job: the batch targets already recorded, and the output a crashed attempt was writing
|
||||
/// (its leftovers are removed before that target renders again).
|
||||
/// </summary>
|
||||
internal sealed record RenderCheckpoint(IReadOnlyList<int> Done, int? Pending, string? PendingOutput)
|
||||
{
|
||||
public static RenderCheckpoint Empty { get; } = new([], null, null);
|
||||
|
||||
public static RenderCheckpoint Parse(string? json)
|
||||
{
|
||||
if (string.IsNullOrEmpty(json)) return Empty;
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(json, ProcessingJson.Default.RenderCheckpoint) ?? Empty;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
}
|
||||
|
||||
public string ToJson() => JsonSerializer.Serialize(this, ProcessingJson.Default.RenderCheckpoint);
|
||||
}
|
||||
|
||||
/// <summary>JSON written into render jobs: presets (the export section of <c>project.json</c>) and checkpoints.</summary>
|
||||
[JsonSourceGenerationOptions(
|
||||
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
UseStringEnumConverter = true,
|
||||
PropertyNameCaseInsensitive = true)]
|
||||
[JsonSerializable(typeof(ExportSettings))]
|
||||
[JsonSerializable(typeof(RenderCheckpoint))]
|
||||
internal sealed partial class ProcessingJson : JsonSerializerContext;
|
||||
@@ -0,0 +1,43 @@
|
||||
using Titano.Core.Analysis.Deep;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Render;
|
||||
using Titano.Core.Video;
|
||||
using Titano.Core.Video.Managed;
|
||||
using Titano.Data.Archiving;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// What differs between the hosts that analyse and render (BRIEF §4.2): decoders (inside <see cref="Archive"/>),
|
||||
/// encoders and where the work runs. Client: WIC then managed decoders, Media Foundation then managed encoders;
|
||||
/// server: managed only.
|
||||
/// </summary>
|
||||
public sealed record ProcessingOptions
|
||||
{
|
||||
/// <summary>Decoders, lease owner and lease timing for the sidecar and project writes.</summary>
|
||||
public ArchiveServiceOptions Archive { get; init; } = new();
|
||||
|
||||
/// <summary>Encoder providers in priority order.</summary>
|
||||
public IReadOnlyList<IVideoEncoderProvider> Encoders { get; init; } = [new ManagedEncoderProvider()];
|
||||
|
||||
/// <summary>The server delegates platform codecs to render nodes and never runs them itself.</summary>
|
||||
public EncoderHost Host { get; init; } = EncoderHost.Client;
|
||||
|
||||
/// <summary>Recorded with every render this host produces (client or server name).</summary>
|
||||
public string ProducedBy { get; init; } = Environment.MachineName;
|
||||
|
||||
public DeepAnalysisOptions DeepAnalysis { get; init; } = DeepAnalysisOptions.Default;
|
||||
|
||||
/// <summary>Read-ahead and memory budget of the render decode window.</summary>
|
||||
public FrameWindowOptions FrameWindow { get; init; } = new();
|
||||
|
||||
public RenderOptions Render { get; init; } = RenderOptions.Default;
|
||||
|
||||
/// <summary>
|
||||
/// Encoders the advisor may count on: this machine's on a client; none listed on the server (render nodes can
|
||||
/// produce any codec, so the advisor rules none out).
|
||||
/// </summary>
|
||||
internal IReadOnlyList<EncoderCapability> AdvisorCodecs =>
|
||||
Host == EncoderHost.Server ? [] : [.. Encoders.SelectMany(e => e.Capabilities)];
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Jobs;
|
||||
using Titano.Data.Processing.Jobs;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// The analysis and render engine of a host (BRIEF §4.3, §10, §11): job handlers for light analysis, deep analysis and
|
||||
/// renders over the archive in the index, built with the host's decoders and encoders (<see cref="ProcessingOptions"/>).
|
||||
/// Register it on the <see cref="JobEngine"/> next to the archive engine; a server engine leaves render jobs to render
|
||||
/// nodes. Create jobs with <see cref="ProcessingJobs"/>.
|
||||
/// </summary>
|
||||
public sealed class ProcessingService : IJobHandlerProvider
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceAnalyser _analyser;
|
||||
|
||||
public ProcessingService(TitanoDatabase db, ProcessingOptions? options = null)
|
||||
{
|
||||
_db = db;
|
||||
Options = options ?? new ProcessingOptions();
|
||||
// Sidecar and project writes coordinate with the archive engine through the archive's lease files.
|
||||
var writer = new SequenceWriter(Options.Archive);
|
||||
_analyser = new SequenceAnalyser(db, Options, writer, new ArchiveIndexer(db, writer), new ProjectStore(db, writer));
|
||||
}
|
||||
|
||||
public ProcessingOptions Options { get; }
|
||||
|
||||
/// <summary>Handlers for <see cref="JobKind.LightAnalyse"/>, <see cref="JobKind.DeepAnalyse"/> and <see cref="JobKind.Render"/>.</summary>
|
||||
public IReadOnlyList<IJobHandler> CreateJobHandlers() =>
|
||||
[
|
||||
new AnalyseJobHandler(_db, _analyser, JobKind.LightAnalyse),
|
||||
new AnalyseJobHandler(_db, _analyser, JobKind.DeepAnalyse),
|
||||
new RenderJobHandler(_db, Options),
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// Delivers the synchronous progress reports of the engine (analysis, render stages) to an asynchronous sink (job
|
||||
/// context, render-node protocol) one after the other, in report order. Await <see cref="FlushAsync"/> before a
|
||||
/// checkpoint or the end of the work to observe the deliveries.
|
||||
/// </summary>
|
||||
internal sealed class ProgressChain(Func<long, long, string?, ValueTask> sink)
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
private Task _pending = Task.CompletedTask;
|
||||
|
||||
/// <summary>A progress receiver mapping its reports to done, total and message.</summary>
|
||||
public IProgress<T> For<T>(Func<T, (long Done, long Total, string? Message)> map) => new Forward<T>(this, map);
|
||||
|
||||
public Task FlushAsync()
|
||||
{
|
||||
lock (_gate) return _pending;
|
||||
}
|
||||
|
||||
private void Post(long done, long total, string? message)
|
||||
{
|
||||
lock (_gate) _pending = AfterAsync(_pending, done, total, message);
|
||||
}
|
||||
|
||||
private async Task AfterAsync(Task previous, long done, long total, string? message)
|
||||
{
|
||||
await previous.ConfigureAwait(false);
|
||||
await sink(done, total, message).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private sealed class Forward<T>(ProgressChain chain, Func<T, (long Done, long Total, string? Message)> map) : IProgress<T>
|
||||
{
|
||||
public void Report(T value)
|
||||
{
|
||||
var (done, total, message) = map(value);
|
||||
chain.Post(done, total, message);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// The project of a sequence (BRIEF §7 <c>project.json</c>), mirrored in the index's settings column that the
|
||||
/// server's API edits. The index copy is read first; writes go to both, under the sequence lock and with the index's
|
||||
/// settings version as an optimistic check, so a concurrent edit is applied on top of, never overwritten.
|
||||
/// </summary>
|
||||
internal sealed class ProjectStore(TitanoDatabase db, SequenceWriter writer)
|
||||
{
|
||||
private const int Attempts = 3;
|
||||
|
||||
/// <summary>The settings in force: the index copy, else <c>project.json</c>, else defaults.</summary>
|
||||
/// <exception cref="InvalidDataException">The settings are not valid JSON or come from a newer Titano.</exception>
|
||||
public static ProjectSettings Read(Sequence sequence, SequencePaths folder) =>
|
||||
Parse(sequence.SettingsJson ?? (File.Exists(folder.ProjectPath) ? File.ReadAllText(folder.ProjectPath) : null));
|
||||
|
||||
/// <exception cref="InvalidDataException">The settings are not valid JSON or come from a newer Titano.</exception>
|
||||
public static ProjectSettings Parse(string? json)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(json)) return new ProjectSettings();
|
||||
try
|
||||
{
|
||||
return ProjectSettings.FromJson(json);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException(ex.Message, ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Applies <paramref name="edit"/> to the settings in force and stores the result when it differs; false when nothing changed.</summary>
|
||||
/// <exception cref="InvalidDataException">The stored settings cannot be read; nothing is written.</exception>
|
||||
public async Task<bool> UpdateAsync(ArchivePaths paths, Sequence sequence, Func<ProjectSettings, ProjectSettings> edit, CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence;
|
||||
var current = Read(row, folder);
|
||||
var before = current.ToJson();
|
||||
var after = edit(current).ToJson();
|
||||
if (after == before) return false;
|
||||
AtomicFile.WriteAllText(folder.ProjectPath, after);
|
||||
var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after }, row.SettingsVersion, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
// A conflict means the API changed the settings meanwhile: apply the edit again on top of them.
|
||||
if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,455 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Render;
|
||||
using Titano.Core.Video;
|
||||
using Titano.Protocol;
|
||||
using Titano.Protocol.Client;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>Settings of a <see cref="RenderNodeClient"/>.</summary>
|
||||
public sealed record RenderNodeOptions
|
||||
{
|
||||
/// <summary>Name the server lists the node under; one node per user and name.</summary>
|
||||
public required string ClientName { get; init; }
|
||||
|
||||
/// <summary>Folder for downloaded originals and for outputs until they are uploaded (inside the client's cache).</summary>
|
||||
public required string CacheDirectory { get; init; }
|
||||
|
||||
/// <summary>Downloaded originals beyond this size are deleted, least recently used first, after every job.</summary>
|
||||
public long CacheLimitBytes { get; init; } = 20L << 30;
|
||||
|
||||
/// <summary>Wait between claims while the queue has nothing for this node.</summary>
|
||||
public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>Sign of life; the server gives the jobs of a node silent for 90 s to others.</summary>
|
||||
public TimeSpan HeartbeatInterval { get; init; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>Shortest interval between progress reports to the server.</summary>
|
||||
public TimeSpan ProgressInterval { get; init; } = TimeSpan.FromSeconds(2);
|
||||
|
||||
/// <summary>Concurrent frame downloads.</summary>
|
||||
public int Downloads { get; init; } = 4;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// "Offer this PC as render node" (BRIEF §14.7): registers this machine's single-file encoders with the server,
|
||||
/// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals
|
||||
/// hash-verified into a content-addressed cache, or proxies when the job asks), renders the job's preset with the
|
||||
/// host's encoders and uploads the output, which the server files as a render of the sequence. Progress goes to the
|
||||
/// server; a job cancelled or reassigned there stops here; a failure is reported with its reason. Stopping abandons
|
||||
/// the job in flight without reporting it, so the server requeues it once the heartbeats stop. Image sequences are
|
||||
/// not offered: the protocol uploads one file per job.
|
||||
/// </summary>
|
||||
public sealed class RenderNodeClient : IAsyncDisposable
|
||||
{
|
||||
private const string LogCategory = "render-node";
|
||||
|
||||
private readonly TitanoClient _client;
|
||||
private readonly ProcessingOptions _processing;
|
||||
private readonly RenderNodeOptions _options;
|
||||
private readonly CancellationTokenSource _stopping = new();
|
||||
private Task _running = Task.CompletedTask;
|
||||
private Guid? _nodeId;
|
||||
|
||||
public RenderNodeClient(TitanoClient client, ProcessingOptions processing, RenderNodeOptions options)
|
||||
{
|
||||
_client = client;
|
||||
_processing = processing;
|
||||
_options = options;
|
||||
}
|
||||
|
||||
/// <summary>The encoders this node offers: every single-file capability of the host's providers.</summary>
|
||||
public IReadOnlyList<NodeEncoder> Encoders =>
|
||||
[
|
||||
.. _processing.Encoders.SelectMany(p => p.Capabilities)
|
||||
.Where(c => VideoCodecInfo.DefaultContainer(c.Codec) != ContainerFormat.ImageSequence)
|
||||
.Select(c => new NodeEncoder(ProcessingJobs.CodecName(c.Codec), c.Name, c.Hardware, c.SupportsTenBit)),
|
||||
];
|
||||
|
||||
/// <summary>Starts registering, heart-beating and claiming in the background.</summary>
|
||||
public void Start() => _running = Task.Run(() => RunAsync(_stopping.Token));
|
||||
|
||||
/// <summary>Stops claiming, abandons the job in flight and waits for the background work to end.</summary>
|
||||
public async Task StopAsync()
|
||||
{
|
||||
await _stopping.CancelAsync().ConfigureAwait(false);
|
||||
await _running.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await StopAsync().ConfigureAwait(false);
|
||||
_stopping.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>Registers this node (again) and returns its id.</summary>
|
||||
public async Task<Guid> RegisterAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var capabilities = _processing.Encoders.SelectMany(p => p.Capabilities).ToList();
|
||||
var request = new RegisterNodeRequest(_options.ClientName, Encoders, capabilities.Count > 0 ? capabilities.Max(c => c.MaxWidth) : 0,
|
||||
capabilities.Count > 0 ? capabilities.Max(c => c.MaxHeight) : 0);
|
||||
var node = await _client.RegisterNodeAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
_nodeId = node.Id;
|
||||
return node.Id;
|
||||
}
|
||||
|
||||
/// <summary>Claims one job this node can encode and runs it to the end; false when the queue had none.</summary>
|
||||
public async Task<bool> RunOnceAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var nodeId = _nodeId ?? await RegisterAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (await _client.ClaimAsync(nodeId, cancellationToken).ConfigureAwait(false) is not { } claimed) return false;
|
||||
var work = Path.Combine(_options.CacheDirectory, "jobs", claimed.Job.Id.ToString("N"));
|
||||
using var job = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
var reporter = new NodeReporter(_client, nodeId, claimed.Job.Id, _options.ProgressInterval, job);
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(work);
|
||||
await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (job.IsCancellationRequested)
|
||||
{
|
||||
// Stopping, or the server cancelled or reassigned the job: either way it is no longer ours to report.
|
||||
Log.Info(LogCategory, $"Render job {claimed.Job.Id} abandoned: {ex.Message}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Whatever went wrong belongs to this job: report it and stay available for the next one.
|
||||
Log.Warning(LogCategory, $"Render job {claimed.Job.Id} failed", ex);
|
||||
await ReportFailureAsync(nodeId, claimed.Job.Id, SequenceRender.Reason(ex), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(work);
|
||||
TrimCache();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task RunAsync(CancellationToken stopping)
|
||||
{
|
||||
using var heartbeat = CancellationTokenSource.CreateLinkedTokenSource(stopping);
|
||||
Task? beating = null;
|
||||
while (!stopping.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var nodeId = _nodeId ?? await RegisterAsync(stopping).ConfigureAwait(false);
|
||||
beating ??= HeartbeatAsync(heartbeat.Token);
|
||||
if (await RunOnceAsync(stopping).ConfigureAwait(false)) continue;
|
||||
}
|
||||
catch (OperationCanceledException) when (stopping.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden)
|
||||
{
|
||||
// Signed out, or the account lacks the render-node right: nothing to retry until the user acts.
|
||||
Log.Warning(LogCategory, "The server refused this render node", ex);
|
||||
break;
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
_nodeId = null; // removed on the server: register again
|
||||
continue;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Server unreachable or busy: the node keeps trying until it is stopped.
|
||||
Log.Warning(LogCategory, "Render node cycle failed; retrying", ex);
|
||||
}
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.PollInterval, stopping).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
await heartbeat.CancelAsync().ConfigureAwait(false);
|
||||
if (beating is not null) await beating.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task HeartbeatAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
using var timer = new PeriodicTimer(_options.HeartbeatInterval);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (_nodeId is not { } nodeId) continue;
|
||||
try
|
||||
{
|
||||
await _client.HeartbeatAsync(nodeId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
_nodeId = null;
|
||||
}
|
||||
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException or UntrustedCertificateException)
|
||||
{
|
||||
Log.Warning(LogCategory, "Heartbeat failed", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
|
||||
{
|
||||
var export = ProcessingJobs.ReadExport(claimed.PresetJson);
|
||||
var resolved = ExportResolver.Resolve(export);
|
||||
var target = new ExportTarget(resolved, ProcessingJobs.PresetName(resolved));
|
||||
if (target.Export.Container == ContainerFormat.ImageSequence) throw new InvalidOperationException(Loc.L("Errors.Render.NodeSequence"));
|
||||
var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export };
|
||||
|
||||
var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false));
|
||||
var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
var files = await DownloadAsync(claimed, work, reporter, cancellationToken).ConfigureAwait(false);
|
||||
var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id))]);
|
||||
|
||||
IReadOnlyList<TargetOutcome> outcomes;
|
||||
await using (var render = new SequenceRender(input, project, null, _processing))
|
||||
{
|
||||
outcomes = await render.RenderAllAsync(
|
||||
[(0, target)],
|
||||
(t, setup) => ExportNaming.OutputPath(work, project.Export.NameTemplate, SequenceRender.NameContext(sequence, t, setup)),
|
||||
(_, _) => ValueTask.CompletedTask,
|
||||
(_, t, result) => new ValueTask(UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken)),
|
||||
reporter.ReportAsync,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (outcomes[0].Error is { } error) throw new InvalidOperationException(error);
|
||||
}
|
||||
|
||||
private async Task UploadAsync(Guid nodeId, Guid jobId, ExportTarget target, RenderResult result, CancellationToken cancellationToken)
|
||||
{
|
||||
var size = new FileInfo(result.OutputPath).Length;
|
||||
var sha = await TitanoClient.HashFileAsync(result.OutputPath, cancellationToken).ConfigureAwait(false);
|
||||
var request = new RenderResultRequest(Path.GetFileName(result.OutputPath), size, sha, target.PresetName, ProcessingJobs.CodecName(result.Codec),
|
||||
result.Width, result.Height, result.FrameRate.Value, result.DurationSeconds);
|
||||
await _client.UploadRenderResultAsync(nodeId, jobId, result.OutputPath, request, null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<List<Frame>> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
var frames = new List<Frame>();
|
||||
string? cursor = null;
|
||||
do
|
||||
{
|
||||
var page = await _client.ListFramesAsync(sequenceId, cursor, 1000, cancellationToken).ConfigureAwait(false);
|
||||
frames.AddRange(page.Items.Select(ToFrame));
|
||||
cursor = page.NextCursor;
|
||||
}
|
||||
while (cursor is not null);
|
||||
frames.Sort((a, b) => a.Index.CompareTo(b.Index));
|
||||
return frames;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused
|
||||
/// across jobs), or proxies into the job folder when the job asks for them and the server has them.
|
||||
/// </summary>
|
||||
private async Task<Dictionary<Guid, string>> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
|
||||
{
|
||||
var files = new Dictionary<Guid, string>();
|
||||
var done = 0;
|
||||
await reporter.ReportAsync(0, claimed.Frames.Count, Loc.L("Export.Job.Downloading")).ConfigureAwait(false);
|
||||
await Parallel.ForEachAsync(claimed.Frames, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken },
|
||||
async (frame, ct) =>
|
||||
{
|
||||
var path = claimed.UseProxies && frame.HasProxy
|
||||
? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false)
|
||||
: await DownloadOriginalAsync(frame, ct).ConfigureAwait(false);
|
||||
lock (files) files[frame.Id] = path;
|
||||
await reporter.ReportAsync(Interlocked.Increment(ref done), claimed.Frames.Count, null).ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
return files;
|
||||
}
|
||||
|
||||
/// <summary>A proxy into the job folder (regenerable, so not cached); the original when the server has none for the frame.</summary>
|
||||
private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||
{
|
||||
var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
try
|
||||
{
|
||||
await using (var file = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
|
||||
await _client.DownloadFrameAsync(frame.Id, "proxy", file, null, cancellationToken).ConfigureAwait(false);
|
||||
return path;
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
File.Delete(path);
|
||||
return await DownloadOriginalAsync(frame, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256.</summary>
|
||||
private async Task<string> DownloadOriginalAsync(ManifestFrame frame, CancellationToken cancellationToken)
|
||||
{
|
||||
var sha = frame.Sha256.ToLowerInvariant();
|
||||
if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
|
||||
var path = Path.Combine(_options.CacheDirectory, "originals", sha[..2], sha + Path.GetExtension(frame.FileName).ToLowerInvariant());
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.SetLastWriteTimeUtc(path, DateTime.UtcNow); // most recently used: kept by the cache trim
|
||||
return path;
|
||||
}
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
var part = path + ".part";
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
|
||||
await _client.DownloadFrameAsync(frame.Id, "original", file, null, cancellationToken).ConfigureAwait(false);
|
||||
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
|
||||
{
|
||||
File.Move(part, path, overwrite: true);
|
||||
return path;
|
||||
}
|
||||
File.Delete(part);
|
||||
if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReportFailureAsync(Guid nodeId, Guid jobId, string error, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _client.ReportNodeFailureAsync(nodeId, jobId, error, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException or OperationCanceledException)
|
||||
{
|
||||
// Cancelled or reassigned meanwhile, or the server is gone: it requeues the job when the heartbeats stop.
|
||||
Log.Warning(LogCategory, $"The failure of job {jobId} could not be reported", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Deletes the least recently used originals until the cache fits its limit.</summary>
|
||||
private void TrimCache()
|
||||
{
|
||||
try
|
||||
{
|
||||
var root = new DirectoryInfo(Path.Combine(_options.CacheDirectory, "originals"));
|
||||
if (!root.Exists) return;
|
||||
var files = root.EnumerateFiles("*", SearchOption.AllDirectories).OrderBy(f => f.LastWriteTimeUtc).ToList();
|
||||
var size = files.Sum(f => f.Length);
|
||||
foreach (var file in files)
|
||||
{
|
||||
if (size <= _options.CacheLimitBytes) break;
|
||||
size -= file.Length;
|
||||
file.Delete();
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning(LogCategory, "The render node cache could not be trimmed", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string folder)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(folder)) Directory.Delete(folder, recursive: true);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning(LogCategory, $"Could not remove {folder}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static Sequence ToSequence(SequenceDto s) => new()
|
||||
{
|
||||
Id = s.Id,
|
||||
ArchiveId = Guid.Empty,
|
||||
Folder = s.Folder,
|
||||
Title = s.Title,
|
||||
CameraMake = s.CameraMake,
|
||||
CameraModel = s.CameraModel,
|
||||
Lens = s.Lens,
|
||||
FirstCapture = s.FirstCapture,
|
||||
LastCapture = s.LastCapture,
|
||||
FrameCount = s.FrameCount,
|
||||
};
|
||||
|
||||
private static Frame ToFrame(FrameDto f) => new()
|
||||
{
|
||||
Id = f.Id,
|
||||
SequenceId = f.SequenceId,
|
||||
Index = f.Index,
|
||||
RelativePath = f.FileName,
|
||||
FileName = f.FileName,
|
||||
Size = f.Size,
|
||||
Sha256 = f.Sha256,
|
||||
CapturedAt = f.CapturedAt,
|
||||
UtcOffsetMinutes = f.UtcOffsetMinutes,
|
||||
ExposureTime = f.ExposureTime,
|
||||
FNumber = f.FNumber,
|
||||
Iso = f.Iso,
|
||||
FocalLength = f.FocalLength,
|
||||
Orientation = f.Orientation,
|
||||
Width = f.Width,
|
||||
Height = f.Height,
|
||||
Format = Enum.TryParse<ImageFileFormat>(f.Format, ignoreCase: true, out var format) ? format : default,
|
||||
Flags = (FrameFlags)f.Flags,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Progress of the job in flight to the server, at most every interval (and at the end). A refusal (409) means the
|
||||
/// job was cancelled or given to another node: <see cref="Lost"/> is set and the job's token cancelled.
|
||||
/// </summary>
|
||||
private sealed class NodeReporter(TitanoClient client, Guid nodeId, Guid jobId, TimeSpan interval, CancellationTokenSource job)
|
||||
{
|
||||
private readonly Stopwatch _clock = Stopwatch.StartNew();
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
private TimeSpan _last = TimeSpan.MinValue;
|
||||
private long _lastDone;
|
||||
private string? _lastMessage;
|
||||
|
||||
public Guid NodeId => nodeId;
|
||||
|
||||
public bool Lost { get; private set; }
|
||||
|
||||
public async ValueTask ReportAsync(long done, long total, string? message)
|
||||
{
|
||||
await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var now = _clock.Elapsed;
|
||||
var line = message is not null && message != _lastMessage ? message : null;
|
||||
if (line is null && done < total && now - _last < interval) return;
|
||||
double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3);
|
||||
(_last, _lastDone) = (now, done);
|
||||
if (line is not null) _lastMessage = line;
|
||||
await client.ReportNodeProgressAsync(nodeId, jobId, new NodeProgress(done, total, rate, line), job.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict)
|
||||
{
|
||||
Lost = true;
|
||||
await job.CancelAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException)
|
||||
{
|
||||
Log.Warning(LogCategory, "Progress could not be reported", ex);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// The job is stopping; its outcome is handled by the caller.
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gate.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Titano.Core.Advice;
|
||||
using Titano.Core.Analysis;
|
||||
using Titano.Core.Analysis.Deep;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>What an analysis of one sequence found and changed.</summary>
|
||||
/// <param name="Warnings">Findings of warning severity or worse (light and, when current, deep).</param>
|
||||
/// <param name="SettingsChanged">The advisor changed Auto values of the project.</param>
|
||||
/// <param name="SettingsUnreadable">The project could not be read, so its Auto values were left alone.</param>
|
||||
/// <param name="Presets">Sequence presets that fit the content (deep analysis only).</param>
|
||||
internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable,
|
||||
IReadOnlyList<SequencePreset> Presets);
|
||||
|
||||
/// <summary>
|
||||
/// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the
|
||||
/// Auto values of the project. The light pass reads the index only (plus one file for the GPS fix); the deep pass decodes
|
||||
/// every frame, checkpointing into <c>.titano/analysis.partial</c> so a cancelled, paused or crashed run resumes, and
|
||||
/// saves <c>analysis.json</c> atomically. Results land in the sidecar status, the index (status, warning count, compact
|
||||
/// summary, per-frame light statistics) and <c>project.json</c>.
|
||||
/// </summary>
|
||||
internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions options, SequenceWriter writer, ArchiveIndexer indexer, ProjectStore projects)
|
||||
{
|
||||
/// <summary>Analyses one sequence; null when it is no longer in the index.</summary>
|
||||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
|
||||
public async Task<AnalysisOutcome?> AnalyseAsync(Guid sequenceId, bool deep, IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
|
||||
{
|
||||
if (await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } sequence
|
||||
|| await db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive) return null;
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, cancellationToken).ConfigureAwait(false);
|
||||
var metadata = frames.LightFrames();
|
||||
var light = LightAnalyzer.Analyze(metadata);
|
||||
|
||||
var report = deep
|
||||
? await AnalyseDeepAsync(frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false)
|
||||
: ReadReport(folder, frames.Count);
|
||||
var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs);
|
||||
var advice = Advisor.Recommend(light, report, context);
|
||||
var warnings = advice.Findings.Count(f => f.Severity >= FindingSeverity.Warning);
|
||||
|
||||
bool changed, unreadable = false;
|
||||
try
|
||||
{
|
||||
changed = await projects.UpdateAsync(paths, sequence, settings => Advisor.Apply(settings, advice), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (InvalidDataException ex)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex);
|
||||
(changed, unreadable) = (false, true);
|
||||
}
|
||||
|
||||
var set = SequenceStatus.AnalyzedLight | (report is null ? SequenceStatus.None : SequenceStatus.AnalyzedDeep)
|
||||
| (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None);
|
||||
// Without a current report (none, or made before frames were added) the deep status no longer holds.
|
||||
var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings);
|
||||
var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null;
|
||||
await writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
|
||||
{
|
||||
var status = (m.Status | set) & ~clear;
|
||||
return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false);
|
||||
await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var presets = deep ? SequencePresets.Generate(light, report, context) : [];
|
||||
return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets);
|
||||
}
|
||||
|
||||
/// <summary>The saved deep report when it describes the current frames; null when absent, stale or unreadable.</summary>
|
||||
public static DeepAnalysisReport? ReadReport(SequencePaths folder, int frameCount)
|
||||
{
|
||||
if (!File.Exists(folder.AnalysisPath)) return null;
|
||||
try
|
||||
{
|
||||
var report = DeepAnalysisReport.Load(folder.AnalysisPath);
|
||||
return report.FrameCount == frameCount ? report : null;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or System.Text.Json.JsonException)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("processing", $"{folder.AnalysisPath} cannot be read", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<DeepAnalysisReport> AnalyseDeepAsync(SequenceFrames frames, IReadOnlyList<LightFrame> metadata, SequencePaths folder,
|
||||
IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
|
||||
{
|
||||
var file = new DeepAnalysisCheckpointFile(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
|
||||
DeepAnalysisReport report;
|
||||
await using (var source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder))
|
||||
{
|
||||
report = await DeepAnalyzer.AnalyzeAsync(source, options.DeepAnalysis, new FramesCheckpoint(file, ContentKey(frames)), progress, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
report.Save(folder.AnalysisPath);
|
||||
file.Delete();
|
||||
|
||||
var columns = report.Frames;
|
||||
var stats = new FrameLightStats[frames.Count];
|
||||
for (var i = 0; i < stats.Length; i++)
|
||||
stats[i] = new FrameLightStats(frames.Frames[i].Id, columns.LogLuminance[i], columns.ClipLow[i], columns.ClipHigh[i]);
|
||||
await db.Frames.UpdateLightStatsAsync(stats, cancellationToken).ConfigureAwait(false);
|
||||
return report;
|
||||
}
|
||||
|
||||
/// <summary>Identity of the analysed content: frames replaced or renumbered since a checkpoint was written (with the
|
||||
/// same count) make it stale, like changed options do.</summary>
|
||||
private static string ContentKey(SequenceFrames frames)
|
||||
{
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
foreach (var f in frames.Frames) hash.AppendData(Encoding.UTF8.GetBytes($"{f.Id:N}/{f.Sha256}/{f.Size}/{f.Flags & SequenceFrames.NotAnalysed};"));
|
||||
return Convert.ToHexStringLower(hash.GetHashAndReset())[..16];
|
||||
}
|
||||
|
||||
private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey) : IDeepAnalysisCheckpoint
|
||||
{
|
||||
public ValueTask<IReadOnlyList<DeepFrame>> LoadAsync(string fingerprint, CancellationToken cancellationToken) =>
|
||||
file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken);
|
||||
|
||||
public ValueTask AppendAsync(IReadOnlyList<DeepFrame> frames, CancellationToken cancellationToken) => file.AppendAsync(frames, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
using System.Globalization;
|
||||
using Titano.Core.Advice;
|
||||
using Titano.Core.Analysis;
|
||||
using Titano.Core.Analysis.Deep;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Render;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// The frames of a <see cref="SequenceFrames"/> decoded by the host's <see cref="ImageDecoder"/> (oriented, linear
|
||||
/// light, embedded previews when nothing else decodes). For the deep analysis a frame that is missing, corrupt or
|
||||
/// unreadable is null, as <see cref="IFrameSource"/> expects. Renders exclude flagged frames from the plan, so a
|
||||
/// frame that still cannot be decoded stops the render with <see cref="FrameUnreadableKey"/>; render reads go through
|
||||
/// a <see cref="FrameWindow"/> per decode size, which decodes ahead in index order and spills under memory pressure.
|
||||
/// </summary>
|
||||
internal sealed class SequenceFrameSource : IFrameSource, IRenderFrameSource, IAsyncDisposable
|
||||
{
|
||||
/// <summary>Render failure reason: {0} frame number (from 1), {1} file name.</summary>
|
||||
public const string FrameUnreadableKey = "Export.Render.FrameUnreadable";
|
||||
|
||||
private const int ProbeLongEdge = 64;
|
||||
|
||||
private readonly SequenceFrames _frames;
|
||||
private readonly IReadOnlyList<LightFrame> _metadata;
|
||||
private readonly ImageDecoder _decoder;
|
||||
private readonly FrameWindowOptions _windowOptions;
|
||||
private readonly Lazy<(int Width, int Height)> _size;
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
private FrameWindow? _window;
|
||||
private (int Edge, DecodePurpose Purpose) _windowKey;
|
||||
|
||||
public SequenceFrameSource(SequenceFrames frames, IReadOnlyList<LightFrame> metadata, ImageDecoder decoder, FrameWindowOptions? window = null)
|
||||
{
|
||||
if (metadata.Count != frames.Count) throw new ArgumentException("One metadata entry per frame is required.", nameof(metadata));
|
||||
_frames = frames;
|
||||
_metadata = metadata;
|
||||
_decoder = decoder;
|
||||
_windowOptions = window ?? new FrameWindowOptions();
|
||||
_size = new Lazy<(int, int)>(MeasureSize);
|
||||
}
|
||||
|
||||
public int Count => _frames.Count;
|
||||
|
||||
public int FrameCount => _frames.Count;
|
||||
|
||||
public int Width => _size.Value.Width;
|
||||
|
||||
public int Height => _size.Value.Height;
|
||||
|
||||
public LightFrame Metadata(int index) => _metadata[index];
|
||||
|
||||
public ValueTask<ImageBuffer?> LoadAsync(int index, int maxLongEdge, CancellationToken cancellationToken)
|
||||
{
|
||||
if ((_frames.Frames[index].Flags & SequenceFrames.NotAnalysed) != 0) return ValueTask.FromResult<ImageBuffer?>(null);
|
||||
return ValueTask.FromResult(Decode(index, maxLongEdge, DecodePurpose.Analysis, cancellationToken)?.Pixels);
|
||||
}
|
||||
|
||||
public async ValueTask<ImageBuffer> GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken)
|
||||
{
|
||||
// The pipeline reads one frame at a time; the gate keeps a window switch from disposing a window in use.
|
||||
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_window is null || _windowKey != (maxLongEdge, purpose))
|
||||
{
|
||||
if (_window is not null) await _window.DisposeAsync().ConfigureAwait(false);
|
||||
_windowKey = (maxLongEdge, purpose);
|
||||
_window = new FrameWindow(Count, (i, ct) => ValueTask.FromResult(DecodeForRender(i, maxLongEdge, purpose, ct)), _windowOptions);
|
||||
}
|
||||
_window.MoveTo(index);
|
||||
using var lease = await _window.GetAsync(index, cancellationToken).ConfigureAwait(false);
|
||||
return lease.Image.Clone();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _gate.WaitAsync().ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_window is not null) await _window.DisposeAsync().ConfigureAwait(false);
|
||||
_window = null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private ImageBuffer DecodeForRender(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) =>
|
||||
Decode(index, maxLongEdge, purpose, cancellationToken)?.Pixels ?? throw Unreadable(index);
|
||||
|
||||
private DecodedImage? Decode(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_frames.Files[index] is not { } file) return null;
|
||||
try
|
||||
{
|
||||
return _decoder.Decode(new DecodeRequest(file, maxLongEdge, purpose), cancellationToken);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("processing", $"Frame {file} could not be read", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The dominant oriented size from the metadata, else the size the first renderable frame decodes to.</summary>
|
||||
private (int, int) MeasureSize()
|
||||
{
|
||||
var context = SequenceContext.FromFrames(_frames.Frames);
|
||||
if (context.SourceWidth > 0 && context.SourceHeight > 0) return (context.SourceWidth, context.SourceHeight);
|
||||
var skipped = _frames.NotRenderable().ToHashSet();
|
||||
for (var i = 0; i < Count; i++)
|
||||
{
|
||||
if (skipped.Contains(i)) continue;
|
||||
using var probe = Decode(i, ProbeLongEdge, DecodePurpose.Thumbnail, CancellationToken.None) ?? throw Unreadable(i);
|
||||
return (probe.SourceWidth, probe.SourceHeight);
|
||||
}
|
||||
throw new RenderException("Export.Render.NoFrames", "Every frame is excluded.");
|
||||
}
|
||||
|
||||
private RenderException Unreadable(int index)
|
||||
{
|
||||
var name = _frames.Frames[index].FileName;
|
||||
return new RenderException(FrameUnreadableKey, $"Frame {index + 1} ({name}) could not be decoded.",
|
||||
[(index + 1).ToString(CultureInfo.InvariantCulture), name]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
using Titano.Core.Analysis;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// One sequence as the processing jobs read it: the frame rows in index order and the file to decode for each (an
|
||||
/// archive original or proxy, or a render node's cached copy; null when there is none).
|
||||
/// </summary>
|
||||
internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList<Frame> Frames, IReadOnlyList<string?> Files)
|
||||
{
|
||||
/// <summary>Flags of frames that never reach a render.</summary>
|
||||
public const FrameFlags NotRendered = FrameFlags.Missing | FrameFlags.Corrupt | FrameFlags.Excluded | FrameFlags.Undecodable;
|
||||
|
||||
/// <summary>Flags of frames whose content is not analysed (absent, or not the file that was archived).</summary>
|
||||
public const FrameFlags NotAnalysed = FrameFlags.Missing | FrameFlags.Corrupt;
|
||||
|
||||
public int Count => Frames.Count;
|
||||
|
||||
/// <summary>Every frame row of <paramref name="sequence"/> with its archive file (the proxy when asked and built).</summary>
|
||||
public static async Task<SequenceFrames> LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var frames = await ListAsync(db, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var files = new string?[frames.Count];
|
||||
for (var i = 0; i < files.Length; i++)
|
||||
{
|
||||
var frame = frames[i];
|
||||
if ((frame.Flags & FrameFlags.Missing) != 0) continue;
|
||||
var proxy = useProxies ? folder.Proxy(frame.Index) : null;
|
||||
files[i] = proxy is not null && File.Exists(proxy) ? proxy : paths.FromRelative(frame.RelativePath);
|
||||
}
|
||||
return new SequenceFrames(sequence, frames, files);
|
||||
}
|
||||
|
||||
public static async Task<List<Frame>> ListAsync(TitanoDatabase db, Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
const int pageSize = 1000;
|
||||
var frames = new List<Frame>();
|
||||
while (true)
|
||||
{
|
||||
var after = frames.Count > 0 ? frames[^1].Index : -1;
|
||||
var page = await db.Frames.ListBySequenceAsync(sequenceId, after, pageSize, cancellationToken).ConfigureAwait(false);
|
||||
frames.AddRange(page);
|
||||
if (page.Count < pageSize) return frames;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Light-analysis input (BRIEF §10.1): every frame's metadata with the sequence's camera and lens. The index keeps
|
||||
/// no GPS, so the first readable file's fix is read once: a time-lapse camera stays put, and the analysis applies
|
||||
/// that position (and the clock offset it implies) to the whole sequence.
|
||||
/// </summary>
|
||||
public IReadOnlyList<LightFrame> LightFrames()
|
||||
{
|
||||
var s = Sequence;
|
||||
var light = new LightFrame[Frames.Count];
|
||||
for (var i = 0; i < light.Length; i++)
|
||||
{
|
||||
light[i] = LightFrame.From(Frames[i]) with
|
||||
{
|
||||
Index = i,
|
||||
CameraMake = s.CameraMake,
|
||||
CameraModel = s.CameraModel,
|
||||
CameraSerial = s.CameraSerial,
|
||||
Lens = s.Lens,
|
||||
};
|
||||
}
|
||||
if (FirstGpsFix() is var (index, gps)) light[index] = light[index] with { Latitude = gps.Latitude, Longitude = gps.Longitude, GpsUtc = gps.UtcTimestamp };
|
||||
return light;
|
||||
}
|
||||
|
||||
/// <summary>Positions of the frames a render must skip besides the project's exclusions.</summary>
|
||||
public IEnumerable<int> NotRenderable()
|
||||
{
|
||||
for (var i = 0; i < Frames.Count; i++)
|
||||
{
|
||||
if (Files[i] is null || (Frames[i].Flags & NotRendered) != 0) yield return i;
|
||||
}
|
||||
}
|
||||
|
||||
private (int Index, GpsPosition Gps)? FirstGpsFix()
|
||||
{
|
||||
for (var i = 0; i < Files.Count; i++)
|
||||
{
|
||||
if (Files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue;
|
||||
try
|
||||
{
|
||||
return MetadataReader.Read(file).Gps is { } gps ? (i, gps) : null;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("processing", $"Metadata of {file} could not be read", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
using Titano.Core.Analysis;
|
||||
using Titano.Core.Analysis.Deep;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Core.Render;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>One output of a render job: the resolved export values and the preset name recorded with the render.</summary>
|
||||
internal sealed record ExportTarget(ResolvedExport Export, string PresetName);
|
||||
|
||||
/// <summary>How one target of <see cref="SequenceRender.RenderAllAsync"/> ended: a result, or the localised reason it failed.</summary>
|
||||
internal sealed record TargetOutcome(ExportTarget Target, RenderResult? Result, string? Error);
|
||||
|
||||
/// <summary>
|
||||
/// The render of one sequence (BRIEF §11): the frame source, the clock-corrected capture metadata, the project with the
|
||||
/// frames that cannot be rendered excluded, and the pipeline with the host's encoders. Targets share one preparation
|
||||
/// pass; each is named once its encoder (and so its codec and size) is known, and failures of one target do not stop
|
||||
/// the others. Outputs are written by the encoder sessions to temporary names and renamed on completion, so a
|
||||
/// cancelled or failed target leaves nothing behind.
|
||||
/// </summary>
|
||||
internal sealed class SequenceRender : IAsyncDisposable
|
||||
{
|
||||
private readonly SequenceFrameSource _source;
|
||||
private readonly RenderPipeline _pipeline;
|
||||
private readonly RenderOptions _options;
|
||||
|
||||
/// <param name="analysis">The sequence's deep report, when current: its luminance spares the preparation a decode pass.</param>
|
||||
public SequenceRender(SequenceFrames frames, ProjectSettings project, DeepAnalysisReport? analysis, ProcessingOptions options)
|
||||
{
|
||||
var metadata = frames.LightFrames();
|
||||
_source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder, options.FrameWindow);
|
||||
_pipeline = new RenderPipeline(options.Encoders, options.Host);
|
||||
_options = options.Render;
|
||||
Input = new RenderInput
|
||||
{
|
||||
Source = _source,
|
||||
Frames = SourceFrameInfo.From(frames.Frames, LightAnalyzer.Analyze(metadata).AdjustedTimes),
|
||||
Project = project with { ExcludedFrames = [.. project.ExcludedFrames.Union(frames.NotRenderable()).Order()] },
|
||||
Analysis = analysis?.FrameCount == frames.Count ? new PrecomputedAnalysis { LogLuminance = Luminance(analysis.Frames) } : null,
|
||||
};
|
||||
}
|
||||
|
||||
public RenderInput Input { get; }
|
||||
|
||||
/// <summary>Encoder, geometry and plan of an export before its output is named.</summary>
|
||||
public RenderSetup Setup(ResolvedExport export) => _pipeline.Setup(Input, new RenderTarget(export, ExportNaming.FallbackName), _options);
|
||||
|
||||
/// <summary>
|
||||
/// Prepares the sequence once and renders every target. <paramref name="name"/> gives the output path of a target
|
||||
/// from its setup; <paramref name="starting"/> runs before a target writes anything (checkpoints),
|
||||
/// <paramref name="finished"/> after its output is complete (record it, upload it). Progress counts measured and
|
||||
/// rendered frames over the whole call.
|
||||
/// </summary>
|
||||
public async Task<IReadOnlyList<TargetOutcome>> RenderAllAsync(
|
||||
IReadOnlyList<(int Index, ExportTarget Target)> targets,
|
||||
Func<ExportTarget, RenderSetup, string> name,
|
||||
Func<int, string, ValueTask> starting,
|
||||
Func<int, ExportTarget, RenderResult, ValueTask> finished,
|
||||
Func<long, long, string?, ValueTask> progress,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var outcomes = new List<TargetOutcome>(targets.Count);
|
||||
var ready = new List<(int Index, ExportTarget Target, RenderSetup Setup)>();
|
||||
foreach (var (index, target) in targets)
|
||||
{
|
||||
RenderSetup setup;
|
||||
try
|
||||
{
|
||||
setup = Setup(target.Export);
|
||||
}
|
||||
catch (RenderException ex)
|
||||
{
|
||||
// The source size could not be measured (no renderable frame decodes).
|
||||
outcomes.Add(new TargetOutcome(target, null, Reason(ex)));
|
||||
continue;
|
||||
}
|
||||
if (setup.Encoder.IsReady && setup.Plan.Frames.Count > 0) ready.Add((index, target, setup));
|
||||
else outcomes.Add(new TargetOutcome(target, null, Reason(setup)));
|
||||
}
|
||||
if (ready.Count == 0) return outcomes;
|
||||
|
||||
var preparing = ready[0].Setup.NeedsPreparation ? ready[0].Setup.Plan.IncludedFrames.Count : 0;
|
||||
var total = preparing + ready.Sum(r => (long)r.Setup.Plan.Frames.Count);
|
||||
var stage = Loc.L("Export.Job.Preparing");
|
||||
var chain = new ProgressChain(progress);
|
||||
var corrections = await RenderPreparation.PrepareAsync(Input, _options, Stage(chain, 0, preparing, total, stage), cancellationToken).ConfigureAwait(false);
|
||||
await chain.FlushAsync().ConfigureAwait(false);
|
||||
|
||||
long done = preparing;
|
||||
foreach (var (index, target, setup) in ready)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var output = name(target, setup);
|
||||
await starting(index, output).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var result = await _pipeline.RenderAsync(Input, corrections, new RenderTarget(target.Export, output), _options,
|
||||
Stage(chain, done, setup.Plan.Frames.Count, total, target.PresetName), cancellationToken).ConfigureAwait(false);
|
||||
await chain.FlushAsync().ConfigureAwait(false);
|
||||
await finished(index, target, result).ConfigureAwait(false);
|
||||
outcomes.Add(new TargetOutcome(target, result, null));
|
||||
}
|
||||
catch (Exception ex) when (!(ex is OperationCanceledException && cancellationToken.IsCancellationRequested))
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("processing", $"Render of {target.PresetName} to {output} failed", ex);
|
||||
outcomes.Add(new TargetOutcome(target, null, Reason(ex)));
|
||||
}
|
||||
done += setup.Plan.Frames.Count;
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
|
||||
/// <summary>Values of the output name tokens: the shoot date and title of the sequence, the codec and size the encoder will use.</summary>
|
||||
public static ExportNameContext NameContext(Sequence sequence, ExportTarget target, RenderSetup setup) => new()
|
||||
{
|
||||
Date = sequence.FirstCapture ?? DateTime.Now,
|
||||
Title = sequence.DisplayTitle,
|
||||
PresetName = target.PresetName,
|
||||
Width = setup.Encoder.Settings.Width,
|
||||
Height = setup.Encoder.Settings.Height,
|
||||
FrameRate = setup.Encoder.Settings.FrameRate,
|
||||
Codec = setup.Encoder.Settings.Codec,
|
||||
};
|
||||
|
||||
/// <summary>Localised text of a render failure.</summary>
|
||||
public static string Reason(Exception exception) =>
|
||||
exception is RenderException render ? Loc.L(render.ReasonKey, [.. render.Args]) : exception.Message;
|
||||
|
||||
/// <summary>Localised remarks of a finished render (encoder fallbacks, ignored settings…), without repeats.</summary>
|
||||
public static IEnumerable<string> Remarks(RenderResult result) => result.Warnings.Select(w => Loc.L(w.Key, [.. w.Args])).Distinct();
|
||||
|
||||
public ValueTask DisposeAsync() => _source.DisposeAsync();
|
||||
|
||||
private static string Reason(RenderSetup setup)
|
||||
{
|
||||
if (setup.Plan.Frames.Count == 0) return Loc.L("Export.Render.NoFrames");
|
||||
var notice = setup.Encoder.Notices.LastOrDefault() ?? new EncoderNotice(EncoderNotices.NoEncoder, setup.Encoder.Settings.Codec.ToString());
|
||||
return Loc.L(notice.Key, [.. notice.Args]);
|
||||
}
|
||||
|
||||
private static double[] Luminance(DeepFrameColumns columns) => [.. columns.LogLuminance.Select(v => v ?? double.NaN)];
|
||||
|
||||
/// <summary>One stage's progress (frames measured, or rendered for one target) as frames of the whole call.</summary>
|
||||
private static IProgress<RenderProgress> Stage(ProgressChain chain, long offset, long length, long total, string message) =>
|
||||
chain.For<RenderProgress>(p => (offset + long.Min(length, p.Done), total, message));
|
||||
}
|
||||
@@ -174,6 +174,23 @@ public sealed class SequenceRepository
|
||||
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1",
|
||||
[id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Records an analysis: status flags set and cleared, the warning count and, for a deep analysis, the compact
|
||||
/// summary (<paramref name="analysisJson"/> null keeps the stored one).
|
||||
/// </summary>
|
||||
public async Task<bool> RecordAnalysisAsync(Guid id, SequenceStatus set, SequenceStatus clear, int warningCount, string? analysisJson,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
DbParam[] args = [id, (int)set, (int)clear, warningCount, DateTimeOffset.UtcNow];
|
||||
var sql = "UPDATE sequences SET status = (status | $2) & ~$3, warning_count = $4, updated_at = $5";
|
||||
if (analysisJson is not null)
|
||||
{
|
||||
sql += ", analysis_json = $6";
|
||||
args = [.. args, DbParam.Json(analysisJson)];
|
||||
}
|
||||
return await _db.ExecuteAsync(sql + " WHERE id = $1", args, cancellationToken).ConfigureAwait(false) > 0;
|
||||
}
|
||||
|
||||
/// <summary>Recomputes frame count, total bytes and first/last capture from the frames table.</summary>
|
||||
public Task<Sequence?> RefreshStatsAsync(Guid id, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Titano.Core\Titano.Core.csproj" />
|
||||
<ProjectReference Include="..\Titano.Protocol\Titano.Protocol.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Include="Migrations\**\*.sql" LogicalName="Titano.Data.Migrations.%(RecursiveDir)%(Filename)%(Extension)" />
|
||||
|
||||
@@ -7,17 +7,21 @@ namespace Titano.Server.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and
|
||||
/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance). Decoding is
|
||||
/// managed only (BRIEF §4.2).
|
||||
/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance), plus the
|
||||
/// analysis handlers of <see cref="ProcessingModule"/>. Decoding is managed only (BRIEF §4.2).
|
||||
/// </summary>
|
||||
public static class ArchiveModule
|
||||
{
|
||||
/// <summary>Name the server writes into the archive's lease files.</summary>
|
||||
public static string Owner { get; } = "server@" + Environment.MachineName;
|
||||
|
||||
public static void Register(IServiceCollection services)
|
||||
{
|
||||
services.AddSingleton(sp => new ArchiveService(sp.GetRequiredService<TitanoDatabase>(),
|
||||
new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = "server@" + Environment.MachineName }));
|
||||
new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = Owner }));
|
||||
services.AddSingleton<IArchiveIngest>(sp => sp.GetRequiredService<ArchiveService>().Ingest);
|
||||
services.AddSingleton<IJobHandlerProvider>(sp => sp.GetRequiredService<ArchiveService>());
|
||||
services.AddSingleton<IFrameAssetLocator, ArchiveAssetLocator>();
|
||||
ProcessingModule.Register(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Video.Managed;
|
||||
using Titano.Data;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Jobs;
|
||||
using Titano.Data.Processing;
|
||||
using Titano.Server.Configuration;
|
||||
|
||||
namespace Titano.Server.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Analysis and render engine of the server (BRIEF §4.2, §10, §14.6): light and deep analysis job handlers with managed
|
||||
/// decoders, so the light analysis queued after uploads runs here. Its render handler is registered too, but the job
|
||||
/// engine host never runs render jobs: render nodes claim them (BRIEF §14.7).
|
||||
/// </summary>
|
||||
public static class ProcessingModule
|
||||
{
|
||||
public static void Register(IServiceCollection services)
|
||||
{
|
||||
services.AddSingleton(sp => new ProcessingService(sp.GetRequiredService<TitanoDatabase>(), new ProcessingOptions
|
||||
{
|
||||
Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = ArchiveModule.Owner },
|
||||
Encoders = [new ManagedEncoderProvider()],
|
||||
Host = EncoderHost.Server,
|
||||
ProducedBy = sp.GetRequiredService<ServerOptions>().ServerName,
|
||||
}));
|
||||
services.AddSingleton<IJobHandlerProvider>(sp => sp.GetRequiredService<ProcessingService>());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
using Titano.Core.Analysis.Deep;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics.Synthetic;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Data.Processing;
|
||||
using Titano.Data.Repositories;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Data.Tests.Processing;
|
||||
|
||||
[Category("processing")]
|
||||
public sealed class AnalysisJobTests(TestContext context)
|
||||
{
|
||||
private static SyntheticSequence Flickering(int frames = 40) => SyntheticSequence.Create(new SyntheticSequenceOptions
|
||||
{
|
||||
FrameCount = frames,
|
||||
FlickerEv = 0.1,
|
||||
StartEv = 0.5,
|
||||
EndEv = -1.5,
|
||||
Latitude = 45.4,
|
||||
Longitude = 12.3,
|
||||
});
|
||||
|
||||
[Test]
|
||||
public async Task LightAnalysisFillsAutoValuesAndKeepsManualOnes()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(Flickering());
|
||||
var manual = new ProjectSettings();
|
||||
manual = manual with { Timing = manual.Timing with { FrameRate = manual.Timing.FrameRate.WithManual("25") } };
|
||||
await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = manual.ToJson() }, null, fx.Ct);
|
||||
await fx.StartEngineAsync();
|
||||
|
||||
var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id], sequence.DisplayTitle));
|
||||
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
var folder = fx.Folder(sequence);
|
||||
var project = ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct));
|
||||
Assert.Equal((false, "25"), (project.Timing.FrameRate.IsAuto, project.Timing.FrameRate.Resolve("30")));
|
||||
Assert.True(project.Timing.Mode.HasAdvice, "timing mode advised");
|
||||
Assert.True(project.Timing.FrameRate.HasAdvice, "the advice is kept next to the manual frame rate");
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal(project.ToJson(), ProjectSettings.FromJson(row.SettingsJson!).ToJson());
|
||||
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
|
||||
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), "no deep status without a report");
|
||||
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
|
||||
Assert.Contains(Loc.L("Advisor.Log.AutoUpdated", sequence.DisplayTitle), done.LogTail);
|
||||
|
||||
// Idempotent: a second pass changes nothing.
|
||||
var version = row.SettingsVersion;
|
||||
var again = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||||
await fx.WaitForAsync(again.Id, JobState.Done);
|
||||
Assert.Equal(version, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.SettingsVersion);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DeepAnalysisResumesAfterCancellationWithTheSameReport()
|
||||
{
|
||||
await using var reference = await ProcessingFixture.CreateAsync(context);
|
||||
var expectedSequence = await reference.AddAsync(Flickering());
|
||||
await reference.StartEngineAsync();
|
||||
var full = await reference.EnqueueAsync(ProcessingJobs.DeepAnalysis([expectedSequence.Id]));
|
||||
await reference.WaitForAsync(full.Id, JobState.Done);
|
||||
var expected = await File.ReadAllTextAsync(reference.Folder(expectedSequence).AnalysisPath, reference.Ct);
|
||||
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(Flickering());
|
||||
var folder = fx.Folder(sequence);
|
||||
var partial = Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName);
|
||||
// Frame 25 is not in the region sample, so the hold stops the pass inside the batch of frames 24–25.
|
||||
fx.Decoder.HoldFrame(25);
|
||||
await fx.StartEngineAsync();
|
||||
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||||
await fx.Engine.CancelAsync(job.Id, fx.Ct);
|
||||
await fx.WaitForAsync(job.Id, JobState.Cancelled);
|
||||
var saved = File.ReadAllLines(partial).Length - 1;
|
||||
Assert.Equal(24, saved);
|
||||
Assert.FileMissing(folder.AnalysisPath);
|
||||
|
||||
fx.Decoder.Hold = null;
|
||||
while (fx.Decoder.Requests.TryDequeue(out _))
|
||||
{
|
||||
}
|
||||
await fx.Engine.ResumeAsync(job.Id, fx.Ct);
|
||||
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
Assert.Equal(expected, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct));
|
||||
Assert.FileMissing(partial);
|
||||
// After the region sample, only the last checkpointed frame is decoded again (to restore the comparison state).
|
||||
var analysed = fx.Decoder.Requests.Skip(ProcessingFixture.DeepOptions.RegionSampleFrames)
|
||||
.Select(r => int.Parse(Path.GetFileNameWithoutExtension(r.Path)[4..]) - 1).ToList();
|
||||
Assert.Equal(saved - 1, analysed.Min());
|
||||
Assert.Equal(sequence.FrameCount, (int)done.ProgressDone);
|
||||
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep | SequenceStatus.AnalyzedLight), "index status");
|
||||
var summary = AnalysisSummary.FromJson(row.AnalysisJson!);
|
||||
Assert.Equal(sequence.FrameCount, summary.FrameCount);
|
||||
Assert.NotNull((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[5].MeanLogLuminance);
|
||||
var project = ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct));
|
||||
Assert.True(project.Exposure.Deflicker.HasAdvice, "deflicker advised from the pixels");
|
||||
Assert.Contains(Loc.L("Advisor.Log.Presets", string.Empty), done.LogTail);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task LightAnalysisAfterNewFramesDropsTheStaleDeepStatus()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(Flickering(12));
|
||||
await fx.StartEngineAsync();
|
||||
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||||
await fx.WaitForAsync(deep.Id, JobState.Done);
|
||||
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedDeep), "deep status");
|
||||
|
||||
// A thirteenth frame arrives: the report describes 12 frames, so it no longer counts.
|
||||
await fx.AddAsync(Flickering(13), first: 12);
|
||||
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||||
await fx.WaitForAsync(light.Id, JobState.Done);
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal(13, row.FrameCount);
|
||||
Assert.Equal((true, false), (row.Status.HasFlag(SequenceStatus.AnalyzedLight), row.Status.HasFlag(SequenceStatus.AnalyzedDeep)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Titano.Core.Analysis.Deep;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Diagnostics.Synthetic;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Jobs;
|
||||
using Titano.Data.Processing;
|
||||
using Titano.Data.Tests.Archiving;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Data.Tests.Processing;
|
||||
|
||||
/// <summary>A temporary archive holding synthetic sequences (JPEGs with Exif), a processing engine and its job engine.</summary>
|
||||
internal sealed class ProcessingFixture : IAsyncDisposable
|
||||
{
|
||||
/// <summary>Small sources are analysed at native size; batches of two frames between checkpoints.</summary>
|
||||
public static readonly DeepAnalysisOptions DeepOptions = new() { MinAnalysisLongEdge = 64, FlowEvery = 5, MaxDegreeOfParallelism = 2, RegionSampleFrames = 9 };
|
||||
|
||||
private JobEngine? _engine;
|
||||
|
||||
private ProcessingFixture(ArchiveFixture archive, GateDecoder decoder)
|
||||
{
|
||||
Archive = archive;
|
||||
Decoder = decoder;
|
||||
Processing = new ProcessingService(archive.Db, new ProcessingOptions
|
||||
{
|
||||
Archive = new ArchiveServiceOptions { Decoder = new ImageDecoder([decoder]), Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) },
|
||||
DeepAnalysis = DeepOptions,
|
||||
ProducedBy = "test-pc",
|
||||
});
|
||||
}
|
||||
|
||||
public ArchiveFixture Archive { get; }
|
||||
public GateDecoder Decoder { get; }
|
||||
public ProcessingService Processing { get; }
|
||||
public TitanoDatabase Db => Archive.Db;
|
||||
public CancellationToken Ct => Archive.Ct;
|
||||
|
||||
public JobEngine Engine => _engine ?? throw new InvalidOperationException("Start the engine first.");
|
||||
|
||||
public static async Task<ProcessingFixture> CreateAsync(TestContext context) => new(await ArchiveFixture.CreateAsync(context), new GateDecoder());
|
||||
|
||||
/// <summary>A started engine with the processing handlers (one worker, fast checkpoints).</summary>
|
||||
public async Task<JobEngine> StartEngineAsync()
|
||||
{
|
||||
_engine = new JobEngine(Db, Processing.CreateJobHandlers(),
|
||||
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
|
||||
await _engine.StartAsync(Ct);
|
||||
return _engine;
|
||||
}
|
||||
|
||||
public Task<Job> EnqueueAsync(Titano.Data.Archiving.Jobs.ArchiveJobRequest request) => Engine.EnqueueAsync(request, cancellationToken: Ct);
|
||||
|
||||
public Task<Job> WaitForAsync(Guid jobId, JobState state) => Archive.WaitForAsync(jobId, state, 60);
|
||||
|
||||
/// <summary>Writes frames <paramref name="first"/>… of <paramref name="sequence"/> as JPEGs with their Exif into
|
||||
/// <paramref name="folder"/> (adopted in place) and indexes the archive.</summary>
|
||||
public async Task<Sequence> AddAsync(SyntheticSequence sequence, string folder = "Synthetic", int first = 0)
|
||||
{
|
||||
var directory = Path.Combine(Archive.Paths.Root, folder);
|
||||
Directory.CreateDirectory(directory);
|
||||
for (var i = first; i < sequence.Count; i++)
|
||||
{
|
||||
using var image = sequence.Render(i);
|
||||
var exif = ExifWriter.BuildJpegApp1(sequence.Metadata[i]);
|
||||
File.WriteAllBytes(Path.Combine(directory, FileName(i)), JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 95, App1Segments = [exif[4..]] }));
|
||||
}
|
||||
await Archive.ScanAsync(ScanMode.Full);
|
||||
var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder);
|
||||
return (await Db.Sequences.GetAsync(location.Id, Ct))!;
|
||||
}
|
||||
|
||||
public static string FileName(int index) => $"IMG_{index + 1:0000}.JPG";
|
||||
|
||||
public SequencePaths Folder(Sequence sequence) => Archive.Paths.Sequence(sequence.Folder);
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_engine is not null) await _engine.DisposeAsync();
|
||||
await Archive.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Managed decoding that records every request and can hold one file's decode until the job is cancelled.</summary>
|
||||
internal sealed class GateDecoder : IImageDecoderProvider
|
||||
{
|
||||
private readonly ManagedDecoderProvider _managed = new();
|
||||
|
||||
public string Name => "gate";
|
||||
|
||||
public ConcurrentQueue<DecodeRequest> Requests { get; } = new();
|
||||
|
||||
/// <summary>File name whose decode waits for cancellation; null decodes everything.</summary>
|
||||
public string? Hold { get; set; }
|
||||
|
||||
/// <summary>Holds only decodes of this purpose (null: any).</summary>
|
||||
public DecodePurpose? HoldPurpose { get; set; }
|
||||
|
||||
/// <summary>Holds only decodes with this long edge (null: any).</summary>
|
||||
public int? HoldEdge { get; set; }
|
||||
|
||||
/// <summary>Completed when a decode of <see cref="Hold"/> started waiting.</summary>
|
||||
public TaskCompletionSource Holding { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public void HoldFrame(int index, DecodePurpose? purpose = null, int? edge = null)
|
||||
{
|
||||
Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
(HoldPurpose, HoldEdge) = (purpose, edge);
|
||||
Hold = ProcessingFixture.FileName(index);
|
||||
}
|
||||
|
||||
public DecodeTier? TierFor(ImageFileFormat format) => _managed.TierFor(format);
|
||||
|
||||
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Requests.Enqueue(request);
|
||||
if (Hold is { } hold && Path.GetFileName(request.Path) == hold && (HoldPurpose ?? request.Purpose) == request.Purpose
|
||||
&& (HoldEdge ?? request.MaxLongEdge) == request.MaxLongEdge)
|
||||
{
|
||||
Holding.TrySetResult();
|
||||
cancellationToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(30));
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
}
|
||||
return _managed.Decode(request, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
using System.Text;
|
||||
using Titano.Core.Diagnostics.Synthetic;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Data.Processing;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Data.Tests.Processing;
|
||||
|
||||
[Category("processing")]
|
||||
public sealed class RenderJobTests(TestContext context)
|
||||
{
|
||||
private const int Frames = 24;
|
||||
|
||||
/// <summary>MJPEG at the source size, named by codec so the tests know the file names.</summary>
|
||||
private static readonly ExportSettings Mjpeg = new()
|
||||
{
|
||||
Preset = new Auto<string>().WithManual("mjpeg-avi"),
|
||||
Overrides = new ExportOverrides { Width = 96, Height = 64 },
|
||||
NameTemplate = "clip-{codec}",
|
||||
};
|
||||
|
||||
private static SyntheticSequence Scene() => SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
|
||||
|
||||
[Test]
|
||||
public async Task ABatchWritesPlayableManagedOutputsIntoExportsAndRecordsThem()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(Scene());
|
||||
await fx.StartEngineAsync();
|
||||
|
||||
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["png-sequence", "gif-preview"] }, 96, 64));
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
var exports = fx.Folder(sequence).Exports;
|
||||
var avi = await File.ReadAllBytesAsync(Path.Combine(exports, "clip-MJPEG.avi"), fx.Ct);
|
||||
Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(avi, 0, 4), Encoding.ASCII.GetString(avi, 8, 4)));
|
||||
Assert.Contains("idx1", Encoding.ASCII.GetString(avi));
|
||||
var pngs = Directory.GetFiles(Path.Combine(exports, "clip-PNG")).Order().ToList();
|
||||
Assert.Equal(Frames, pngs.Count);
|
||||
Assert.Equal("clip-PNG_000001.png", Path.GetFileName(pngs[0]));
|
||||
using (var decoded = DecoderFactory.CreateServer().Decode(new DecodeRequest(pngs[^1])))
|
||||
Assert.Equal((96, 64), (Assert.NotNull(decoded).Pixels.Width, decoded.Pixels.Height));
|
||||
var gif = await File.ReadAllBytesAsync(Path.Combine(exports, "clip-GIF.gif"), fx.Ct);
|
||||
Assert.Equal("GIF89a", Encoding.ASCII.GetString(gif, 0, 6));
|
||||
Assert.Empty(Directory.EnumerateFileSystemEntries(exports, ".*"));
|
||||
|
||||
var renders = (await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).OrderBy(r => r.OutputPath, StringComparer.Ordinal).ToList();
|
||||
Assert.SequenceEqual(["Synthetic/exports/clip-GIF.gif", "Synthetic/exports/clip-MJPEG.avi", "Synthetic/exports/clip-PNG"], renders.Select(r => r.OutputPath));
|
||||
var mjpeg = renders[1];
|
||||
Assert.Equal((Loc.L("Export.Preset.Custom"), "Mjpeg", 96, 64, "test-pc", (long)avi.Length),
|
||||
(mjpeg.PresetName, mjpeg.Codec, mjpeg.Width, mjpeg.Height, mjpeg.ProducedBy, mjpeg.Bytes));
|
||||
Assert.Equal(Loc.L("Export.Preset.Gif"), renders[0].PresetName);
|
||||
Assert.Equal(3, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACancelledRenderLeavesNothingBehind()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(Scene());
|
||||
fx.Decoder.HoldFrame(12, DecodePurpose.Render);
|
||||
await fx.StartEngineAsync();
|
||||
|
||||
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
|
||||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||||
await fx.Engine.CancelAsync(job.Id, fx.Ct);
|
||||
await fx.WaitForAsync(job.Id, JobState.Cancelled);
|
||||
await WaitUntilIdleAsync(fx, job.Id);
|
||||
|
||||
Assert.Empty(Entries(fx.Folder(sequence).Exports));
|
||||
Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFailedRenderLeavesNothingBehindAndSaysWhy()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(Scene());
|
||||
// Damaged after indexing: nothing flags the frame, so only decoding finds out.
|
||||
await File.WriteAllBytesAsync(Path.Combine(fx.Archive.Paths.Root, "Synthetic", ProcessingFixture.FileName(10)), new byte[5000], fx.Ct);
|
||||
await fx.StartEngineAsync();
|
||||
|
||||
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
|
||||
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
|
||||
|
||||
Assert.Equal(Loc.L(SequenceFrameSource.FrameUnreadableKey, "11", ProcessingFixture.FileName(10)), failed.Error);
|
||||
Assert.Empty(Entries(fx.Folder(sequence).Exports));
|
||||
Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARenderInterruptedByACrashStartsItsOutputAgainCleanly()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(Scene());
|
||||
var exports = fx.Folder(sequence).Exports;
|
||||
var output = Path.Combine(exports, "clip-MJPEG.avi");
|
||||
// The crashed attempt left its encoder's temporary file and a finished output it never recorded.
|
||||
Directory.CreateDirectory(exports);
|
||||
await File.WriteAllBytesAsync(Path.Combine(exports, ".clip-MJPEG.avi.0badf00d.tmp"), new byte[100], fx.Ct);
|
||||
await File.WriteAllBytesAsync(output, new byte[100], fx.Ct);
|
||||
await fx.StartEngineAsync();
|
||||
|
||||
var job = await fx.Archive.EnqueueWithCheckpointAsync(fx.Engine, ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64),
|
||||
new RenderCheckpoint([], 0, output).ToJson());
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
Assert.SequenceEqual(["clip-GIF.gif", "clip-MJPEG.avi"], Entries(exports));
|
||||
Assert.Equal("RIFF", Encoding.ASCII.GetString((await File.ReadAllBytesAsync(output, fx.Ct))[..4]));
|
||||
Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARenderInterruptedAfterRecordingAnOutputResumesWithTheNextOne()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(Scene());
|
||||
var exports = fx.Folder(sequence).Exports;
|
||||
var output = Path.Combine(exports, "clip-MJPEG.avi");
|
||||
Directory.CreateDirectory(exports);
|
||||
var recorded = Encoding.ASCII.GetBytes("RIFF recorded before the crash");
|
||||
await File.WriteAllBytesAsync(output, recorded, fx.Ct);
|
||||
await fx.Db.Renders.InsertAsync(new Render
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
SequenceId = sequence.Id,
|
||||
PresetName = "MJPEG",
|
||||
OutputPath = "Synthetic/exports/clip-MJPEG.avi",
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
}, fx.Ct);
|
||||
await fx.StartEngineAsync();
|
||||
|
||||
var job = await fx.Archive.EnqueueWithCheckpointAsync(fx.Engine, ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64),
|
||||
new RenderCheckpoint([], 0, output).ToJson());
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
var kept = await File.ReadAllBytesAsync(output, fx.Ct);
|
||||
Assert.BytesEqual(recorded, kept);
|
||||
Assert.SequenceEqual(["clip-GIF.gif", "clip-MJPEG.avi"], Entries(exports));
|
||||
Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
|
||||
}
|
||||
|
||||
private static List<string> Entries(string folder) =>
|
||||
Directory.Exists(folder) ? [.. Directory.EnumerateFileSystemEntries(folder).Select(e => Path.GetFileName(e)).Order(StringComparer.Ordinal)] : [];
|
||||
|
||||
/// <summary>The engine marks a job cancelled before its handler has unwound; wait until the worker let it go.</summary>
|
||||
private static async Task WaitUntilIdleAsync(ProcessingFixture fx, Guid jobId)
|
||||
{
|
||||
var until = DateTime.UtcNow.AddSeconds(30);
|
||||
while (fx.Engine.RunningJobIds.Contains(jobId))
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < until, "the render did not stop");
|
||||
await Task.Delay(20, fx.Ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
using System.Text;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Diagnostics.Synthetic;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Processing;
|
||||
using Titano.Protocol;
|
||||
using Titano.Protocol.Client;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Server.Tests.Jobs;
|
||||
|
||||
/// <summary>The desktop client's render node (managed encoders here) against the in-process server.</summary>
|
||||
[Category("processing")]
|
||||
public sealed class RenderNodeClientTests(TestContext context)
|
||||
{
|
||||
private const int Frames = 12;
|
||||
|
||||
private static readonly ExportSettings Mjpeg = new()
|
||||
{
|
||||
Preset = new Auto<string>().WithManual("mjpeg-avi"),
|
||||
Overrides = new ExportOverrides { Width = 96, Height = 64 },
|
||||
NameTemplate = "{title} - {codec}",
|
||||
};
|
||||
|
||||
[Test]
|
||||
public async Task ANodeClaimsRendersWithManagedEncodersAndUploadsTheResult()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
||||
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
||||
var job = await CreateRenderAsync(server, editor, sequenceId);
|
||||
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
||||
var cache = context.CreateTempDirectory("node");
|
||||
await using var node = Node(client, cache);
|
||||
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
||||
Assert.False(await node.RunOnceAsync(context.Cancellation), "the queue is empty");
|
||||
|
||||
var done = await JobApiTests.GetAsync(editor, job.Id);
|
||||
Assert.Equal("done", done.State);
|
||||
var render = (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Single();
|
||||
Assert.Equal(("studio-pc", "mjpeg", 96, 64), (render.ProducedBy, render.Codec, render.Width, render.Height));
|
||||
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
|
||||
var file = await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, render.OutputPath));
|
||||
Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(file, 0, 4), Encoding.ASCII.GetString(file, 8, 4)));
|
||||
Assert.Equal($"{sequence.Folder}/exports/{sequence.DisplayTitle} - MJPEG.avi", render.OutputPath);
|
||||
Assert.Equal(1, sequence.RenderedCount);
|
||||
// Originals stay cached by content for the next job; the job's own folder is gone.
|
||||
Assert.Count(Frames, Directory.GetFiles(Path.Combine(cache, "originals"), "*.jpg", SearchOption.AllDirectories));
|
||||
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANodeReportsWhyItCouldNotRender()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
||||
// Uploads are opaque to the server's fake ingest: these "photos" decode to nothing.
|
||||
var sequenceId = await UploadSequenceAsync(editor, [Uploads.Photo(4000, 1), Uploads.Photo(4000, 2)]);
|
||||
var job = await CreateRenderAsync(server, editor, sequenceId);
|
||||
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
||||
await using var node = Node(client, context.CreateTempDirectory("node"));
|
||||
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
||||
|
||||
var failed = await JobApiTests.GetAsync(editor, job.Id);
|
||||
var first = (await server.Database.Frames.ListBySequenceAsync(sequenceId))[0];
|
||||
Assert.Equal(("failed", Loc.L("Export.Render.FrameUnreadable", "1", first.FileName)), (failed.State, failed.Error));
|
||||
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequenceId));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANodeOffersOnlyEncodersWhoseOutputIsOneFile()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (_, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
||||
await using var node = Node(client, context.CreateTempDirectory("node"));
|
||||
|
||||
var id = await node.RegisterAsync(context.Cancellation);
|
||||
var registered = (await client.ListNodesAsync(context.Cancellation)).Single(n => n.Id == id);
|
||||
Assert.SequenceEqual(["apng", "gif", "mjpeg"], registered.Encoders.Select(e => e.Codec).Order(StringComparer.Ordinal));
|
||||
}
|
||||
|
||||
private static RenderNodeClient Node(TitanoClient client, string cache) =>
|
||||
new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } },
|
||||
new RenderNodeOptions { ClientName = "studio-pc", CacheDirectory = cache, ProgressInterval = TimeSpan.Zero });
|
||||
|
||||
private static async Task<JobDto> CreateRenderAsync(TestServer server, HttpClient editor, Guid sequenceId)
|
||||
{
|
||||
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
|
||||
var request = ProcessingJobs.Render(sequence, Mjpeg, 96, 64);
|
||||
return await JobApiTests.CreateAsync(editor, "render", sequenceId, request.ParametersJson);
|
||||
}
|
||||
|
||||
/// <summary>Uploads the photos as IMG_0001.JPG… into one new sequence; returns its id.</summary>
|
||||
private static async Task<Guid> UploadSequenceAsync(HttpClient client, IReadOnlyList<byte[]> photos)
|
||||
{
|
||||
Guid? sequenceId = null;
|
||||
for (var i = 0; i < photos.Count; i++)
|
||||
sequenceId = (await Uploads.UploadAsync(client, photos[i], $"IMG_{i + 1:0000}.JPG", sequenceId)).SequenceId;
|
||||
return sequenceId!.Value;
|
||||
}
|
||||
|
||||
private static List<byte[]> Photos()
|
||||
{
|
||||
var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
|
||||
var photos = new List<byte[]>(Frames);
|
||||
for (var i = 0; i < Frames; i++)
|
||||
{
|
||||
using var image = scene.Render(i);
|
||||
photos.Add(JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90 }));
|
||||
}
|
||||
return photos;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Diagnostics.Synthetic;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Server.Storage;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Server.Tests.Storage;
|
||||
|
||||
/// <summary>The production composition: frames committed by the archive engine are light-analysed by the processing handlers.</summary>
|
||||
[Category("processing")]
|
||||
public sealed class ProcessingModuleTests(TestContext context)
|
||||
{
|
||||
[Test]
|
||||
public async Task UploadedFramesAreLightAnalysedAndAdvised()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 10, StartEv = 0.5, EndEv = -0.5 });
|
||||
Guid sequenceId = default;
|
||||
for (var i = 0; i < scene.Count; i++)
|
||||
{
|
||||
using var image = scene.Render(i);
|
||||
var exif = ExifWriter.BuildJpegApp1(scene.Metadata[i]);
|
||||
var photo = JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90, App1Segments = [exif[4..]] });
|
||||
sequenceId = (await Uploads.UploadAsync(editor, photo, $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
}
|
||||
|
||||
// The commits queue a light analysis; its handler comes from the processing module.
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
Sequence sequence;
|
||||
while (true)
|
||||
{
|
||||
sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
|
||||
if (sequence.Status.HasFlag(SequenceStatus.AnalyzedLight) && sequence.FrameCount == scene.Count
|
||||
&& !await server.Database.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequenceId)) break;
|
||||
Assert.True(DateTime.UtcNow < until, $"not analysed: {sequence.Status}, {sequence.FrameCount} frames");
|
||||
await Task.Delay(100);
|
||||
}
|
||||
var project = ProjectSettings.FromJson(Assert.NotNull(sequence.SettingsJson));
|
||||
Assert.True(project.Timing.Mode.HasAdvice, "timing mode advised");
|
||||
var folder = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder);
|
||||
Assert.FileExists(folder.ProjectPath);
|
||||
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user