diff --git a/i18n/en/Advisor.resx b/i18n/en/Advisor.resx index fad2d883..111c3a07 100644 --- a/i18n/en/Advisor.resx +++ b/i18n/en/Advisor.resx @@ -31,6 +31,11 @@ {0} frames could not be decoded.{0} frame count Colour balance jumps by {0:0.#}% between frames.{0} RMS colour-ratio jitter in percent {0} frames are blown out to white.{0} frame count + Automatic values of {0} updated.job log line; {0} sequence title + Deep analysis of {0}: {1} frames, {2} warnings.job log line; {0} sequence title; {1} frame count; {2} warning count + Light analysis of {0}: {1} frames, {2} warnings.job log line; {0} sequence title; {1} frame count; {2} warning count + Suggested presets: {0}job log line; {0} preset names separated by dots + The project settings of {0} cannot be read; their automatic values were left as they were.job log line; {0} sequence title Needs the deep analysis of the pixels, which has not run yet; nothing was changed.no placeholders Shutter time or aperture is missing from the metadata, so this cannot be decided; nothing was changed.no placeholders Too few frame pairs to measure motion; nothing was changed.no placeholders diff --git a/i18n/en/Errors.resx b/i18n/en/Errors.resx index a57f5095..22ce17de 100644 --- a/i18n/en/Errors.resx +++ b/i18n/en/Errors.resx @@ -11,6 +11,11 @@ This path is a file, not a folder. The drive of this folder is not available. Enter a full path that starts with a drive or a network share. + {0} could not be downloaded intact.{0} file name + {0} of {1} exports failed; see the job log.{0} failed count; {1} total count + Render nodes send back one file per job; image sequences render where the archive is. + The render job has no valid export settings. + The sequence to render is no longer in the archive. Enter a server address such as nas.local:8443 or https://titano.example.com. This file does not contain Titano settings. The template is empty. diff --git a/i18n/en/Export.resx b/i18n/en/Export.resx index b5dafdeb..3477a2d4 100644 --- a/i18n/en/Export.resx +++ b/i18n/en/Export.resx @@ -13,6 +13,10 @@ Resolution reduced to {1}, the maximum of level {0}.long; {0} codec level name; {1} size as WIDTHxHEIGHT No hardware encoder: {0} encodes in software, which is slower.long; {0} encoder name {0} cannot encode 10-bit; 8-bit is used.long; {0} encoder name + Downloading framesrender node progress stage + Measuring framesjob progress stage + {0} failed: {1}long: job log line; {0} preset name; {1} reason + {0} saved as {1}long: job log line; {0} preset name; {1} output file name Playback rate not understood; the export frame rate is used.long No capture times: frames are spaced evenly.long Every frame is excluded.long @@ -28,6 +32,7 @@ 8K archive Animated PNG Cinema 21:9invariant + Custompreset name when the values match no preset DCI 4Kinvariant Animated GIF JPEG frames @@ -44,6 +49,7 @@ Motion blur is off while frames are stacked.long The crop lies outside the frame and is ignored.long Easing “{0}” not understood; linear is used.long; {0} easing name as entered + Frame {0} ({1}) cannot be decoded; exclude it or check the file.long; {0} frame number from 1; {1} file name The keyframe at frame {0} is invalid and was skipped.long; {0} source frame number Highlight knee {0} is out of range and is ignored.long; {0} knee value as entered Every frame is excluded; there is nothing to render.long diff --git a/i18n/it/Advisor.it.resx b/i18n/it/Advisor.it.resx index 88685171..892a7f6e 100644 --- a/i18n/it/Advisor.it.resx +++ b/i18n/it/Advisor.it.resx @@ -31,6 +31,11 @@ {0} fotogrammi non sono decodificabili. Il bilanciamento del colore varia del {0:0.#}% tra un fotogramma e l'altro. {0} fotogrammi sono completamente bruciati. + Valori automatici di {0} aggiornati. + Analisi approfondita di {0}: {1} fotogrammi, {2} avvisi. + Analisi rapida di {0}: {1} fotogrammi, {2} avvisi. + Preset suggeriti: {0} + Le impostazioni del progetto di {0} non sono leggibili; i valori automatici sono rimasti invariati. Serve l'analisi approfondita dei pixel, non ancora eseguita; nulla è stato cambiato. Tempo di scatto o diaframma mancano nei metadati, quindi non si può decidere; nulla è stato cambiato. Troppo poche coppie di fotogrammi per misurare il movimento; nulla è stato cambiato. diff --git a/i18n/it/Errors.it.resx b/i18n/it/Errors.it.resx index d3b8318d..31b8b300 100644 --- a/i18n/it/Errors.it.resx +++ b/i18n/it/Errors.it.resx @@ -11,6 +11,11 @@ Questo percorso è un file, non una cartella. L'unità di questa cartella non è disponibile. Inserisci un percorso completo che inizi con un'unità o una condivisione di rete. + {0} non è stato scaricato integro. + {0} esportazioni su {1} non riuscite; vedi il registro del job. + I nodi di rendering restituiscono un file per job; le sequenze di immagini si renderizzano dove si trova l'archivio. + Il job di rendering non ha impostazioni di esportazione valide. + La sequenza da renderizzare non è più nell'archivio. Inserisci l'indirizzo di un server, ad esempio nas.local:8443 o https://titano.example.com. Questo file non contiene impostazioni di Titano. Il modello è vuoto. diff --git a/i18n/it/Export.it.resx b/i18n/it/Export.it.resx index 513b88a6..27aed9a6 100644 --- a/i18n/it/Export.it.resx +++ b/i18n/it/Export.it.resx @@ -13,6 +13,10 @@ Risoluzione ridotta a {1}, il massimo del livello {0}. Nessun encoder hardware: {0} codifica via software, più lentamente. {0} non codifica a 10 bit; viene usato 8 bit. + Download dei fotogrammi + Misura dei fotogrammi + {0} non riuscito: {1} + {0} salvato come {1} Velocità di riproduzione non valida; viene usata la frequenza di esportazione. Nessun orario di scatto: fotogrammi distribuiti in modo uniforme. Tutti i fotogrammi sono esclusi. @@ -28,6 +32,7 @@ Archivio 8K PNG animato Cinema 21:9 + Personalizzato DCI 4K GIF animata Fotogrammi JPEG @@ -44,6 +49,7 @@ La sfocatura di movimento è disattivata mentre i fotogrammi sono sovrapposti. Il ritaglio è fuori dall'immagine e viene ignorato. Andamento «{0}» non valido; viene usato quello lineare. + Il fotogramma {0} ({1}) non è decodificabile; escludilo o controlla il file. Il fotogramma chiave {0} non è valido ed è stato saltato. Compressione delle alte luci {0} fuori intervallo, ignorata. Tutti i fotogrammi sono esclusi: non c'è nulla da renderizzare. diff --git a/src/Titano.App/Infrastructure/AppServices.cs b/src/Titano.App/Infrastructure/AppServices.cs index f366bf3e..5ed5098f 100644 --- a/src/Titano.App/Infrastructure/AppServices.cs +++ b/src/Titano.App/Infrastructure/AppServices.cs @@ -1,3 +1,4 @@ +using System.ComponentModel; using System.IO; using System.Net.Http; using System.Reflection; @@ -8,21 +9,29 @@ using Titano.App.Server; using Titano.App.Settings; using Titano.Core.Archiving; using Titano.Core.Diagnostics; +using Titano.Core.Export; +using Titano.Core.Imaging; using Titano.Core.Imaging.Decoders; using Titano.Core.Model; +using Titano.Core.Video.Managed; +using Titano.Core.Windows.MediaFoundation; using Titano.Core.Windows.Wic; using Titano.Data; using Titano.Data.Archiving; using Titano.Data.Jobs; +using Titano.Data.Processing; +using Titano.Protocol.Client; namespace Titano.App.Infrastructure; /// /// Composition root of local mode (BRIEF §4.4): the SQLite index under the data folder (migrated on open), the -/// archive engine (WIC first, then managed decoders), the one of this PC with the handlers of -/// , thumbnails, the server session, the encoder inventory and the update check. -/// Started after the window shows, stopped when the app exits. The last archive is reopened at start and rescanned -/// every (BRIEF §7: polling, no file-system watchers). +/// archive engine (WIC first, then managed decoders), the analysis and render engine (the same decoders; Media +/// Foundation, then managed encoders), the one of this PC with the handlers of +/// , thumbnails, the server session, the render node, the encoder inventory and the +/// update check. Started after the window shows, stopped when the app exits. The last archive is reopened at start and +/// rescanned every (BRIEF §7: polling, no file-system watchers). The render node runs while +/// the user offers this PC () and a server session is open. /// public sealed class AppServices : IAsyncDisposable { @@ -32,6 +41,9 @@ public sealed class AppServices : IAsyncDisposable private readonly CancellationTokenSource _stopping = new(); private readonly Dictionary _scans = []; private Task _polling = Task.CompletedTask; + private ServerSession _server = null!; + private Task _nodeSwitch = Task.CompletedTask; + private (RenderNodeClient Node, TitanoClient Client, Uri Address, string Token)? _node; public AppServices(AppEnvironment environment, AppSettings settings, Action saveSettings, IAppDialogs dialogs) { @@ -57,7 +69,20 @@ public sealed class AppServices : IAsyncDisposable /// The archive engine; null until the index is open. public ArchiveService? Archives { get; private set; } - public ServerSession Server { get; set; } + /// Analysis and render engine; null until the index is open. + public ProcessingService? Processing { get; private set; } + + public ServerSession Server + { + get => _server; + set + { + if (_server is not null) _server.PropertyChanged -= OnServerChanged; + _server = value; + _server.PropertyChanged += OnServerChanged; + UpdateRenderNode(); + } + } public IEncoderInventory Encoders { get; set; } @@ -86,7 +111,7 @@ public sealed class AppServices : IAsyncDisposable /// The job handlers of this PC's engine — the single registration list; a kind without a handler stays /// queued (it is never claimed here). public static IReadOnlyList CreateJobHandlers(AppServices services) => - services.Archives?.CreateJobHandlers() ?? []; + [.. services.Archives?.CreateJobHandlers() ?? [], .. services.Processing?.CreateJobHandlers() ?? []]; /// Queues a scan of unless one is already waiting or running. public async Task QueueScanAsync(Archive archive, bool full, CancellationToken ct = default) @@ -105,7 +130,14 @@ public sealed class AppServices : IAsyncDisposable } } - public void SaveSettings() => SaveSettingsAction(); + public void SaveSettings() + { + SaveSettingsAction(); + UpdateRenderNode(); + } + + /// Starts or stops the render node to match the opt-in and the server session (BRIEF §14.7); switches run in order. + public void UpdateRenderNode() => _nodeSwitch = SwitchRenderNodeAsync(_nodeSwitch); /// Opens (or creates) the index at , migrates it and starts the job engine. public async Task OpenAsync(string databasePath, CancellationToken ct = default) @@ -139,6 +171,9 @@ public sealed class AppServices : IAsyncDisposable await _stopping.CancelAsync(); await _polling; _stopping.Dispose(); + _server.PropertyChanged -= OnServerChanged; + await _nodeSwitch; + await StopRenderNodeAsync(); (Jobs as IDisposable)?.Dispose(); if (Engine is not null) await Engine.DisposeAsync(); Thumbnails?.Dispose(); @@ -171,7 +206,9 @@ public sealed class AppServices : IAsyncDisposable private async Task StartJobsAsync(TitanoDatabase database, bool startEngine, CancellationToken ct) { - Archives = new ArchiveService(database, new ArchiveServiceOptions { Decoder = DecoderFactory.CreateClient(new WicDecoderProvider()) }); + var archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateClient(new WicDecoderProvider()) }; + Archives = new ArchiveService(database, archive); + Processing = new ProcessingService(database, ProcessingOptions(archive)); var handlers = CreateJobHandlers(this); var kinds = handlers.Select(h => h.Kind).Distinct().ToArray(); Engine = new JobEngine(database, handlers, new JobEngineOptions { WorkerId = WorkerId, Kinds = kinds }); @@ -180,5 +217,65 @@ public sealed class AppServices : IAsyncDisposable if (started) await Engine.StartAsync(ct); Jobs = new EngineJobQueue(database, Engine, started); Log.Info("jobs", started ? $"Job engine started with {Engine.Options.Workers} worker(s) for {string.Join(", ", kinds)}" : "No job handlers registered; jobs stay queued"); + UpdateRenderNode(); + } + + /// Media Foundation (hardware first, software H.264) then managed encoders; frames spill into the cache folder + /// within the memory budget of Settings › Rendering (Auto: a quarter of the installed memory). + private ProcessingOptions ProcessingOptions(ArchiveServiceOptions archive) => new() + { + Archive = archive, + Encoders = [new MediaFoundationEncoderProvider(), new ManagedEncoderProvider()], + Host = EncoderHost.Client, + ProducedBy = System.Environment.MachineName, + FrameWindow = new FrameWindowOptions + { + MemoryBudget = Settings.MemoryBudgetGb is { } gb ? gb * (1L << 30) : GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 4, + SpillDirectory = CacheStore.Folder(Environment, Settings), + }, + }; + + private void OnServerChanged(object? sender, PropertyChangedEventArgs e) => UpdateRenderNode(); + + /// Waits for the previous switch, then runs the node for the current session, or stops it. + private async Task SwitchRenderNodeAsync(Task previous) + { + await previous; + var flow = _server.Flow; + (ProcessingService Engine, Uri Address, string Token)? wanted = + Settings.RenderNodeOptIn && _server.IsConnected && Processing is { } processing && flow.Address is { } address && flow.Token is { } token + ? (processing, address, token) + : null; + if (_node is { } running && wanted is { } same && running.Address == same.Address && running.Token == same.Token) return; + await StopRenderNodeAsync(); + if (wanted is not { } start) return; + try + { + var cache = CacheStore.Folder(Environment, Settings); + CacheStore.Prepare(cache); + var client = new TitanoClient(start.Address, new CertificatePinning(_server.Fingerprints), $"Titano ({System.Environment.MachineName})") { Token = start.Token }; + var node = new RenderNodeClient(client, start.Engine.Options, new RenderNodeOptions + { + ClientName = System.Environment.MachineName, + CacheDirectory = Path.Combine(cache, "render-node"), + CacheLimitBytes = Settings.CacheLimitGb * (1L << 30), + }); + node.Start(); + _node = (node, client, start.Address, start.Token); + Log.Info("render-node", $"Offering this PC as render node to {start.Address}"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("render-node", "The render node could not start", ex); + } + } + + private async Task StopRenderNodeAsync() + { + if (_node is not { } running) return; + _node = null; + await running.Node.DisposeAsync(); + running.Client.Dispose(); + Log.Info("render-node", "Render node stopped"); } } diff --git a/src/Titano.Core/Archiving/LeaseFile.cs b/src/Titano.Core/Archiving/LeaseFile.cs index dbd1ced7..86f88e8c 100644 --- a/src/Titano.Core/Archiving/LeaseFile.cs +++ b/src/Titano.Core/Archiving/LeaseFile.cs @@ -189,16 +189,23 @@ public sealed class LeaseFile : IDisposable private static bool TryCreate(string path, LeaseRecord record) { - try + for (var attempt = 1; ; attempt++) { - using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read); - fs.Write(JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord)); - fs.Flush(flushToDisk: true); - return true; - } - catch (IOException) when (File.Exists(path)) - { - return false; + try + { + using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read); + fs.Write(JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord)); + fs.Flush(flushToDisk: true); + return true; + } + catch (IOException) when (File.Exists(path)) + { + return false; + } + catch (IOException) when (attempt < 3) + { + // The holder released the lease between the failed create and the check: the name is free again. + } } } diff --git a/src/Titano.Core/Project/ProjectSettings.cs b/src/Titano.Core/Project/ProjectSettings.cs index 82bffb78..970a6dc7 100644 --- a/src/Titano.Core/Project/ProjectSettings.cs +++ b/src/Titano.Core/Project/ProjectSettings.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace Titano.Core.Project; @@ -181,13 +182,40 @@ public sealed record ProjectSettings public string ToJson() => JsonSerializer.Serialize(this, ProjectJson.Default.ProjectSettings); + /// + /// Reads a project; members missing from (an API client's partial settings, an older file) + /// keep their defaults. The JSON is laid over the default project first, because generated deserialisation leaves + /// missing init-only members null or false rather than at their initial values. + /// + /// The text is not a JSON object. + /// The project was written by a newer Titano. public static ProjectSettings FromJson(string json) { - var settings = JsonSerializer.Deserialize(json, ProjectJson.Default.ProjectSettings) ?? new ProjectSettings(); + var options = new JsonNodeOptions { PropertyNameCaseInsensitive = true }; + var merged = JsonNode.Parse(new ProjectSettings().ToJson(), options)!.AsObject(); + switch (JsonNode.Parse(json, options)) + { + case JsonObject given: + Overlay(merged, given); + break; + case not null: + throw new JsonException("A project must be a JSON object."); + } + var settings = JsonSerializer.Deserialize(merged, ProjectJson.Default.ProjectSettings) ?? new ProjectSettings(); if (settings.Version > CurrentVersion) throw new InvalidDataException($"project.json version {settings.Version} is newer than this Titano ({CurrentVersion})."); return settings; } + + /// Copies into , merging nested objects; a null never replaces an object. + private static void Overlay(JsonObject target, JsonObject source) + { + foreach (var (name, value) in source) + { + if (value is JsonObject child && target[name] is JsonObject existing) Overlay(existing, child); + else if (value is not null || target[name] is not JsonObject) target[name] = value?.DeepClone(); + } + } } [JsonSourceGenerationOptions( diff --git a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs new file mode 100644 index 00000000..26a35203 --- /dev/null +++ b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs @@ -0,0 +1,53 @@ +using System.Globalization; +using Titano.Core.Analysis.Deep; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Processing.Jobs; + +/// +/// and over the target sequences, one after the +/// other. Progress counts sequences (light) or frames (deep); checkpoint: the number of sequences done. A deep pass +/// interrupted inside a sequence resumes from that sequence's analysis.partial. +/// +internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser analyser, JobKind kind) : IJobHandler +{ + public JobKind Kind => kind; + + public async Task RunAsync(JobContext context) + { + var deep = kind == JobKind.DeepAnalyse; + var targets = context.Job.TargetIds; + var first = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var next) ? next : 0; + // Deep progress is in frames across all targets, so the bar and the ETA cover the whole job. + var frames = new long[targets.Count]; + if (deep) + { + for (var i = 0; i < targets.Count; i++) + frames[i] = (await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false))?.FrameCount ?? 0; + } + var total = deep ? frames.Sum() : targets.Count; + for (var i = first; i < targets.Count; i++) + { + context.Cancellation.ThrowIfCancellationRequested(); + var before = deep ? frames[..i].Sum() : i; + var chain = new ProgressChain(context.ReportAsync); + var progress = deep ? chain.For(p => (before + p.Done, total, null)) : null; + var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false); + await chain.FlushAsync().ConfigureAwait(false); + if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i])); + else Report(context, outcome, deep); + await context.CheckpointAsync((i + 1).ToString(CultureInfo.InvariantCulture), deep ? before + frames[i] : i + 1, total).ConfigureAwait(false); + } + } + + private static void Report(JobContext context, AnalysisOutcome outcome, bool deep) + { + var title = outcome.Sequence.DisplayTitle; + context.Log(Loc.L(deep ? "Advisor.Log.DeepDone" : "Advisor.Log.LightDone", title, outcome.Frames, outcome.Warnings)); + if (outcome.SettingsChanged) context.Log(Loc.L("Advisor.Log.AutoUpdated", title)); + if (outcome.SettingsUnreadable) context.Log(Loc.L("Advisor.Log.ProjectUnreadable", title)); + if (outcome.Presets.Count > 0) context.Log(Loc.L("Advisor.Log.Presets", string.Join(" · ", outcome.Presets.Select(p => p.DisplayName())))); + } +} diff --git a/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs b/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs new file mode 100644 index 00000000..515ea14c --- /dev/null +++ b/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs @@ -0,0 +1,111 @@ +using Titano.Core.Archiving; +using Titano.Core.Export; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Core.Project; +using Titano.Core.Render; +using Titano.Data.Jobs; + +namespace Titano.Data.Processing.Jobs; + +/// +/// on this machine (BRIEF §11.5): target = the sequence; parameters = Protocol +/// RenderJobParameters whose preset JSON is the export section of a project (preset, overrides, name template, +/// folder, batch). The project is the embedded one or the sequence's own; frames flagged missing, corrupt, excluded or +/// undecodable are left out. The preset and every batch preset render after one preparation pass into the sequence's +/// exports/ (or the export folder), named by the template, and each finished output is recorded at once. +/// Checkpoint: the batch entries recorded and the output being written, so a crashed attempt's leftovers are removed and +/// only unfinished entries render again. A failed entry fails the job after the others ran; retrying renders it alone. +/// +internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions options) : IJobHandler +{ + public JobKind Kind => JobKind.Render; + + public async Task RunAsync(JobContext context) + { + var cancellation = context.Cancellation; + var (parameters, export) = ProcessingJobs.ReadRender(context.Job.ParametersJson); + if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, cancellation).ConfigureAwait(false) is not { } sequence + || await db.Archives.GetAsync(sequence.ArchiveId, cancellation).ConfigureAwait(false) is not { } archive) + throw new InvalidOperationException(Loc.L("Errors.Render.Sequence")); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var folder = paths.Sequence(sequence.Folder); + var project = (parameters.SettingsJson is { } embedded ? ProjectStore.Parse(embedded) : ProjectStore.Read(sequence, folder)) with { Export = export }; + var settingsJson = project.ToJson(); + + var resume = RenderCheckpoint.Parse(context.Checkpoint); + var done = resume.Done.ToList(); + if (resume is { Pending: { } pending, PendingOutput: { } output }) + { + // A crash after the render row was written but before the checkpoint: that entry is finished. + var recorded = await db.Renders.ListBySequenceAsync(sequence.Id, cancellation).ConfigureAwait(false); + if (recorded.Any(r => r.OutputPath == Stored(paths, output))) done.Add(pending); + else Discard(output); + } + var targets = Targets(export, parameters.PresetName).Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).ToList(); + if (targets.Count == 0) return; + + var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, cancellation).ConfigureAwait(false); + IReadOnlyList outcomes; + await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames.Count), options)) + { + outcomes = await render.RenderAllAsync( + targets, + (target, setup) => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup)), + (index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total), + async (index, target, result) => + { + // The output is complete: record it even if the job is being cancelled, so it is never orphaned. + var entity = result.ToEntity(sequence.Id, target.PresetName, settingsJson, options.ProducedBy) with { OutputPath = Stored(paths, result.OutputPath) }; + await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false); + done.Add(index); + await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false); + context.Log(Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath))); + foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark); + }, + context.ReportAsync, + cancellation).ConfigureAwait(false); + } + var failed = outcomes.Where(o => o.Error is not null).ToList(); + foreach (var outcome in failed) context.Log(Loc.L("Export.Log.Failed", outcome.Target.PresetName, outcome.Error)); + if (failed.Count > 0) + throw new InvalidOperationException(failed.Count == 1 ? failed[0].Error : Loc.L("Errors.Render.Failed", failed.Count, outcomes.Count)); + } + + /// The job's preset, then every other preset of the batch with its own values. + internal static List Targets(ExportSettings export, string presetName) + { + var main = ExportResolver.Resolve(export); + var targets = new List { new(main, presetName) }; + foreach (var id in export.Batch.Distinct(StringComparer.OrdinalIgnoreCase)) + { + if (id.Equals(main.BasePresetId, StringComparison.OrdinalIgnoreCase)) continue; + var resolved = ExportResolver.Resolve(new ExportSettings { Preset = new Auto().WithManual(id) }); + targets.Add(new ExportTarget(resolved, ProcessingJobs.PresetName(resolved))); + } + return targets; + } + + /// Archive-relative for outputs inside the archive (like the server's renders), absolute otherwise. + private static string Stored(ArchivePaths paths, string output) + { + var relative = paths.ToRelative(output); + return relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative) ? output : relative; + } + + /// + /// Removes what an interrupted attempt left of : the encoder's hidden temporary file or + /// folder, and an output that was completed but never recorded (it renders again under the same name). + /// + private static void Discard(string output) + { + var parent = Path.GetDirectoryName(output); + if (parent is null || !Directory.Exists(parent)) return; + foreach (var leftover in Directory.EnumerateFileSystemEntries(parent, "." + Path.GetFileName(output) + ".*.tmp").Append(output)) + { + if (Directory.Exists(leftover)) Directory.Delete(leftover, recursive: true); + else if (File.Exists(leftover)) File.Delete(leftover); + } + } +} diff --git a/src/Titano.Data/Processing/ProcessingJobs.cs b/src/Titano.Data/Processing/ProcessingJobs.cs new file mode 100644 index 00000000..2dac8db8 --- /dev/null +++ b/src/Titano.Data/Processing/ProcessingJobs.cs @@ -0,0 +1,122 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Titano.Core.Export; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Core.Project; +using Titano.Core.Video; +using Titano.Data.Archiving.Jobs; +using Titano.Protocol; + +namespace Titano.Data.Processing; + +/// +/// Builds the analysis and render jobs; enqueue them with (local +/// engine) or post the same kind, targets and parameters to /api/v1/jobs (server). +/// +public static class ProcessingJobs +{ + /// : targets = sequences; no parameters. + public static ArchiveJobRequest LightAnalysis(IReadOnlyList sequenceIds, string? label = null) => + new(JobKind.LightAnalyse, sequenceIds, label, null, JobPriority.Background); + + /// : targets = sequences; no parameters. + public static ArchiveJobRequest DeepAnalysis(IReadOnlyList sequenceIds, string? label = null) => + new(JobKind.DeepAnalyse, sequenceIds, label, null, JobPriority.User); + + /// + /// of one sequence. (preset, overrides, name template, + /// folder, batch) replaces the project's export section; embeds the settings to render, + /// null renders the sequence's current project. × + /// stands in for a preset without a fixed size, so render nodes can match the job by resolution. + /// + public static ArchiveJobRequest Render(Sequence sequence, ExportSettings export, int sourceWidth, int sourceHeight, ProjectSettings? project = null, + bool useProxies = false) + { + ArgumentNullException.ThrowIfNull(sequence); + ArgumentNullException.ThrowIfNull(export); + var resolved = ExportResolver.Resolve(export); + var name = PresetName(resolved); + var parameters = new RenderJobParameters(name, JsonSerializer.Serialize(export, ProcessingJson.Default.ExportSettings), CodecName(resolved.Codec), + resolved.Width > 0 ? resolved.Width : sourceWidth, resolved.Height > 0 ? resolved.Height : sourceHeight, useProxies, project?.ToJson()); + return new ArchiveJobRequest(JobKind.Render, [sequence.Id], $"{sequence.DisplayTitle} → {name}", + JsonSerializer.Serialize(parameters, ProtocolJson.Default.RenderJobParameters), JobPriority.User); + } + + /// Codec name in job parameters and render-node capabilities ("h264", "hevc", "mjpeg", "pngsequence"…). + public static string CodecName(VideoCodec codec) => codec.ToString().ToLowerInvariant(); + + /// Display name of an export's preset in the current language ("Custom" when the values match none). + public static string PresetName(ResolvedExport export) + { + ArgumentNullException.ThrowIfNull(export); + return Loc.L(export.NameKey ?? "Export.Preset.Custom"); + } + + /// The parameters of a render job and the export they select. + /// The job has no valid render parameters. + internal static (RenderJobParameters Parameters, ExportSettings Export) ReadRender(string? parametersJson) + { + try + { + if (parametersJson is { Length: > 0 } && JsonSerializer.Deserialize(parametersJson, ProtocolJson.Default.RenderJobParameters) is { } parameters) + return (parameters, ReadExport(parameters.PresetJson)); + } + catch (JsonException ex) + { + throw new InvalidDataException(Loc.L("Errors.Render.Parameters"), ex); + } + throw new InvalidDataException(Loc.L("Errors.Render.Parameters")); + } + + /// The export section in a render job's preset JSON; values it leaves out take their defaults. + /// The JSON is not an export section. + internal static ExportSettings ReadExport(string? presetJson) + { + try + { + // Read as the export section of a project, so missing values keep their defaults (ProjectSettings.FromJson). + var project = new JsonObject { ["export"] = JsonNode.Parse(presetJson ?? string.Empty) }; + return ProjectSettings.FromJson(project.ToJsonString()).Export; + } + catch (JsonException ex) + { + throw new InvalidDataException(Loc.L("Errors.Render.Parameters"), ex); + } + } +} + +/// +/// Resume point of a render job: the batch targets already recorded, and the output a crashed attempt was writing +/// (its leftovers are removed before that target renders again). +/// +internal sealed record RenderCheckpoint(IReadOnlyList Done, int? Pending, string? PendingOutput) +{ + public static RenderCheckpoint Empty { get; } = new([], null, null); + + public static RenderCheckpoint Parse(string? json) + { + if (string.IsNullOrEmpty(json)) return Empty; + try + { + return JsonSerializer.Deserialize(json, ProcessingJson.Default.RenderCheckpoint) ?? Empty; + } + catch (JsonException) + { + return Empty; + } + } + + public string ToJson() => JsonSerializer.Serialize(this, ProcessingJson.Default.RenderCheckpoint); +} + +/// JSON written into render jobs: presets (the export section of project.json) and checkpoints. +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + UseStringEnumConverter = true, + PropertyNameCaseInsensitive = true)] +[JsonSerializable(typeof(ExportSettings))] +[JsonSerializable(typeof(RenderCheckpoint))] +internal sealed partial class ProcessingJson : JsonSerializerContext; diff --git a/src/Titano.Data/Processing/ProcessingOptions.cs b/src/Titano.Data/Processing/ProcessingOptions.cs new file mode 100644 index 00000000..b6cfcf78 --- /dev/null +++ b/src/Titano.Data/Processing/ProcessingOptions.cs @@ -0,0 +1,43 @@ +using Titano.Core.Analysis.Deep; +using Titano.Core.Export; +using Titano.Core.Imaging; +using Titano.Core.Render; +using Titano.Core.Video; +using Titano.Core.Video.Managed; +using Titano.Data.Archiving; + +namespace Titano.Data.Processing; + +/// +/// What differs between the hosts that analyse and render (BRIEF §4.2): decoders (inside ), +/// encoders and where the work runs. Client: WIC then managed decoders, Media Foundation then managed encoders; +/// server: managed only. +/// +public sealed record ProcessingOptions +{ + /// Decoders, lease owner and lease timing for the sidecar and project writes. + public ArchiveServiceOptions Archive { get; init; } = new(); + + /// Encoder providers in priority order. + public IReadOnlyList Encoders { get; init; } = [new ManagedEncoderProvider()]; + + /// The server delegates platform codecs to render nodes and never runs them itself. + public EncoderHost Host { get; init; } = EncoderHost.Client; + + /// Recorded with every render this host produces (client or server name). + public string ProducedBy { get; init; } = Environment.MachineName; + + public DeepAnalysisOptions DeepAnalysis { get; init; } = DeepAnalysisOptions.Default; + + /// Read-ahead and memory budget of the render decode window. + public FrameWindowOptions FrameWindow { get; init; } = new(); + + public RenderOptions Render { get; init; } = RenderOptions.Default; + + /// + /// Encoders the advisor may count on: this machine's on a client; none listed on the server (render nodes can + /// produce any codec, so the advisor rules none out). + /// + internal IReadOnlyList AdvisorCodecs => + Host == EncoderHost.Server ? [] : [.. Encoders.SelectMany(e => e.Capabilities)]; +} diff --git a/src/Titano.Data/Processing/ProcessingService.cs b/src/Titano.Data/Processing/ProcessingService.cs new file mode 100644 index 00000000..3a06bd42 --- /dev/null +++ b/src/Titano.Data/Processing/ProcessingService.cs @@ -0,0 +1,37 @@ +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Jobs; +using Titano.Data.Processing.Jobs; + +namespace Titano.Data.Processing; + +/// +/// The analysis and render engine of a host (BRIEF §4.3, §10, §11): job handlers for light analysis, deep analysis and +/// renders over the archive in the index, built with the host's decoders and encoders (). +/// Register it on the next to the archive engine; a server engine leaves render jobs to render +/// nodes. Create jobs with . +/// +public sealed class ProcessingService : IJobHandlerProvider +{ + private readonly TitanoDatabase _db; + private readonly SequenceAnalyser _analyser; + + public ProcessingService(TitanoDatabase db, ProcessingOptions? options = null) + { + _db = db; + Options = options ?? new ProcessingOptions(); + // Sidecar and project writes coordinate with the archive engine through the archive's lease files. + var writer = new SequenceWriter(Options.Archive); + _analyser = new SequenceAnalyser(db, Options, writer, new ArchiveIndexer(db, writer), new ProjectStore(db, writer)); + } + + public ProcessingOptions Options { get; } + + /// Handlers for , and . + public IReadOnlyList CreateJobHandlers() => + [ + new AnalyseJobHandler(_db, _analyser, JobKind.LightAnalyse), + new AnalyseJobHandler(_db, _analyser, JobKind.DeepAnalyse), + new RenderJobHandler(_db, Options), + ]; +} diff --git a/src/Titano.Data/Processing/ProgressChain.cs b/src/Titano.Data/Processing/ProgressChain.cs new file mode 100644 index 00000000..c3d8ef9c --- /dev/null +++ b/src/Titano.Data/Processing/ProgressChain.cs @@ -0,0 +1,40 @@ +namespace Titano.Data.Processing; + +/// +/// Delivers the synchronous progress reports of the engine (analysis, render stages) to an asynchronous sink (job +/// context, render-node protocol) one after the other, in report order. Await before a +/// checkpoint or the end of the work to observe the deliveries. +/// +internal sealed class ProgressChain(Func sink) +{ + private readonly Lock _gate = new(); + private Task _pending = Task.CompletedTask; + + /// A progress receiver mapping its reports to done, total and message. + public IProgress For(Func map) => new Forward(this, map); + + public Task FlushAsync() + { + lock (_gate) return _pending; + } + + private void Post(long done, long total, string? message) + { + lock (_gate) _pending = AfterAsync(_pending, done, total, message); + } + + private async Task AfterAsync(Task previous, long done, long total, string? message) + { + await previous.ConfigureAwait(false); + await sink(done, total, message).ConfigureAwait(false); + } + + private sealed class Forward(ProgressChain chain, Func map) : IProgress + { + public void Report(T value) + { + var (done, total, message) = map(value); + chain.Post(done, total, message); + } + } +} diff --git a/src/Titano.Data/Processing/ProjectStore.cs b/src/Titano.Data/Processing/ProjectStore.cs new file mode 100644 index 00000000..046d94f0 --- /dev/null +++ b/src/Titano.Data/Processing/ProjectStore.cs @@ -0,0 +1,61 @@ +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Core.IO; +using Titano.Core.Model; +using Titano.Core.Project; +using Titano.Data.Archiving; +using Titano.Data.Repositories; + +namespace Titano.Data.Processing; + +/// +/// The project of a sequence (BRIEF §7 project.json), mirrored in the index's settings column that the +/// server's API edits. The index copy is read first; writes go to both, under the sequence lock and with the index's +/// settings version as an optimistic check, so a concurrent edit is applied on top of, never overwritten. +/// +internal sealed class ProjectStore(TitanoDatabase db, SequenceWriter writer) +{ + private const int Attempts = 3; + + /// The settings in force: the index copy, else project.json, else defaults. + /// The settings are not valid JSON or come from a newer Titano. + public static ProjectSettings Read(Sequence sequence, SequencePaths folder) => + Parse(sequence.SettingsJson ?? (File.Exists(folder.ProjectPath) ? File.ReadAllText(folder.ProjectPath) : null)); + + /// The settings are not valid JSON or come from a newer Titano. + public static ProjectSettings Parse(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return new ProjectSettings(); + try + { + return ProjectSettings.FromJson(json); + } + catch (JsonException ex) + { + throw new InvalidDataException(ex.Message, ex); + } + } + + /// Applies to the settings in force and stores the result when it differs; false when nothing changed. + /// The stored settings cannot be read; nothing is written. + public async Task UpdateAsync(ArchivePaths paths, Sequence sequence, Func edit, CancellationToken cancellationToken) + { + var folder = paths.Sequence(sequence.Folder); + await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) + { + for (var attempt = 1; ; attempt++) + { + var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence; + var current = Read(row, folder); + var before = current.ToJson(); + var after = edit(current).ToJson(); + if (after == before) return false; + AtomicFile.WriteAllText(folder.ProjectPath, after); + var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after }, row.SettingsVersion, cancellationToken) + .ConfigureAwait(false); + // A conflict means the API changed the settings meanwhile: apply the edit again on top of them. + if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true; + } + } + } +} diff --git a/src/Titano.Data/Processing/RenderNodeClient.cs b/src/Titano.Data/Processing/RenderNodeClient.cs new file mode 100644 index 00000000..8361caab --- /dev/null +++ b/src/Titano.Data/Processing/RenderNodeClient.cs @@ -0,0 +1,455 @@ +using System.Diagnostics; +using System.Net; +using System.Text.Json; +using Titano.Core.Diagnostics; +using Titano.Core.Export; +using Titano.Core.Imaging; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Core.Render; +using Titano.Core.Video; +using Titano.Protocol; +using Titano.Protocol.Client; + +namespace Titano.Data.Processing; + +/// Settings of a . +public sealed record RenderNodeOptions +{ + /// Name the server lists the node under; one node per user and name. + public required string ClientName { get; init; } + + /// Folder for downloaded originals and for outputs until they are uploaded (inside the client's cache). + public required string CacheDirectory { get; init; } + + /// Downloaded originals beyond this size are deleted, least recently used first, after every job. + public long CacheLimitBytes { get; init; } = 20L << 30; + + /// Wait between claims while the queue has nothing for this node. + public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(10); + + /// Sign of life; the server gives the jobs of a node silent for 90 s to others. + public TimeSpan HeartbeatInterval { get; init; } = TimeSpan.FromSeconds(30); + + /// Shortest interval between progress reports to the server. + public TimeSpan ProgressInterval { get; init; } = TimeSpan.FromSeconds(2); + + /// Concurrent frame downloads. + public int Downloads { get; init; } = 4; +} + +/// +/// "Offer this PC as render node" (BRIEF §14.7): registers this machine's single-file encoders with the server, +/// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals +/// hash-verified into a content-addressed cache, or proxies when the job asks), renders the job's preset with the +/// host's encoders and uploads the output, which the server files as a render of the sequence. Progress goes to the +/// server; a job cancelled or reassigned there stops here; a failure is reported with its reason. Stopping abandons +/// the job in flight without reporting it, so the server requeues it once the heartbeats stop. Image sequences are +/// not offered: the protocol uploads one file per job. +/// +public sealed class RenderNodeClient : IAsyncDisposable +{ + private const string LogCategory = "render-node"; + + private readonly TitanoClient _client; + private readonly ProcessingOptions _processing; + private readonly RenderNodeOptions _options; + private readonly CancellationTokenSource _stopping = new(); + private Task _running = Task.CompletedTask; + private Guid? _nodeId; + + public RenderNodeClient(TitanoClient client, ProcessingOptions processing, RenderNodeOptions options) + { + _client = client; + _processing = processing; + _options = options; + } + + /// The encoders this node offers: every single-file capability of the host's providers. + public IReadOnlyList Encoders => + [ + .. _processing.Encoders.SelectMany(p => p.Capabilities) + .Where(c => VideoCodecInfo.DefaultContainer(c.Codec) != ContainerFormat.ImageSequence) + .Select(c => new NodeEncoder(ProcessingJobs.CodecName(c.Codec), c.Name, c.Hardware, c.SupportsTenBit)), + ]; + + /// Starts registering, heart-beating and claiming in the background. + public void Start() => _running = Task.Run(() => RunAsync(_stopping.Token)); + + /// Stops claiming, abandons the job in flight and waits for the background work to end. + public async Task StopAsync() + { + await _stopping.CancelAsync().ConfigureAwait(false); + await _running.ConfigureAwait(false); + } + + public async ValueTask DisposeAsync() + { + await StopAsync().ConfigureAwait(false); + _stopping.Dispose(); + } + + /// Registers this node (again) and returns its id. + public async Task RegisterAsync(CancellationToken cancellationToken) + { + var capabilities = _processing.Encoders.SelectMany(p => p.Capabilities).ToList(); + var request = new RegisterNodeRequest(_options.ClientName, Encoders, capabilities.Count > 0 ? capabilities.Max(c => c.MaxWidth) : 0, + capabilities.Count > 0 ? capabilities.Max(c => c.MaxHeight) : 0); + var node = await _client.RegisterNodeAsync(request, cancellationToken).ConfigureAwait(false); + _nodeId = node.Id; + return node.Id; + } + + /// Claims one job this node can encode and runs it to the end; false when the queue had none. + public async Task RunOnceAsync(CancellationToken cancellationToken) + { + var nodeId = _nodeId ?? await RegisterAsync(cancellationToken).ConfigureAwait(false); + if (await _client.ClaimAsync(nodeId, cancellationToken).ConfigureAwait(false) is not { } claimed) return false; + var work = Path.Combine(_options.CacheDirectory, "jobs", claimed.Job.Id.ToString("N")); + using var job = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var reporter = new NodeReporter(_client, nodeId, claimed.Job.Id, _options.ProgressInterval, job); + try + { + Directory.CreateDirectory(work); + await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false); + } + catch (Exception ex) when (job.IsCancellationRequested) + { + // Stopping, or the server cancelled or reassigned the job: either way it is no longer ours to report. + Log.Info(LogCategory, $"Render job {claimed.Job.Id} abandoned: {ex.Message}"); + } + catch (Exception ex) + { + // Whatever went wrong belongs to this job: report it and stay available for the next one. + Log.Warning(LogCategory, $"Render job {claimed.Job.Id} failed", ex); + await ReportFailureAsync(nodeId, claimed.Job.Id, SequenceRender.Reason(ex), cancellationToken).ConfigureAwait(false); + } + finally + { + TryDelete(work); + TrimCache(); + } + return true; + } + + private async Task RunAsync(CancellationToken stopping) + { + using var heartbeat = CancellationTokenSource.CreateLinkedTokenSource(stopping); + Task? beating = null; + while (!stopping.IsCancellationRequested) + { + try + { + var nodeId = _nodeId ?? await RegisterAsync(stopping).ConfigureAwait(false); + beating ??= HeartbeatAsync(heartbeat.Token); + if (await RunOnceAsync(stopping).ConfigureAwait(false)) continue; + } + catch (OperationCanceledException) when (stopping.IsCancellationRequested) + { + break; + } + catch (TitanoApiException ex) when (ex.Status is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden) + { + // Signed out, or the account lacks the render-node right: nothing to retry until the user acts. + Log.Warning(LogCategory, "The server refused this render node", ex); + break; + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + _nodeId = null; // removed on the server: register again + continue; + } + catch (Exception ex) + { + // Server unreachable or busy: the node keeps trying until it is stopped. + Log.Warning(LogCategory, "Render node cycle failed; retrying", ex); + } + try + { + await Task.Delay(_options.PollInterval, stopping).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + } + await heartbeat.CancelAsync().ConfigureAwait(false); + if (beating is not null) await beating.ConfigureAwait(false); + } + + private async Task HeartbeatAsync(CancellationToken cancellationToken) + { + using var timer = new PeriodicTimer(_options.HeartbeatInterval); + try + { + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) + { + if (_nodeId is not { } nodeId) continue; + try + { + await _client.HeartbeatAsync(nodeId, cancellationToken).ConfigureAwait(false); + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + _nodeId = null; + } + catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException or UntrustedCertificateException) + { + Log.Warning(LogCategory, "Heartbeat failed", ex); + } + } + } + catch (OperationCanceledException) + { + } + } + + private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken) + { + var export = ProcessingJobs.ReadExport(claimed.PresetJson); + var resolved = ExportResolver.Resolve(export); + var target = new ExportTarget(resolved, ProcessingJobs.PresetName(resolved)); + if (target.Export.Container == ContainerFormat.ImageSequence) throw new InvalidOperationException(Loc.L("Errors.Render.NodeSequence")); + var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export }; + + var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false)); + var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false); + var files = await DownloadAsync(claimed, work, reporter, cancellationToken).ConfigureAwait(false); + var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id))]); + + IReadOnlyList outcomes; + await using (var render = new SequenceRender(input, project, null, _processing)) + { + outcomes = await render.RenderAllAsync( + [(0, target)], + (t, setup) => ExportNaming.OutputPath(work, project.Export.NameTemplate, SequenceRender.NameContext(sequence, t, setup)), + (_, _) => ValueTask.CompletedTask, + (_, t, result) => new ValueTask(UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken)), + reporter.ReportAsync, + cancellationToken).ConfigureAwait(false); + } + if (outcomes[0].Error is { } error) throw new InvalidOperationException(error); + } + + private async Task UploadAsync(Guid nodeId, Guid jobId, ExportTarget target, RenderResult result, CancellationToken cancellationToken) + { + var size = new FileInfo(result.OutputPath).Length; + var sha = await TitanoClient.HashFileAsync(result.OutputPath, cancellationToken).ConfigureAwait(false); + var request = new RenderResultRequest(Path.GetFileName(result.OutputPath), size, sha, target.PresetName, ProcessingJobs.CodecName(result.Codec), + result.Width, result.Height, result.FrameRate.Value, result.DurationSeconds); + await _client.UploadRenderResultAsync(nodeId, jobId, result.OutputPath, request, null, cancellationToken).ConfigureAwait(false); + } + + private async Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken) + { + var frames = new List(); + string? cursor = null; + do + { + var page = await _client.ListFramesAsync(sequenceId, cursor, 1000, cancellationToken).ConfigureAwait(false); + frames.AddRange(page.Items.Select(ToFrame)); + cursor = page.NextCursor; + } + while (cursor is not null); + frames.Sort((a, b) => a.Index.CompareTo(b.Index)); + return frames; + } + + /// + /// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused + /// across jobs), or proxies into the job folder when the job asks for them and the server has them. + /// + private async Task> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken) + { + var files = new Dictionary(); + var done = 0; + await reporter.ReportAsync(0, claimed.Frames.Count, Loc.L("Export.Job.Downloading")).ConfigureAwait(false); + await Parallel.ForEachAsync(claimed.Frames, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken }, + async (frame, ct) => + { + var path = claimed.UseProxies && frame.HasProxy + ? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false) + : await DownloadOriginalAsync(frame, ct).ConfigureAwait(false); + lock (files) files[frame.Id] = path; + await reporter.ReportAsync(Interlocked.Increment(ref done), claimed.Frames.Count, null).ConfigureAwait(false); + }).ConfigureAwait(false); + return files; + } + + /// A proxy into the job folder (regenerable, so not cached); the original when the server has none for the frame. + private async Task DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) + { + var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg"); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + try + { + await using (var file = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) + await _client.DownloadFrameAsync(frame.Id, "proxy", file, null, cancellationToken).ConfigureAwait(false); + return path; + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + File.Delete(path); + return await DownloadOriginalAsync(frame, cancellationToken).ConfigureAwait(false); + } + } + + /// A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256. + private async Task DownloadOriginalAsync(ManifestFrame frame, CancellationToken cancellationToken) + { + var sha = frame.Sha256.ToLowerInvariant(); + if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName)); + var path = Path.Combine(_options.CacheDirectory, "originals", sha[..2], sha + Path.GetExtension(frame.FileName).ToLowerInvariant()); + if (File.Exists(path)) + { + File.SetLastWriteTimeUtc(path, DateTime.UtcNow); // most recently used: kept by the cache trim + return path; + } + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var part = path + ".part"; + for (var attempt = 1; ; attempt++) + { + await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + await _client.DownloadFrameAsync(frame.Id, "original", file, null, cancellationToken).ConfigureAwait(false); + if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha) + { + File.Move(part, path, overwrite: true); + return path; + } + File.Delete(part); + if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName)); + } + } + + private async Task ReportFailureAsync(Guid nodeId, Guid jobId, string error, CancellationToken cancellationToken) + { + try + { + await _client.ReportNodeFailureAsync(nodeId, jobId, error, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException or OperationCanceledException) + { + // Cancelled or reassigned meanwhile, or the server is gone: it requeues the job when the heartbeats stop. + Log.Warning(LogCategory, $"The failure of job {jobId} could not be reported", ex); + } + } + + /// Deletes the least recently used originals until the cache fits its limit. + private void TrimCache() + { + try + { + var root = new DirectoryInfo(Path.Combine(_options.CacheDirectory, "originals")); + if (!root.Exists) return; + var files = root.EnumerateFiles("*", SearchOption.AllDirectories).OrderBy(f => f.LastWriteTimeUtc).ToList(); + var size = files.Sum(f => f.Length); + foreach (var file in files) + { + if (size <= _options.CacheLimitBytes) break; + size -= file.Length; + file.Delete(); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning(LogCategory, "The render node cache could not be trimmed", ex); + } + } + + private static void TryDelete(string folder) + { + try + { + if (Directory.Exists(folder)) Directory.Delete(folder, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning(LogCategory, $"Could not remove {folder}", ex); + } + } + + private static Sequence ToSequence(SequenceDto s) => new() + { + Id = s.Id, + ArchiveId = Guid.Empty, + Folder = s.Folder, + Title = s.Title, + CameraMake = s.CameraMake, + CameraModel = s.CameraModel, + Lens = s.Lens, + FirstCapture = s.FirstCapture, + LastCapture = s.LastCapture, + FrameCount = s.FrameCount, + }; + + private static Frame ToFrame(FrameDto f) => new() + { + Id = f.Id, + SequenceId = f.SequenceId, + Index = f.Index, + RelativePath = f.FileName, + FileName = f.FileName, + Size = f.Size, + Sha256 = f.Sha256, + CapturedAt = f.CapturedAt, + UtcOffsetMinutes = f.UtcOffsetMinutes, + ExposureTime = f.ExposureTime, + FNumber = f.FNumber, + Iso = f.Iso, + FocalLength = f.FocalLength, + Orientation = f.Orientation, + Width = f.Width, + Height = f.Height, + Format = Enum.TryParse(f.Format, ignoreCase: true, out var format) ? format : default, + Flags = (FrameFlags)f.Flags, + }; + + /// + /// Progress of the job in flight to the server, at most every interval (and at the end). A refusal (409) means the + /// job was cancelled or given to another node: is set and the job's token cancelled. + /// + private sealed class NodeReporter(TitanoClient client, Guid nodeId, Guid jobId, TimeSpan interval, CancellationTokenSource job) + { + private readonly Stopwatch _clock = Stopwatch.StartNew(); + private readonly SemaphoreSlim _gate = new(1, 1); + private TimeSpan _last = TimeSpan.MinValue; + private long _lastDone; + private string? _lastMessage; + + public Guid NodeId => nodeId; + + public bool Lost { get; private set; } + + public async ValueTask ReportAsync(long done, long total, string? message) + { + await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false); + try + { + var now = _clock.Elapsed; + var line = message is not null && message != _lastMessage ? message : null; + if (line is null && done < total && now - _last < interval) return; + double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3); + (_last, _lastDone) = (now, done); + if (line is not null) _lastMessage = line; + await client.ReportNodeProgressAsync(nodeId, jobId, new NodeProgress(done, total, rate, line), job.Token).ConfigureAwait(false); + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict) + { + Lost = true; + await job.CancelAsync().ConfigureAwait(false); + } + catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException) + { + Log.Warning(LogCategory, "Progress could not be reported", ex); + } + catch (OperationCanceledException) + { + // The job is stopping; its outcome is handled by the caller. + } + finally + { + _gate.Release(); + } + } + } +} diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs new file mode 100644 index 00000000..ef663e28 --- /dev/null +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -0,0 +1,131 @@ +using System.Security.Cryptography; +using System.Text; +using Titano.Core.Advice; +using Titano.Core.Analysis; +using Titano.Core.Analysis.Deep; +using Titano.Core.Archiving; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Repositories; + +namespace Titano.Data.Processing; + +/// What an analysis of one sequence found and changed. +/// Findings of warning severity or worse (light and, when current, deep). +/// The advisor changed Auto values of the project. +/// The project could not be read, so its Auto values were left alone. +/// Sequence presets that fit the content (deep analysis only). +internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable, + IReadOnlyList Presets); + +/// +/// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the +/// Auto values of the project. The light pass reads the index only (plus one file for the GPS fix); the deep pass decodes +/// every frame, checkpointing into .titano/analysis.partial so a cancelled, paused or crashed run resumes, and +/// saves analysis.json atomically. Results land in the sidecar status, the index (status, warning count, compact +/// summary, per-frame light statistics) and project.json. +/// +internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions options, SequenceWriter writer, ArchiveIndexer indexer, ProjectStore projects) +{ + /// Analyses one sequence; null when it is no longer in the index. + /// The archive is offline or read-only. + public async Task AnalyseAsync(Guid sequenceId, bool deep, IProgress? progress, CancellationToken cancellationToken) + { + if (await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } sequence + || await db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive) return null; + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var folder = paths.Sequence(sequence.Folder); + var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, cancellationToken).ConfigureAwait(false); + var metadata = frames.LightFrames(); + var light = LightAnalyzer.Analyze(metadata); + + var report = deep + ? await AnalyseDeepAsync(frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false) + : ReadReport(folder, frames.Count); + var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs); + var advice = Advisor.Recommend(light, report, context); + var warnings = advice.Findings.Count(f => f.Severity >= FindingSeverity.Warning); + + bool changed, unreadable = false; + try + { + changed = await projects.UpdateAsync(paths, sequence, settings => Advisor.Apply(settings, advice), cancellationToken).ConfigureAwait(false); + } + catch (InvalidDataException ex) + { + Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex); + (changed, unreadable) = (false, true); + } + + var set = SequenceStatus.AnalyzedLight | (report is null ? SequenceStatus.None : SequenceStatus.AnalyzedDeep) + | (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None); + // Without a current report (none, or made before frames were added) the deep status no longer holds. + var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings); + var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null; + await writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => + { + var status = (m.Status | set) & ~clear; + return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; + }, cancellationToken).ConfigureAwait(false); + await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false); + await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); + + var presets = deep ? SequencePresets.Generate(light, report, context) : []; + return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets); + } + + /// The saved deep report when it describes the current frames; null when absent, stale or unreadable. + public static DeepAnalysisReport? ReadReport(SequencePaths folder, int frameCount) + { + if (!File.Exists(folder.AnalysisPath)) return null; + try + { + var report = DeepAnalysisReport.Load(folder.AnalysisPath); + return report.FrameCount == frameCount ? report : null; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or System.Text.Json.JsonException) + { + Core.Diagnostics.Log.Warning("processing", $"{folder.AnalysisPath} cannot be read", ex); + return null; + } + } + + private async Task AnalyseDeepAsync(SequenceFrames frames, IReadOnlyList metadata, SequencePaths folder, + IProgress? progress, CancellationToken cancellationToken) + { + var file = new DeepAnalysisCheckpointFile(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName)); + DeepAnalysisReport report; + await using (var source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder)) + { + report = await DeepAnalyzer.AnalyzeAsync(source, options.DeepAnalysis, new FramesCheckpoint(file, ContentKey(frames)), progress, cancellationToken) + .ConfigureAwait(false); + } + report.Save(folder.AnalysisPath); + file.Delete(); + + var columns = report.Frames; + var stats = new FrameLightStats[frames.Count]; + for (var i = 0; i < stats.Length; i++) + stats[i] = new FrameLightStats(frames.Frames[i].Id, columns.LogLuminance[i], columns.ClipLow[i], columns.ClipHigh[i]); + await db.Frames.UpdateLightStatsAsync(stats, cancellationToken).ConfigureAwait(false); + return report; + } + + /// Identity of the analysed content: frames replaced or renumbered since a checkpoint was written (with the + /// same count) make it stale, like changed options do. + private static string ContentKey(SequenceFrames frames) + { + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + foreach (var f in frames.Frames) hash.AppendData(Encoding.UTF8.GetBytes($"{f.Id:N}/{f.Sha256}/{f.Size}/{f.Flags & SequenceFrames.NotAnalysed};")); + return Convert.ToHexStringLower(hash.GetHashAndReset())[..16]; + } + + private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey) : IDeepAnalysisCheckpoint + { + public ValueTask> LoadAsync(string fingerprint, CancellationToken cancellationToken) => + file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken); + + public ValueTask AppendAsync(IReadOnlyList frames, CancellationToken cancellationToken) => file.AppendAsync(frames, cancellationToken); + } +} diff --git a/src/Titano.Data/Processing/SequenceFrameSource.cs b/src/Titano.Data/Processing/SequenceFrameSource.cs new file mode 100644 index 00000000..34f6a268 --- /dev/null +++ b/src/Titano.Data/Processing/SequenceFrameSource.cs @@ -0,0 +1,133 @@ +using System.Globalization; +using Titano.Core.Advice; +using Titano.Core.Analysis; +using Titano.Core.Analysis.Deep; +using Titano.Core.Imaging; +using Titano.Core.Render; + +namespace Titano.Data.Processing; + +/// +/// The frames of a decoded by the host's (oriented, linear +/// light, embedded previews when nothing else decodes). For the deep analysis a frame that is missing, corrupt or +/// unreadable is null, as expects. Renders exclude flagged frames from the plan, so a +/// frame that still cannot be decoded stops the render with ; render reads go through +/// a per decode size, which decodes ahead in index order and spills under memory pressure. +/// +internal sealed class SequenceFrameSource : IFrameSource, IRenderFrameSource, IAsyncDisposable +{ + /// Render failure reason: {0} frame number (from 1), {1} file name. + public const string FrameUnreadableKey = "Export.Render.FrameUnreadable"; + + private const int ProbeLongEdge = 64; + + private readonly SequenceFrames _frames; + private readonly IReadOnlyList _metadata; + private readonly ImageDecoder _decoder; + private readonly FrameWindowOptions _windowOptions; + private readonly Lazy<(int Width, int Height)> _size; + private readonly SemaphoreSlim _gate = new(1, 1); + private FrameWindow? _window; + private (int Edge, DecodePurpose Purpose) _windowKey; + + public SequenceFrameSource(SequenceFrames frames, IReadOnlyList metadata, ImageDecoder decoder, FrameWindowOptions? window = null) + { + if (metadata.Count != frames.Count) throw new ArgumentException("One metadata entry per frame is required.", nameof(metadata)); + _frames = frames; + _metadata = metadata; + _decoder = decoder; + _windowOptions = window ?? new FrameWindowOptions(); + _size = new Lazy<(int, int)>(MeasureSize); + } + + public int Count => _frames.Count; + + public int FrameCount => _frames.Count; + + public int Width => _size.Value.Width; + + public int Height => _size.Value.Height; + + public LightFrame Metadata(int index) => _metadata[index]; + + public ValueTask LoadAsync(int index, int maxLongEdge, CancellationToken cancellationToken) + { + if ((_frames.Frames[index].Flags & SequenceFrames.NotAnalysed) != 0) return ValueTask.FromResult(null); + return ValueTask.FromResult(Decode(index, maxLongEdge, DecodePurpose.Analysis, cancellationToken)?.Pixels); + } + + public async ValueTask GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) + { + // The pipeline reads one frame at a time; the gate keeps a window switch from disposing a window in use. + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_window is null || _windowKey != (maxLongEdge, purpose)) + { + if (_window is not null) await _window.DisposeAsync().ConfigureAwait(false); + _windowKey = (maxLongEdge, purpose); + _window = new FrameWindow(Count, (i, ct) => ValueTask.FromResult(DecodeForRender(i, maxLongEdge, purpose, ct)), _windowOptions); + } + _window.MoveTo(index); + using var lease = await _window.GetAsync(index, cancellationToken).ConfigureAwait(false); + return lease.Image.Clone(); + } + finally + { + _gate.Release(); + } + } + + public async ValueTask DisposeAsync() + { + await _gate.WaitAsync().ConfigureAwait(false); + try + { + if (_window is not null) await _window.DisposeAsync().ConfigureAwait(false); + _window = null; + } + finally + { + _gate.Release(); + } + } + + private ImageBuffer DecodeForRender(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) => + Decode(index, maxLongEdge, purpose, cancellationToken)?.Pixels ?? throw Unreadable(index); + + private DecodedImage? Decode(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) + { + if (_frames.Files[index] is not { } file) return null; + try + { + return _decoder.Decode(new DecodeRequest(file, maxLongEdge, purpose), cancellationToken); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Core.Diagnostics.Log.Warning("processing", $"Frame {file} could not be read", ex); + return null; + } + } + + /// The dominant oriented size from the metadata, else the size the first renderable frame decodes to. + private (int, int) MeasureSize() + { + var context = SequenceContext.FromFrames(_frames.Frames); + if (context.SourceWidth > 0 && context.SourceHeight > 0) return (context.SourceWidth, context.SourceHeight); + var skipped = _frames.NotRenderable().ToHashSet(); + for (var i = 0; i < Count; i++) + { + if (skipped.Contains(i)) continue; + using var probe = Decode(i, ProbeLongEdge, DecodePurpose.Thumbnail, CancellationToken.None) ?? throw Unreadable(i); + return (probe.SourceWidth, probe.SourceHeight); + } + throw new RenderException("Export.Render.NoFrames", "Every frame is excluded."); + } + + private RenderException Unreadable(int index) + { + var name = _frames.Frames[index].FileName; + return new RenderException(FrameUnreadableKey, $"Frame {index + 1} ({name}) could not be decoded.", + [(index + 1).ToString(CultureInfo.InvariantCulture), name]); + } +} diff --git a/src/Titano.Data/Processing/SequenceFrames.cs b/src/Titano.Data/Processing/SequenceFrames.cs new file mode 100644 index 00000000..6d9f7cc0 --- /dev/null +++ b/src/Titano.Data/Processing/SequenceFrames.cs @@ -0,0 +1,104 @@ +using Titano.Core.Analysis; +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.Metadata; +using Titano.Core.Model; + +namespace Titano.Data.Processing; + +/// +/// One sequence as the processing jobs read it: the frame rows in index order and the file to decode for each (an +/// archive original or proxy, or a render node's cached copy; null when there is none). +/// +internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList Frames, IReadOnlyList Files) +{ + /// Flags of frames that never reach a render. + public const FrameFlags NotRendered = FrameFlags.Missing | FrameFlags.Corrupt | FrameFlags.Excluded | FrameFlags.Undecodable; + + /// Flags of frames whose content is not analysed (absent, or not the file that was archived). + public const FrameFlags NotAnalysed = FrameFlags.Missing | FrameFlags.Corrupt; + + public int Count => Frames.Count; + + /// Every frame row of with its archive file (the proxy when asked and built). + public static async Task LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies, + CancellationToken cancellationToken) + { + var frames = await ListAsync(db, sequence.Id, cancellationToken).ConfigureAwait(false); + var paths = new ArchivePaths(archive.RootPath); + var folder = paths.Sequence(sequence.Folder); + var files = new string?[frames.Count]; + for (var i = 0; i < files.Length; i++) + { + var frame = frames[i]; + if ((frame.Flags & FrameFlags.Missing) != 0) continue; + var proxy = useProxies ? folder.Proxy(frame.Index) : null; + files[i] = proxy is not null && File.Exists(proxy) ? proxy : paths.FromRelative(frame.RelativePath); + } + return new SequenceFrames(sequence, frames, files); + } + + public static async Task> ListAsync(TitanoDatabase db, Guid sequenceId, CancellationToken cancellationToken) + { + const int pageSize = 1000; + var frames = new List(); + while (true) + { + var after = frames.Count > 0 ? frames[^1].Index : -1; + var page = await db.Frames.ListBySequenceAsync(sequenceId, after, pageSize, cancellationToken).ConfigureAwait(false); + frames.AddRange(page); + if (page.Count < pageSize) return frames; + } + } + + /// + /// Light-analysis input (BRIEF §10.1): every frame's metadata with the sequence's camera and lens. The index keeps + /// no GPS, so the first readable file's fix is read once: a time-lapse camera stays put, and the analysis applies + /// that position (and the clock offset it implies) to the whole sequence. + /// + public IReadOnlyList LightFrames() + { + var s = Sequence; + var light = new LightFrame[Frames.Count]; + for (var i = 0; i < light.Length; i++) + { + light[i] = LightFrame.From(Frames[i]) with + { + Index = i, + CameraMake = s.CameraMake, + CameraModel = s.CameraModel, + CameraSerial = s.CameraSerial, + Lens = s.Lens, + }; + } + if (FirstGpsFix() is var (index, gps)) light[index] = light[index] with { Latitude = gps.Latitude, Longitude = gps.Longitude, GpsUtc = gps.UtcTimestamp }; + return light; + } + + /// Positions of the frames a render must skip besides the project's exclusions. + public IEnumerable NotRenderable() + { + for (var i = 0; i < Frames.Count; i++) + { + if (Files[i] is null || (Frames[i].Flags & NotRendered) != 0) yield return i; + } + } + + private (int Index, GpsPosition Gps)? FirstGpsFix() + { + for (var i = 0; i < Files.Count; i++) + { + if (Files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue; + try + { + return MetadataReader.Read(file).Gps is { } gps ? (i, gps) : null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("processing", $"Metadata of {file} could not be read", ex); + return null; + } + } + return null; + } +} diff --git a/src/Titano.Data/Processing/SequenceRender.cs b/src/Titano.Data/Processing/SequenceRender.cs new file mode 100644 index 00000000..3ab64003 --- /dev/null +++ b/src/Titano.Data/Processing/SequenceRender.cs @@ -0,0 +1,149 @@ +using Titano.Core.Analysis; +using Titano.Core.Analysis.Deep; +using Titano.Core.Export; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Core.Project; +using Titano.Core.Render; + +namespace Titano.Data.Processing; + +/// One output of a render job: the resolved export values and the preset name recorded with the render. +internal sealed record ExportTarget(ResolvedExport Export, string PresetName); + +/// How one target of ended: a result, or the localised reason it failed. +internal sealed record TargetOutcome(ExportTarget Target, RenderResult? Result, string? Error); + +/// +/// The render of one sequence (BRIEF §11): the frame source, the clock-corrected capture metadata, the project with the +/// frames that cannot be rendered excluded, and the pipeline with the host's encoders. Targets share one preparation +/// pass; each is named once its encoder (and so its codec and size) is known, and failures of one target do not stop +/// the others. Outputs are written by the encoder sessions to temporary names and renamed on completion, so a +/// cancelled or failed target leaves nothing behind. +/// +internal sealed class SequenceRender : IAsyncDisposable +{ + private readonly SequenceFrameSource _source; + private readonly RenderPipeline _pipeline; + private readonly RenderOptions _options; + + /// The sequence's deep report, when current: its luminance spares the preparation a decode pass. + public SequenceRender(SequenceFrames frames, ProjectSettings project, DeepAnalysisReport? analysis, ProcessingOptions options) + { + var metadata = frames.LightFrames(); + _source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder, options.FrameWindow); + _pipeline = new RenderPipeline(options.Encoders, options.Host); + _options = options.Render; + Input = new RenderInput + { + Source = _source, + Frames = SourceFrameInfo.From(frames.Frames, LightAnalyzer.Analyze(metadata).AdjustedTimes), + Project = project with { ExcludedFrames = [.. project.ExcludedFrames.Union(frames.NotRenderable()).Order()] }, + Analysis = analysis?.FrameCount == frames.Count ? new PrecomputedAnalysis { LogLuminance = Luminance(analysis.Frames) } : null, + }; + } + + public RenderInput Input { get; } + + /// Encoder, geometry and plan of an export before its output is named. + public RenderSetup Setup(ResolvedExport export) => _pipeline.Setup(Input, new RenderTarget(export, ExportNaming.FallbackName), _options); + + /// + /// Prepares the sequence once and renders every target. gives the output path of a target + /// from its setup; runs before a target writes anything (checkpoints), + /// after its output is complete (record it, upload it). Progress counts measured and + /// rendered frames over the whole call. + /// + public async Task> RenderAllAsync( + IReadOnlyList<(int Index, ExportTarget Target)> targets, + Func name, + Func starting, + Func finished, + Func progress, + CancellationToken cancellationToken) + { + var outcomes = new List(targets.Count); + var ready = new List<(int Index, ExportTarget Target, RenderSetup Setup)>(); + foreach (var (index, target) in targets) + { + RenderSetup setup; + try + { + setup = Setup(target.Export); + } + catch (RenderException ex) + { + // The source size could not be measured (no renderable frame decodes). + outcomes.Add(new TargetOutcome(target, null, Reason(ex))); + continue; + } + if (setup.Encoder.IsReady && setup.Plan.Frames.Count > 0) ready.Add((index, target, setup)); + else outcomes.Add(new TargetOutcome(target, null, Reason(setup))); + } + if (ready.Count == 0) return outcomes; + + var preparing = ready[0].Setup.NeedsPreparation ? ready[0].Setup.Plan.IncludedFrames.Count : 0; + var total = preparing + ready.Sum(r => (long)r.Setup.Plan.Frames.Count); + var stage = Loc.L("Export.Job.Preparing"); + var chain = new ProgressChain(progress); + var corrections = await RenderPreparation.PrepareAsync(Input, _options, Stage(chain, 0, preparing, total, stage), cancellationToken).ConfigureAwait(false); + await chain.FlushAsync().ConfigureAwait(false); + + long done = preparing; + foreach (var (index, target, setup) in ready) + { + cancellationToken.ThrowIfCancellationRequested(); + var output = name(target, setup); + await starting(index, output).ConfigureAwait(false); + try + { + var result = await _pipeline.RenderAsync(Input, corrections, new RenderTarget(target.Export, output), _options, + Stage(chain, done, setup.Plan.Frames.Count, total, target.PresetName), cancellationToken).ConfigureAwait(false); + await chain.FlushAsync().ConfigureAwait(false); + await finished(index, target, result).ConfigureAwait(false); + outcomes.Add(new TargetOutcome(target, result, null)); + } + catch (Exception ex) when (!(ex is OperationCanceledException && cancellationToken.IsCancellationRequested)) + { + Core.Diagnostics.Log.Warning("processing", $"Render of {target.PresetName} to {output} failed", ex); + outcomes.Add(new TargetOutcome(target, null, Reason(ex))); + } + done += setup.Plan.Frames.Count; + } + return outcomes; + } + + /// Values of the output name tokens: the shoot date and title of the sequence, the codec and size the encoder will use. + public static ExportNameContext NameContext(Sequence sequence, ExportTarget target, RenderSetup setup) => new() + { + Date = sequence.FirstCapture ?? DateTime.Now, + Title = sequence.DisplayTitle, + PresetName = target.PresetName, + Width = setup.Encoder.Settings.Width, + Height = setup.Encoder.Settings.Height, + FrameRate = setup.Encoder.Settings.FrameRate, + Codec = setup.Encoder.Settings.Codec, + }; + + /// Localised text of a render failure. + public static string Reason(Exception exception) => + exception is RenderException render ? Loc.L(render.ReasonKey, [.. render.Args]) : exception.Message; + + /// Localised remarks of a finished render (encoder fallbacks, ignored settings…), without repeats. + public static IEnumerable Remarks(RenderResult result) => result.Warnings.Select(w => Loc.L(w.Key, [.. w.Args])).Distinct(); + + public ValueTask DisposeAsync() => _source.DisposeAsync(); + + private static string Reason(RenderSetup setup) + { + if (setup.Plan.Frames.Count == 0) return Loc.L("Export.Render.NoFrames"); + var notice = setup.Encoder.Notices.LastOrDefault() ?? new EncoderNotice(EncoderNotices.NoEncoder, setup.Encoder.Settings.Codec.ToString()); + return Loc.L(notice.Key, [.. notice.Args]); + } + + private static double[] Luminance(DeepFrameColumns columns) => [.. columns.LogLuminance.Select(v => v ?? double.NaN)]; + + /// One stage's progress (frames measured, or rendered for one target) as frames of the whole call. + private static IProgress Stage(ProgressChain chain, long offset, long length, long total, string message) => + chain.For(p => (offset + long.Min(length, p.Done), total, message)); +} diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index 31619615..943d2e57 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -174,6 +174,23 @@ public sealed class SequenceRepository await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1", [id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; + /// + /// Records an analysis: status flags set and cleared, the warning count and, for a deep analysis, the compact + /// summary ( null keeps the stored one). + /// + public async Task RecordAnalysisAsync(Guid id, SequenceStatus set, SequenceStatus clear, int warningCount, string? analysisJson, + CancellationToken cancellationToken = default) + { + DbParam[] args = [id, (int)set, (int)clear, warningCount, DateTimeOffset.UtcNow]; + var sql = "UPDATE sequences SET status = (status | $2) & ~$3, warning_count = $4, updated_at = $5"; + if (analysisJson is not null) + { + sql += ", analysis_json = $6"; + args = [.. args, DbParam.Json(analysisJson)]; + } + return await _db.ExecuteAsync(sql + " WHERE id = $1", args, cancellationToken).ConfigureAwait(false) > 0; + } + /// Recomputes frame count, total bytes and first/last capture from the frames table. public Task RefreshStatsAsync(Guid id, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => diff --git a/src/Titano.Data/Titano.Data.csproj b/src/Titano.Data/Titano.Data.csproj index ee1ac377..fc96b46f 100644 --- a/src/Titano.Data/Titano.Data.csproj +++ b/src/Titano.Data/Titano.Data.csproj @@ -5,6 +5,7 @@ + diff --git a/src/Titano.Server/Storage/ArchiveModule.cs b/src/Titano.Server/Storage/ArchiveModule.cs index d2152ec9..d1738a91 100644 --- a/src/Titano.Server/Storage/ArchiveModule.cs +++ b/src/Titano.Server/Storage/ArchiveModule.cs @@ -7,17 +7,21 @@ namespace Titano.Server.Storage; /// /// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and -/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance). Decoding is -/// managed only (BRIEF §4.2). +/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance), plus the +/// analysis handlers of . Decoding is managed only (BRIEF §4.2). /// public static class ArchiveModule { + /// Name the server writes into the archive's lease files. + public static string Owner { get; } = "server@" + Environment.MachineName; + public static void Register(IServiceCollection services) { services.AddSingleton(sp => new ArchiveService(sp.GetRequiredService(), - new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = "server@" + Environment.MachineName })); + new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = Owner })); services.AddSingleton(sp => sp.GetRequiredService().Ingest); services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); + ProcessingModule.Register(services); } } diff --git a/src/Titano.Server/Storage/ProcessingModule.cs b/src/Titano.Server/Storage/ProcessingModule.cs new file mode 100644 index 00000000..591ef564 --- /dev/null +++ b/src/Titano.Server/Storage/ProcessingModule.cs @@ -0,0 +1,30 @@ +using Titano.Core.Export; +using Titano.Core.Imaging.Decoders; +using Titano.Core.Video.Managed; +using Titano.Data; +using Titano.Data.Archiving; +using Titano.Data.Jobs; +using Titano.Data.Processing; +using Titano.Server.Configuration; + +namespace Titano.Server.Storage; + +/// +/// Analysis and render engine of the server (BRIEF §4.2, §10, §14.6): light and deep analysis job handlers with managed +/// decoders, so the light analysis queued after uploads runs here. Its render handler is registered too, but the job +/// engine host never runs render jobs: render nodes claim them (BRIEF §14.7). +/// +public static class ProcessingModule +{ + public static void Register(IServiceCollection services) + { + services.AddSingleton(sp => new ProcessingService(sp.GetRequiredService(), new ProcessingOptions + { + Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = ArchiveModule.Owner }, + Encoders = [new ManagedEncoderProvider()], + Host = EncoderHost.Server, + ProducedBy = sp.GetRequiredService().ServerName, + })); + services.AddSingleton(sp => sp.GetRequiredService()); + } +} diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs new file mode 100644 index 00000000..7673915d --- /dev/null +++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs @@ -0,0 +1,126 @@ +using Titano.Core.Analysis.Deep; +using Titano.Core.Archiving; +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Core.Project; +using Titano.Data.Processing; +using Titano.Data.Repositories; +using Titano.Testing; + +namespace Titano.Data.Tests.Processing; + +[Category("processing")] +public sealed class AnalysisJobTests(TestContext context) +{ + private static SyntheticSequence Flickering(int frames = 40) => SyntheticSequence.Create(new SyntheticSequenceOptions + { + FrameCount = frames, + FlickerEv = 0.1, + StartEv = 0.5, + EndEv = -1.5, + Latitude = 45.4, + Longitude = 12.3, + }); + + [Test] + public async Task LightAnalysisFillsAutoValuesAndKeepsManualOnes() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering()); + var manual = new ProjectSettings(); + manual = manual with { Timing = manual.Timing with { FrameRate = manual.Timing.FrameRate.WithManual("25") } }; + await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = manual.ToJson() }, null, fx.Ct); + await fx.StartEngineAsync(); + + var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id], sequence.DisplayTitle)); + var done = await fx.WaitForAsync(job.Id, JobState.Done); + + var folder = fx.Folder(sequence); + var project = ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)); + Assert.Equal((false, "25"), (project.Timing.FrameRate.IsAuto, project.Timing.FrameRate.Resolve("30"))); + Assert.True(project.Timing.Mode.HasAdvice, "timing mode advised"); + Assert.True(project.Timing.FrameRate.HasAdvice, "the advice is kept next to the manual frame rate"); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(project.ToJson(), ProjectSettings.FromJson(row.SettingsJson!).ToJson()); + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status"); + Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), "no deep status without a report"); + Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); + Assert.Contains(Loc.L("Advisor.Log.AutoUpdated", sequence.DisplayTitle), done.LogTail); + + // Idempotent: a second pass changes nothing. + var version = row.SettingsVersion; + var again = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id])); + await fx.WaitForAsync(again.Id, JobState.Done); + Assert.Equal(version, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.SettingsVersion); + } + + [Test] + public async Task DeepAnalysisResumesAfterCancellationWithTheSameReport() + { + await using var reference = await ProcessingFixture.CreateAsync(context); + var expectedSequence = await reference.AddAsync(Flickering()); + await reference.StartEngineAsync(); + var full = await reference.EnqueueAsync(ProcessingJobs.DeepAnalysis([expectedSequence.Id])); + await reference.WaitForAsync(full.Id, JobState.Done); + var expected = await File.ReadAllTextAsync(reference.Folder(expectedSequence).AnalysisPath, reference.Ct); + + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering()); + var folder = fx.Folder(sequence); + var partial = Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName); + // Frame 25 is not in the region sample, so the hold stops the pass inside the batch of frames 24–25. + fx.Decoder.HoldFrame(25); + await fx.StartEngineAsync(); + var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + await fx.Engine.CancelAsync(job.Id, fx.Ct); + await fx.WaitForAsync(job.Id, JobState.Cancelled); + var saved = File.ReadAllLines(partial).Length - 1; + Assert.Equal(24, saved); + Assert.FileMissing(folder.AnalysisPath); + + fx.Decoder.Hold = null; + while (fx.Decoder.Requests.TryDequeue(out _)) + { + } + await fx.Engine.ResumeAsync(job.Id, fx.Ct); + var done = await fx.WaitForAsync(job.Id, JobState.Done); + + Assert.Equal(expected, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct)); + Assert.FileMissing(partial); + // After the region sample, only the last checkpointed frame is decoded again (to restore the comparison state). + var analysed = fx.Decoder.Requests.Skip(ProcessingFixture.DeepOptions.RegionSampleFrames) + .Select(r => int.Parse(Path.GetFileNameWithoutExtension(r.Path)[4..]) - 1).ToList(); + Assert.Equal(saved - 1, analysed.Min()); + Assert.Equal(sequence.FrameCount, (int)done.ProgressDone); + + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep | SequenceStatus.AnalyzedLight), "index status"); + var summary = AnalysisSummary.FromJson(row.AnalysisJson!); + Assert.Equal(sequence.FrameCount, summary.FrameCount); + Assert.NotNull((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[5].MeanLogLuminance); + var project = ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)); + Assert.True(project.Exposure.Deflicker.HasAdvice, "deflicker advised from the pixels"); + Assert.Contains(Loc.L("Advisor.Log.Presets", string.Empty), done.LogTail); + } + + [Test] + public async Task LightAnalysisAfterNewFramesDropsTheStaleDeepStatus() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + await fx.StartEngineAsync(); + var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.WaitForAsync(deep.Id, JobState.Done); + Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedDeep), "deep status"); + + // A thirteenth frame arrives: the report describes 12 frames, so it no longer counts. + await fx.AddAsync(Flickering(13), first: 12); + var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id])); + await fx.WaitForAsync(light.Id, JobState.Done); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(13, row.FrameCount); + Assert.Equal((true, false), (row.Status.HasFlag(SequenceStatus.AnalyzedLight), row.Status.HasFlag(SequenceStatus.AnalyzedDeep))); + } +} diff --git a/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs b/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs new file mode 100644 index 00000000..39939b2e --- /dev/null +++ b/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs @@ -0,0 +1,131 @@ +using System.Collections.Concurrent; +using Titano.Core.Analysis.Deep; +using Titano.Core.Archiving; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Decoders; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Jobs; +using Titano.Data.Processing; +using Titano.Data.Tests.Archiving; +using Titano.Testing; + +namespace Titano.Data.Tests.Processing; + +/// A temporary archive holding synthetic sequences (JPEGs with Exif), a processing engine and its job engine. +internal sealed class ProcessingFixture : IAsyncDisposable +{ + /// Small sources are analysed at native size; batches of two frames between checkpoints. + public static readonly DeepAnalysisOptions DeepOptions = new() { MinAnalysisLongEdge = 64, FlowEvery = 5, MaxDegreeOfParallelism = 2, RegionSampleFrames = 9 }; + + private JobEngine? _engine; + + private ProcessingFixture(ArchiveFixture archive, GateDecoder decoder) + { + Archive = archive; + Decoder = decoder; + Processing = new ProcessingService(archive.Db, new ProcessingOptions + { + Archive = new ArchiveServiceOptions { Decoder = new ImageDecoder([decoder]), Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }, + DeepAnalysis = DeepOptions, + ProducedBy = "test-pc", + }); + } + + public ArchiveFixture Archive { get; } + public GateDecoder Decoder { get; } + public ProcessingService Processing { get; } + public TitanoDatabase Db => Archive.Db; + public CancellationToken Ct => Archive.Ct; + + public JobEngine Engine => _engine ?? throw new InvalidOperationException("Start the engine first."); + + public static async Task CreateAsync(TestContext context) => new(await ArchiveFixture.CreateAsync(context), new GateDecoder()); + + /// A started engine with the processing handlers (one worker, fast checkpoints). + public async Task StartEngineAsync() + { + _engine = new JobEngine(Db, Processing.CreateJobHandlers(), + new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await _engine.StartAsync(Ct); + return _engine; + } + + public Task EnqueueAsync(Titano.Data.Archiving.Jobs.ArchiveJobRequest request) => Engine.EnqueueAsync(request, cancellationToken: Ct); + + public Task WaitForAsync(Guid jobId, JobState state) => Archive.WaitForAsync(jobId, state, 60); + + /// Writes frames … of as JPEGs with their Exif into + /// (adopted in place) and indexes the archive. + public async Task AddAsync(SyntheticSequence sequence, string folder = "Synthetic", int first = 0) + { + var directory = Path.Combine(Archive.Paths.Root, folder); + Directory.CreateDirectory(directory); + for (var i = first; i < sequence.Count; i++) + { + using var image = sequence.Render(i); + var exif = ExifWriter.BuildJpegApp1(sequence.Metadata[i]); + File.WriteAllBytes(Path.Combine(directory, FileName(i)), JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 95, App1Segments = [exif[4..]] })); + } + await Archive.ScanAsync(ScanMode.Full); + var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder); + return (await Db.Sequences.GetAsync(location.Id, Ct))!; + } + + public static string FileName(int index) => $"IMG_{index + 1:0000}.JPG"; + + public SequencePaths Folder(Sequence sequence) => Archive.Paths.Sequence(sequence.Folder); + + public async ValueTask DisposeAsync() + { + if (_engine is not null) await _engine.DisposeAsync(); + await Archive.DisposeAsync(); + } +} + +/// Managed decoding that records every request and can hold one file's decode until the job is cancelled. +internal sealed class GateDecoder : IImageDecoderProvider +{ + private readonly ManagedDecoderProvider _managed = new(); + + public string Name => "gate"; + + public ConcurrentQueue Requests { get; } = new(); + + /// File name whose decode waits for cancellation; null decodes everything. + public string? Hold { get; set; } + + /// Holds only decodes of this purpose (null: any). + public DecodePurpose? HoldPurpose { get; set; } + + /// Holds only decodes with this long edge (null: any). + public int? HoldEdge { get; set; } + + /// Completed when a decode of started waiting. + public TaskCompletionSource Holding { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void HoldFrame(int index, DecodePurpose? purpose = null, int? edge = null) + { + Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + (HoldPurpose, HoldEdge) = (purpose, edge); + Hold = ProcessingFixture.FileName(index); + } + + public DecodeTier? TierFor(ImageFileFormat format) => _managed.TierFor(format); + + public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) + { + Requests.Enqueue(request); + if (Hold is { } hold && Path.GetFileName(request.Path) == hold && (HoldPurpose ?? request.Purpose) == request.Purpose + && (HoldEdge ?? request.MaxLongEdge) == request.MaxLongEdge) + { + Holding.TrySetResult(); + cancellationToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(30)); + cancellationToken.ThrowIfCancellationRequested(); + } + return _managed.Decode(request, cancellationToken); + } +} diff --git a/tests/Titano.Data.Tests/Processing/RenderJobTests.cs b/tests/Titano.Data.Tests/Processing/RenderJobTests.cs new file mode 100644 index 00000000..bf1bc8a7 --- /dev/null +++ b/tests/Titano.Data.Tests/Processing/RenderJobTests.cs @@ -0,0 +1,161 @@ +using System.Text; +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Export; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Decoders; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Core.Project; +using Titano.Data.Processing; +using Titano.Testing; + +namespace Titano.Data.Tests.Processing; + +[Category("processing")] +public sealed class RenderJobTests(TestContext context) +{ + private const int Frames = 24; + + /// MJPEG at the source size, named by codec so the tests know the file names. + private static readonly ExportSettings Mjpeg = new() + { + Preset = new Auto().WithManual("mjpeg-avi"), + Overrides = new ExportOverrides { Width = 96, Height = 64 }, + NameTemplate = "clip-{codec}", + }; + + private static SyntheticSequence Scene() => SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 }); + + [Test] + public async Task ABatchWritesPlayableManagedOutputsIntoExportsAndRecordsThem() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Scene()); + await fx.StartEngineAsync(); + + var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["png-sequence", "gif-preview"] }, 96, 64)); + await fx.WaitForAsync(job.Id, JobState.Done); + + var exports = fx.Folder(sequence).Exports; + var avi = await File.ReadAllBytesAsync(Path.Combine(exports, "clip-MJPEG.avi"), fx.Ct); + Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(avi, 0, 4), Encoding.ASCII.GetString(avi, 8, 4))); + Assert.Contains("idx1", Encoding.ASCII.GetString(avi)); + var pngs = Directory.GetFiles(Path.Combine(exports, "clip-PNG")).Order().ToList(); + Assert.Equal(Frames, pngs.Count); + Assert.Equal("clip-PNG_000001.png", Path.GetFileName(pngs[0])); + using (var decoded = DecoderFactory.CreateServer().Decode(new DecodeRequest(pngs[^1]))) + Assert.Equal((96, 64), (Assert.NotNull(decoded).Pixels.Width, decoded.Pixels.Height)); + var gif = await File.ReadAllBytesAsync(Path.Combine(exports, "clip-GIF.gif"), fx.Ct); + Assert.Equal("GIF89a", Encoding.ASCII.GetString(gif, 0, 6)); + Assert.Empty(Directory.EnumerateFileSystemEntries(exports, ".*")); + + var renders = (await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).OrderBy(r => r.OutputPath, StringComparer.Ordinal).ToList(); + Assert.SequenceEqual(["Synthetic/exports/clip-GIF.gif", "Synthetic/exports/clip-MJPEG.avi", "Synthetic/exports/clip-PNG"], renders.Select(r => r.OutputPath)); + var mjpeg = renders[1]; + Assert.Equal((Loc.L("Export.Preset.Custom"), "Mjpeg", 96, 64, "test-pc", (long)avi.Length), + (mjpeg.PresetName, mjpeg.Codec, mjpeg.Width, mjpeg.Height, mjpeg.ProducedBy, mjpeg.Bytes)); + Assert.Equal(Loc.L("Export.Preset.Gif"), renders[0].PresetName); + Assert.Equal(3, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount); + } + + [Test] + public async Task ACancelledRenderLeavesNothingBehind() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Scene()); + fx.Decoder.HoldFrame(12, DecodePurpose.Render); + await fx.StartEngineAsync(); + + var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64)); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + await fx.Engine.CancelAsync(job.Id, fx.Ct); + await fx.WaitForAsync(job.Id, JobState.Cancelled); + await WaitUntilIdleAsync(fx, job.Id); + + Assert.Empty(Entries(fx.Folder(sequence).Exports)); + Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)); + } + + [Test] + public async Task AFailedRenderLeavesNothingBehindAndSaysWhy() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Scene()); + // Damaged after indexing: nothing flags the frame, so only decoding finds out. + await File.WriteAllBytesAsync(Path.Combine(fx.Archive.Paths.Root, "Synthetic", ProcessingFixture.FileName(10)), new byte[5000], fx.Ct); + await fx.StartEngineAsync(); + + var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64)); + var failed = await fx.WaitForAsync(job.Id, JobState.Failed); + + Assert.Equal(Loc.L(SequenceFrameSource.FrameUnreadableKey, "11", ProcessingFixture.FileName(10)), failed.Error); + Assert.Empty(Entries(fx.Folder(sequence).Exports)); + Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)); + } + + [Test] + public async Task ARenderInterruptedByACrashStartsItsOutputAgainCleanly() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Scene()); + var exports = fx.Folder(sequence).Exports; + var output = Path.Combine(exports, "clip-MJPEG.avi"); + // The crashed attempt left its encoder's temporary file and a finished output it never recorded. + Directory.CreateDirectory(exports); + await File.WriteAllBytesAsync(Path.Combine(exports, ".clip-MJPEG.avi.0badf00d.tmp"), new byte[100], fx.Ct); + await File.WriteAllBytesAsync(output, new byte[100], fx.Ct); + await fx.StartEngineAsync(); + + var job = await fx.Archive.EnqueueWithCheckpointAsync(fx.Engine, ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64), + new RenderCheckpoint([], 0, output).ToJson()); + await fx.WaitForAsync(job.Id, JobState.Done); + + Assert.SequenceEqual(["clip-GIF.gif", "clip-MJPEG.avi"], Entries(exports)); + Assert.Equal("RIFF", Encoding.ASCII.GetString((await File.ReadAllBytesAsync(output, fx.Ct))[..4])); + Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)); + } + + [Test] + public async Task ARenderInterruptedAfterRecordingAnOutputResumesWithTheNextOne() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Scene()); + var exports = fx.Folder(sequence).Exports; + var output = Path.Combine(exports, "clip-MJPEG.avi"); + Directory.CreateDirectory(exports); + var recorded = Encoding.ASCII.GetBytes("RIFF recorded before the crash"); + await File.WriteAllBytesAsync(output, recorded, fx.Ct); + await fx.Db.Renders.InsertAsync(new Render + { + Id = Guid.CreateVersion7(), + SequenceId = sequence.Id, + PresetName = "MJPEG", + OutputPath = "Synthetic/exports/clip-MJPEG.avi", + Created = DateTimeOffset.UtcNow, + }, fx.Ct); + await fx.StartEngineAsync(); + + var job = await fx.Archive.EnqueueWithCheckpointAsync(fx.Engine, ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64), + new RenderCheckpoint([], 0, output).ToJson()); + await fx.WaitForAsync(job.Id, JobState.Done); + + var kept = await File.ReadAllBytesAsync(output, fx.Ct); + Assert.BytesEqual(recorded, kept); + Assert.SequenceEqual(["clip-GIF.gif", "clip-MJPEG.avi"], Entries(exports)); + Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)); + } + + private static List Entries(string folder) => + Directory.Exists(folder) ? [.. Directory.EnumerateFileSystemEntries(folder).Select(e => Path.GetFileName(e)).Order(StringComparer.Ordinal)] : []; + + /// The engine marks a job cancelled before its handler has unwound; wait until the worker let it go. + private static async Task WaitUntilIdleAsync(ProcessingFixture fx, Guid jobId) + { + var until = DateTime.UtcNow.AddSeconds(30); + while (fx.Engine.RunningJobIds.Contains(jobId)) + { + Assert.True(DateTime.UtcNow < until, "the render did not stop"); + await Task.Delay(20, fx.Ct); + } + } +} diff --git a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs new file mode 100644 index 00000000..f5ea40c8 --- /dev/null +++ b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs @@ -0,0 +1,120 @@ +using System.Text; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Export; +using Titano.Core.Imaging.Decoders; +using Titano.Core.Localization; +using Titano.Core.Project; +using Titano.Data.Archiving; +using Titano.Data.Processing; +using Titano.Protocol; +using Titano.Protocol.Client; +using Titano.Testing; + +namespace Titano.Server.Tests.Jobs; + +/// The desktop client's render node (managed encoders here) against the in-process server. +[Category("processing")] +public sealed class RenderNodeClientTests(TestContext context) +{ + private const int Frames = 12; + + private static readonly ExportSettings Mjpeg = new() + { + Preset = new Auto().WithManual("mjpeg-avi"), + Overrides = new ExportOverrides { Width = 96, Height = 64 }, + NameTemplate = "{title} - {codec}", + }; + + [Test] + public async Task ANodeClaimsRendersWithManagedEncodersAndUploadsTheResult() + { + await using var server = await TestServer.StartAsync(context); + var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); + var sequenceId = await UploadSequenceAsync(editor, Photos()); + var job = await CreateRenderAsync(server, editor, sequenceId); + + using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; + var cache = context.CreateTempDirectory("node"); + await using var node = Node(client, cache); + Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); + Assert.False(await node.RunOnceAsync(context.Cancellation), "the queue is empty"); + + var done = await JobApiTests.GetAsync(editor, job.Id); + Assert.Equal("done", done.State); + var render = (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Single(); + Assert.Equal(("studio-pc", "mjpeg", 96, 64), (render.ProducedBy, render.Codec, render.Width, render.Height)); + var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!; + var file = await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, render.OutputPath)); + Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(file, 0, 4), Encoding.ASCII.GetString(file, 8, 4))); + Assert.Equal($"{sequence.Folder}/exports/{sequence.DisplayTitle} - MJPEG.avi", render.OutputPath); + Assert.Equal(1, sequence.RenderedCount); + // Originals stay cached by content for the next job; the job's own folder is gone. + Assert.Count(Frames, Directory.GetFiles(Path.Combine(cache, "originals"), "*.jpg", SearchOption.AllDirectories)); + Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed"); + } + + [Test] + public async Task ANodeReportsWhyItCouldNotRender() + { + await using var server = await TestServer.StartAsync(context); + var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); + // Uploads are opaque to the server's fake ingest: these "photos" decode to nothing. + var sequenceId = await UploadSequenceAsync(editor, [Uploads.Photo(4000, 1), Uploads.Photo(4000, 2)]); + var job = await CreateRenderAsync(server, editor, sequenceId); + + using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; + await using var node = Node(client, context.CreateTempDirectory("node")); + Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); + + var failed = await JobApiTests.GetAsync(editor, job.Id); + var first = (await server.Database.Frames.ListBySequenceAsync(sequenceId))[0]; + Assert.Equal(("failed", Loc.L("Export.Render.FrameUnreadable", "1", first.FileName)), (failed.State, failed.Error)); + Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequenceId)); + } + + [Test] + public async Task ANodeOffersOnlyEncodersWhoseOutputIsOneFile() + { + await using var server = await TestServer.StartAsync(context); + var (_, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); + using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; + await using var node = Node(client, context.CreateTempDirectory("node")); + + var id = await node.RegisterAsync(context.Cancellation); + var registered = (await client.ListNodesAsync(context.Cancellation)).Single(n => n.Id == id); + Assert.SequenceEqual(["apng", "gif", "mjpeg"], registered.Encoders.Select(e => e.Codec).Order(StringComparer.Ordinal)); + } + + private static RenderNodeClient Node(TitanoClient client, string cache) => + new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } }, + new RenderNodeOptions { ClientName = "studio-pc", CacheDirectory = cache, ProgressInterval = TimeSpan.Zero }); + + private static async Task CreateRenderAsync(TestServer server, HttpClient editor, Guid sequenceId) + { + var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!; + var request = ProcessingJobs.Render(sequence, Mjpeg, 96, 64); + return await JobApiTests.CreateAsync(editor, "render", sequenceId, request.ParametersJson); + } + + /// Uploads the photos as IMG_0001.JPG… into one new sequence; returns its id. + private static async Task UploadSequenceAsync(HttpClient client, IReadOnlyList photos) + { + Guid? sequenceId = null; + for (var i = 0; i < photos.Count; i++) + sequenceId = (await Uploads.UploadAsync(client, photos[i], $"IMG_{i + 1:0000}.JPG", sequenceId)).SequenceId; + return sequenceId!.Value; + } + + private static List Photos() + { + var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 }); + var photos = new List(Frames); + for (var i = 0; i < Frames; i++) + { + using var image = scene.Render(i); + photos.Add(JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90 })); + } + return photos; + } +} diff --git a/tests/Titano.Server.Tests/Storage/ProcessingModuleTests.cs b/tests/Titano.Server.Tests/Storage/ProcessingModuleTests.cs new file mode 100644 index 00000000..821c4155 --- /dev/null +++ b/tests/Titano.Server.Tests/Storage/ProcessingModuleTests.cs @@ -0,0 +1,48 @@ +using Titano.Core.Archiving; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Core.Project; +using Titano.Server.Storage; +using Titano.Testing; + +namespace Titano.Server.Tests.Storage; + +/// The production composition: frames committed by the archive engine are light-analysed by the processing handlers. +[Category("processing")] +public sealed class ProcessingModuleTests(TestContext context) +{ + [Test] + public async Task UploadedFramesAreLightAnalysedAndAdvised() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 10, StartEv = 0.5, EndEv = -0.5 }); + Guid sequenceId = default; + for (var i = 0; i < scene.Count; i++) + { + using var image = scene.Render(i); + var exif = ExifWriter.BuildJpegApp1(scene.Metadata[i]); + var photo = JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90, App1Segments = [exif[4..]] }); + sequenceId = (await Uploads.UploadAsync(editor, photo, $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId; + } + + // The commits queue a light analysis; its handler comes from the processing module. + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + Sequence sequence; + while (true) + { + sequence = (await server.Database.Sequences.GetAsync(sequenceId))!; + if (sequence.Status.HasFlag(SequenceStatus.AnalyzedLight) && sequence.FrameCount == scene.Count + && !await server.Database.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequenceId)) break; + Assert.True(DateTime.UtcNow < until, $"not analysed: {sequence.Status}, {sequence.FrameCount} frames"); + await Task.Delay(100); + } + var project = ProjectSettings.FromJson(Assert.NotNull(sequence.SettingsJson)); + Assert.True(project.Timing.Mode.HasAdvice, "timing mode advised"); + var folder = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder); + Assert.FileExists(folder.ProjectPath); + Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); + } +}