From 14cdbf1f7d45fb5adc2a3c1898209a4185cde009 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 17:01:50 +0000 Subject: [PATCH] feat(processing): selective render claims, analysis follow-ups, per-sequence failures - JobEngine: ISelectiveJobHandler lets a host decline jobs of a kind it runs; declined jobs stay queued for other claimers without holding up the rest. JobContext.EnqueueFollowUpAsync queues follow-up work on the same engine. - Server renders the jobs whose every output a managed encoder produces (ManagedRenderJobHandler); MP4 jobs stay for render nodes. Program.cs registers ProcessingModule next to ArchiveModule. Server renders ignore a client's export folder. - Light analysis is queued after imports and after scans that indexed new sequences, so the advisor's Auto values exist without user action. - Analysis jobs log a failing sequence and go on; the job then fails and a retry analyses only the failed ones. The advised frame rate follows the user's fit-to duration. - ProcessingJobs.RenderEach splits a batch into one job per preset for servers; render nodes say which batch presets they did not render, fetch each content once and stop on a changed certificate. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- i18n/en/Advisor.resx | 1 + i18n/en/Errors.resx | 1 + i18n/en/Export.resx | 1 + i18n/it/Advisor.it.resx | 1 + i18n/it/Errors.it.resx | 1 + i18n/it/Export.it.resx | 1 + src/Titano.Data/Archiving/ImportService.cs | 12 +++ .../Archiving/Jobs/ImportJobHandler.cs | 4 +- .../Archiving/Jobs/ScanJobHandler.cs | 8 +- src/Titano.Data/Jobs/JobEngine.cs | 41 +++++++++- src/Titano.Data/Jobs/JobHandler.cs | 19 +++++ .../Processing/AnalysisFollowUp.cs | 36 +++++++++ .../Processing/Jobs/AnalyseJobHandler.cs | 74 ++++++++++++++----- .../Jobs/ManagedRenderJobHandler.cs | 40 ++++++++++ .../Processing/Jobs/RenderJobHandler.cs | 22 +++--- src/Titano.Data/Processing/ProcessingJobs.cs | 39 +++++++++- .../Processing/ProcessingService.cs | 22 ++++-- .../Processing/RenderNodeClient.cs | 41 ++++++---- .../Processing/SequenceAnalyser.cs | 8 +- src/Titano.Data/Repositories/JobRepository.cs | 16 ++-- src/Titano.Server/Jobs/JobEngineHost.cs | 8 +- src/Titano.Server/Program.cs | 11 ++- src/Titano.Server/Storage/ArchiveModule.cs | 5 +- src/Titano.Server/Storage/ProcessingModule.cs | 7 +- .../Titano.Data.Tests/Jobs/JobEngineTests.cs | 63 ++++++++++++++++ .../Processing/AnalysisFollowUpTests.cs | 65 ++++++++++++++++ .../Processing/AnalysisJobTests.cs | 50 +++++++++++++ .../Processing/ProcessingFixture.cs | 18 +++-- .../Processing/RenderJobTests.cs | 32 ++++++++ .../Storage/ProcessingModuleTests.cs | 10 ++- 30 files changed, 572 insertions(+), 85 deletions(-) create mode 100644 src/Titano.Data/Processing/AnalysisFollowUp.cs create mode 100644 src/Titano.Data/Processing/Jobs/ManagedRenderJobHandler.cs create mode 100644 tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs diff --git a/i18n/en/Advisor.resx b/i18n/en/Advisor.resx index 111c3a07..7c3ab191 100644 --- a/i18n/en/Advisor.resx +++ b/i18n/en/Advisor.resx @@ -33,6 +33,7 @@ {0} frames are blown out to white.{0} frame count Automatic values of {0} updated.job log line; {0} sequence title Deep analysis of {0}: {1} frames, {2} warnings.job log line; {0} sequence title; {1} frame count; {2} warning count + {0} could not be analysed: {1}long: job log line; {0} sequence title; {1} reason Light analysis of {0}: {1} frames, {2} warnings.job log line; {0} sequence title; {1} frame count; {2} warning count Suggested presets: {0}job log line; {0} preset names separated by dots The project settings of {0} cannot be read; their automatic values were left as they were.job log line; {0} sequence title diff --git a/i18n/en/Errors.resx b/i18n/en/Errors.resx index 22ce17de..0237576a 100644 --- a/i18n/en/Errors.resx +++ b/i18n/en/Errors.resx @@ -4,6 +4,7 @@ 2.0 System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + {0} of {1} sequences could not be analysed; see the job log.long; {0} failed count; {1} total count The archive could not be opened.long Enter a whole number from {0} to {1}. Enter a folder. diff --git a/i18n/en/Export.resx b/i18n/en/Export.resx index 3477a2d4..a328952f 100644 --- a/i18n/en/Export.resx +++ b/i18n/en/Export.resx @@ -16,6 +16,7 @@ Downloading framesrender node progress stage Measuring framesjob progress stage {0} failed: {1}long: job log line; {0} preset name; {1} reason + A render node renders one preset per job; not rendered: {0}long: job log line; {0} preset names separated by commas {0} saved as {1}long: job log line; {0} preset name; {1} output file name Playback rate not understood; the export frame rate is used.long No capture times: frames are spaced evenly.long diff --git a/i18n/it/Advisor.it.resx b/i18n/it/Advisor.it.resx index 892a7f6e..c50b0774 100644 --- a/i18n/it/Advisor.it.resx +++ b/i18n/it/Advisor.it.resx @@ -33,6 +33,7 @@ {0} fotogrammi sono completamente bruciati. Valori automatici di {0} aggiornati. Analisi approfondita di {0}: {1} fotogrammi, {2} avvisi. + Impossibile analizzare {0}: {1} Analisi rapida di {0}: {1} fotogrammi, {2} avvisi. Preset suggeriti: {0} Le impostazioni del progetto di {0} non sono leggibili; i valori automatici sono rimasti invariati. diff --git a/i18n/it/Errors.it.resx b/i18n/it/Errors.it.resx index 31b8b300..32a88246 100644 --- a/i18n/it/Errors.it.resx +++ b/i18n/it/Errors.it.resx @@ -4,6 +4,7 @@ 2.0 System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + {0} sequenze su {1} non sono state analizzate; vedi il registro del job. Impossibile aprire l'archivio. Inserisci un numero intero da {0} a {1}. Inserisci una cartella. diff --git a/i18n/it/Export.it.resx b/i18n/it/Export.it.resx index 27aed9a6..5284a032 100644 --- a/i18n/it/Export.it.resx +++ b/i18n/it/Export.it.resx @@ -16,6 +16,7 @@ Download dei fotogrammi Misura dei fotogrammi {0} non riuscito: {1} + Un nodo di rendering esegue un preset per job; non renderizzati: {0} {0} salvato come {1} Velocità di riproduzione non valida; viene usata la frequenza di esportazione. Nessun orario di scatto: fotogrammi distribuiti in modo uniforme. diff --git a/src/Titano.Data/Archiving/ImportService.cs b/src/Titano.Data/Archiving/ImportService.cs index e7db6a6e..cc90c435 100644 --- a/src/Titano.Data/Archiving/ImportService.cs +++ b/src/Titano.Data/Archiving/ImportService.cs @@ -143,6 +143,18 @@ public sealed class ImportService }, progress, cancellationToken).ConfigureAwait(false); } + /// The sequences the sessions of an import went into (new or continued), as far as they exist. + internal async Task> SequencesOfAsync(ImportJobParameters job, CancellationToken cancellationToken) + { + var ids = new List(); + foreach (var session in job.Plan.Sessions) + { + if (await _db.Sequences.FindBySessionKeyAsync(job.ArchiveId, session.SessionKey, cancellationToken).ConfigureAwait(false) is { } id + && !ids.Contains(id)) ids.Add(id); + } + return ids; + } + /// Dry run of "delete imported": every card file with a recorded import, checked against its archive copy. public async Task PlanEraseAsync(Archive archive, Guid cardId, string sourceRoot, IProgress? progress = null, CancellationToken cancellationToken = default) diff --git a/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs index bf48eaa5..3ad7cbec 100644 --- a/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs +++ b/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs @@ -4,13 +4,14 @@ using Titano.Core.Import; using Titano.Core.Localization; using Titano.Core.Model; using Titano.Data.Jobs; +using Titano.Data.Processing; namespace Titano.Data.Archiving.Jobs; /// /// : executes the confirmed plan in its parameters; progress in bytes. Checkpoint: the /// number of plan items done. A removed card fails the job at a batch boundary; resuming it after re-insertion -/// continues there. +/// continues there. The sequences the import went into are then queued for light analysis. /// internal sealed class ImportJobHandler(ImportService imports) : IJobHandler { @@ -29,5 +30,6 @@ internal sealed class ImportJobHandler(ImportService imports) : IJobHandler .ConfigureAwait(false); }, p => context.ReportAsync(p.BytesDone, p.BytesTotal, p.Current), context.Cancellation).ConfigureAwait(false); context.Log(Loc.L("Archive.Log.ImportDone", final.Copied, final.Skipped, final.Failed)); + await AnalysisFollowUp.QueueAsync(context, await imports.SequencesOfAsync(parameters, context.Cancellation).ConfigureAwait(false)).ConfigureAwait(false); } } diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs index 3b010fd4..fc111423 100644 --- a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs +++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs @@ -1,6 +1,7 @@ using Titano.Core.Localization; using Titano.Core.Model; using Titano.Data.Jobs; +using Titano.Data.Processing; namespace Titano.Data.Archiving.Jobs; @@ -15,7 +16,8 @@ internal static class JobArchive /// /// (incremental, or full with ) and -/// . Checkpoint: after:<folder>, the last folder indexed. +/// . Checkpoint: after:<folder>, the last folder indexed. When anything was +/// indexed, the sequences never light-analysed (adopted folders, new sidecars) are queued for it. /// internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler { @@ -37,5 +39,9 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, }, context.Cancellation).ConfigureAwait(false); foreach (var (folder, error) in summary.Failed) context.Log(Loc.L("Archive.Log.FolderFailed", folder, error)); context.Log(Loc.L("Archive.Log.ScanDone", summary.Folders, summary.Indexed, summary.Adopted, summary.Removed)); + // A resumed scan may have indexed its folders before the interruption. + if (summary.Indexed > 0 || resume is not null) + await AnalysisFollowUp.QueueAsync(context, await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false)) + .ConfigureAwait(false); } } diff --git a/src/Titano.Data/Jobs/JobEngine.cs b/src/Titano.Data/Jobs/JobEngine.cs index e6229cba..815ccc12 100644 --- a/src/Titano.Data/Jobs/JobEngine.cs +++ b/src/Titano.Data/Jobs/JobEngine.cs @@ -28,13 +28,18 @@ public sealed record JobEngineOptions /// /// Persistent job queue runner (BRIEF §14.6): N workers claim queued jobs from the database by priority, run the /// registered handler, checkpoint progress, and survive crashes (running jobs are requeued at start with their -/// checkpoints). Used by the desktop client (local mode) and the server. +/// checkpoints). Used by the desktop client (local mode) and the server. Jobs an +/// declines stay queued for other claimers. /// public sealed class JobEngine : IAsyncDisposable { + /// Queued jobs of selective kinds examined per claim; jobs declined beyond it wait until those ahead leave the queue. + private const int SelectiveWindow = 100; + private readonly TitanoDatabase _db; private readonly Dictionary _handlers; private IReadOnlyCollection _kinds = []; + private Dictionary _selective = []; private readonly ConcurrentDictionary _running = new(); private readonly Channel _wake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite }); private readonly CancellationTokenSource _shutdown = new(); @@ -65,6 +70,7 @@ public sealed class JobEngine : IAsyncDisposable await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false); // Only kinds this engine can run: an empty filter would claim (and fail) every job, render-node work included. _kinds = [.. (Options.Kinds ?? _handlers.Keys).Where(_handlers.ContainsKey).Distinct()]; + _selective = _kinds.Select(k => _handlers[k]).OfType().ToDictionary(h => h.Kind); if (_kinds.Count == 0) { Log.Info("jobs", "No job handlers; this engine claims nothing"); @@ -150,6 +156,9 @@ public sealed class JobEngine : IAsyncDisposable } } + /// True when this engine claims jobs of (it has a handler and the kind is not filtered out). + internal bool Runs(JobKind kind) => _kinds.Contains(kind); + internal void StopJob(Guid id) { if (_running.TryGetValue(id, out var cts)) cts.Cancel(); @@ -174,7 +183,7 @@ public sealed class JobEngine : IAsyncDisposable Job? job; try { - job = await _db.Jobs.ClaimNextAsync(workerId, _kinds, shutdown).ConfigureAwait(false); + job = await ClaimAsync(workerId, shutdown).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -205,6 +214,34 @@ public sealed class JobEngine : IAsyncDisposable } } + /// + /// The next job in claim order (priority, then age). Without selective handlers the database picks it atomically; + /// with them, the head of the other kinds competes with the first job a selective handler accepts among the first + /// of its kinds, and the winner is claimed if it is still queued. + /// + private async Task ClaimAsync(string workerId, CancellationToken cancellationToken) + { + if (_selective.Count == 0) return await _db.Jobs.ClaimNextAsync(workerId, _kinds, cancellationToken).ConfigureAwait(false); + var others = _kinds.Where(k => !_selective.ContainsKey(k)).ToList(); + for (var attempt = 0; attempt < 3; attempt++) + { + var best = others.Count > 0 ? (await _db.Jobs.ListQueuedAsync(others, 1, cancellationToken).ConfigureAwait(false)).FirstOrDefault() : null; + foreach (var job in await _db.Jobs.ListQueuedAsync(_selective.Keys, SelectiveWindow, cancellationToken).ConfigureAwait(false)) + { + if (!_selective[job.Kind].Accepts(job)) continue; + if (best is null || ClaimsBefore(job, best)) best = job; + break; + } + if (best is null) return null; + // Another claimer may have been faster: look again. + if (await _db.Jobs.TryClaimAsync(best.Id, workerId, cancellationToken).ConfigureAwait(false) is { } claimed) return claimed; + } + return null; + } + + private static bool ClaimsBefore(Job a, Job b) => + a.Priority != b.Priority ? a.Priority > b.Priority : a.Created != b.Created ? a.Created < b.Created : a.Id.CompareTo(b.Id) < 0; + private async Task RunAsync(Job job, CancellationToken shutdown) { using var cts = CancellationTokenSource.CreateLinkedTokenSource(shutdown); diff --git a/src/Titano.Data/Jobs/JobHandler.cs b/src/Titano.Data/Jobs/JobHandler.cs index 52539c9a..6a1af038 100644 --- a/src/Titano.Data/Jobs/JobHandler.cs +++ b/src/Titano.Data/Jobs/JobHandler.cs @@ -13,6 +13,16 @@ public interface IJobHandler Task RunAsync(JobContext context); } +/// +/// A handler that runs only some jobs of its kind, decided from the job alone (its parameters never change). The engine +/// leaves the others queued for other claimers: the server renders what its managed encoders produce and leaves the +/// rest to render nodes (BRIEF §14.5, §14.7). +/// +public interface ISelectiveJobHandler : IJobHandler +{ + bool Accepts(Job job); +} + /// A component that brings several handlers (the archive engine: scan, rebuild, verify, import, images, maintenance). public interface IJobHandlerProvider { @@ -75,6 +85,15 @@ public sealed class JobContext return ReportCoreAsync(done, total, Message, force: true); } + /// + /// Queues follow-up work on this job's engine for the same user (light analysis after an import); null when the + /// engine has no handler for , since nothing here would run it. + /// + public async Task EnqueueFollowUpAsync(JobKind kind, IReadOnlyList targets, string? label, string? parametersJson, JobPriority priority) => + _engine.Runs(kind) + ? await _engine.EnqueueAsync(kind, targets, label, parametersJson, priority, Job.CreatedBy, Cancellation).ConfigureAwait(false) + : null; + /// Appends a line to the job log (the last lines are kept with the job). public void Log(string line) { diff --git a/src/Titano.Data/Processing/AnalysisFollowUp.cs b/src/Titano.Data/Processing/AnalysisFollowUp.cs new file mode 100644 index 00000000..dba09d89 --- /dev/null +++ b/src/Titano.Data/Processing/AnalysisFollowUp.cs @@ -0,0 +1,36 @@ +using Titano.Core.Model; +using Titano.Data.Jobs; +using Titano.Data.Repositories; + +namespace Titano.Data.Processing; + +/// +/// Light analysis queued after work that brings sequences or frames into the index (card import, adoption, scan), so the +/// advisor's Auto values and recommended preset exist without any user action (BRIEF §1.1, §4.3). Nothing is queued +/// when the engine cannot run light analysis. +/// +internal static class AnalysisFollowUp +{ + /// Queues one light analysis of (labelled like the job that found them). + public static async Task QueueAsync(JobContext context, IReadOnlyCollection sequenceIds) + { + if (sequenceIds.Count == 0) return; + var request = ProcessingJobs.LightAnalysis([.. sequenceIds], context.Job.TargetLabel); + await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false); + } + + /// Sequences of that were never light-analysed (new or adopted ones). + public static async Task> UnanalysedAsync(TitanoDatabase db, Guid archiveId, CancellationToken cancellationToken) + { + var ids = new List(); + var query = new SequenceQuery { ArchiveId = archiveId, WithoutStatus = SequenceStatus.AnalyzedLight, Sort = SequenceSort.Oldest, Limit = 1000 }; + do + { + var page = await db.Sequences.ListAsync(query, cancellationToken).ConfigureAwait(false); + ids.AddRange(page.Items.Select(s => s.Id)); + query = query with { Cursor = page.NextCursor }; + } + while (query.Cursor is not null); + return ids; + } +} diff --git a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs index 26a35203..82a7c770 100644 --- a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs +++ b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs @@ -1,5 +1,6 @@ using System.Globalization; using Titano.Core.Analysis.Deep; +using Titano.Core.Archiving; using Titano.Core.Localization; using Titano.Core.Model; using Titano.Data.Jobs; @@ -8,8 +9,10 @@ namespace Titano.Data.Processing.Jobs; /// /// and over the target sequences, one after the -/// other. Progress counts sequences (light) or frames (deep); checkpoint: the number of sequences done. A deep pass -/// interrupted inside a sequence resumes from that sequence's analysis.partial. +/// other. Progress counts sequences (light) or frames (deep). A sequence that cannot be analysed is logged and the +/// others still run; the job then fails, and a retry analyses only the failed ones. An offline archive stops the job at +/// once. Checkpoint: the next target and the failed ones (next[;i,j…]). A deep pass interrupted inside a +/// sequence resumes from that sequence's analysis.partial. /// internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser analyser, JobKind kind) : IJobHandler { @@ -19,29 +22,47 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal { var deep = kind == JobKind.DeepAnalyse; var targets = context.Job.TargetIds; - var first = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var next) ? next : 0; + var (next, retry) = Parse(context.Checkpoint, targets.Count); // Deep progress is in frames across all targets, so the bar and the ETA cover the whole job. - var frames = new long[targets.Count]; - if (deep) - { - for (var i = 0; i < targets.Count; i++) - frames[i] = (await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false))?.FrameCount ?? 0; - } - var total = deep ? frames.Sum() : targets.Count; - for (var i = first; i < targets.Count; i++) + var units = new long[targets.Count]; + for (var i = 0; i < targets.Count; i++) + units[i] = deep ? (await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false))?.FrameCount ?? 0 : 1; + var total = units.Sum(); + var done = Enumerable.Range(0, next).Where(i => !retry.Contains(i)).Sum(i => units[i]); + var failed = new List(); + int[] order = [.. retry, .. Enumerable.Range(next, targets.Count - next)]; + for (var k = 0; k < order.Length; k++) { context.Cancellation.ThrowIfCancellationRequested(); - var before = deep ? frames[..i].Sum() : i; + var i = order[k]; + var before = done; var chain = new ProgressChain(context.ReportAsync); var progress = deep ? chain.For(p => (before + p.Done, total, null)) : null; - var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false); - await chain.FlushAsync().ConfigureAwait(false); - if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i])); - else Report(context, outcome, deep); - await context.CheckpointAsync((i + 1).ToString(CultureInfo.InvariantCulture), deep ? before + frames[i] : i + 1, total).ConfigureAwait(false); + try + { + var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false); + await chain.FlushAsync().ConfigureAwait(false); + if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i])); + else Report(context, outcome, deep); + } + catch (Exception ex) when (ex is not (OperationCanceledException or ArchiveOfflineException)) + { + await chain.FlushAsync().ConfigureAwait(false); + Core.Diagnostics.Log.Warning("processing", $"Analysis of sequence {targets[i]} failed", ex); + context.Log(Loc.L("Advisor.Log.Failed", await TitleAsync(targets[i], context.Cancellation).ConfigureAwait(false), ex.Message)); + failed.Add(i); + } + done += units[i]; + next = int.Max(next, i + 1); + // Failed targets of an earlier attempt not retried yet stay listed. + await context.CheckpointAsync(Format(next, [.. failed, .. retry.Skip(k + 1)]), done, total).ConfigureAwait(false); } + if (failed.Count > 0) throw new InvalidOperationException(Loc.L("Errors.Analysis.Failed", failed.Count, targets.Count)); } + private async Task TitleAsync(Guid sequenceId, CancellationToken cancellationToken) => + (await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false))?.DisplayTitle ?? sequenceId.ToString(); + private static void Report(JobContext context, AnalysisOutcome outcome, bool deep) { var title = outcome.Sequence.DisplayTitle; @@ -50,4 +71,23 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal if (outcome.SettingsUnreadable) context.Log(Loc.L("Advisor.Log.ProjectUnreadable", title)); if (outcome.Presets.Count > 0) context.Log(Loc.L("Advisor.Log.Presets", string.Join(" · ", outcome.Presets.Select(p => p.DisplayName())))); } + + /// Next target and the failed targets to retry first; a missing or unreadable checkpoint starts over. + internal static (int Next, IReadOnlyList Retry) Parse(string? checkpoint, int count) + { + var parts = checkpoint?.Split(';'); + if (parts is not ([_] or [_, _]) || !int.TryParse(parts[0], NumberStyles.None, CultureInfo.InvariantCulture, out var next) || next > count) return (0, []); + var retry = new List(); + foreach (var item in parts.Length == 2 ? parts[1].Split(',') : []) + { + if (!int.TryParse(item, NumberStyles.None, CultureInfo.InvariantCulture, out var index) || index >= next) return (0, []); + if (!retry.Contains(index)) retry.Add(index); + } + return (next, retry); + } + + internal static string Format(int next, IReadOnlyCollection failed) => + failed.Count == 0 + ? next.ToString(CultureInfo.InvariantCulture) + : string.Create(CultureInfo.InvariantCulture, $"{next};{string.Join(',', failed.Order())}"); } diff --git a/src/Titano.Data/Processing/Jobs/ManagedRenderJobHandler.cs b/src/Titano.Data/Processing/Jobs/ManagedRenderJobHandler.cs new file mode 100644 index 00000000..0dfb22fa --- /dev/null +++ b/src/Titano.Data/Processing/Jobs/ManagedRenderJobHandler.cs @@ -0,0 +1,40 @@ +using Titano.Core.Export; +using Titano.Core.Model; +using Titano.Core.Video; +using Titano.Data.Jobs; + +namespace Titano.Data.Processing.Jobs; + +/// +/// on the server (BRIEF §4.4, §14.5): it takes only jobs whose every output (the preset and +/// each batch preset) its managed encoders produce, and runs them like a client does. Jobs that need a platform encoder +/// (MP4) stay queued for render nodes (§14.7). Jobs whose parameters cannot be read are taken, so they fail with their +/// reason instead of waiting for a claimer that never comes. +/// +internal sealed class ManagedRenderJobHandler(RenderJobHandler render, IReadOnlyList encoders) : ISelectiveJobHandler +{ + private readonly EncoderCapability[] _capabilities = [.. encoders.SelectMany(e => e.Capabilities)]; + + public JobKind Kind => JobKind.Render; + + public bool Accepts(Job job) + { + ArgumentNullException.ThrowIfNull(job); + try + { + var parameters = ProcessingJobs.ReadParameters(job.ParametersJson); + // Most declined jobs are decided by the main codec, before the export is read. + return _capabilities.Any(c => string.Equals(ProcessingJobs.CodecName(c.Codec), parameters.Codec, StringComparison.OrdinalIgnoreCase)) + && RenderJobHandler.Targets(ProcessingJobs.ReadExport(parameters.PresetJson), parameters.PresetName).All(t => Produces(t.Export)); + } + catch (InvalidDataException) + { + return true; + } + } + + public Task RunAsync(JobContext context) => render.RunAsync(context); + + private bool Produces(ResolvedExport export) => + _capabilities.Any(c => c.Codec == export.Codec && export.Width <= c.MaxWidth && export.Height <= c.MaxHeight); +} diff --git a/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs b/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs index 515ea14c..c8ea860b 100644 --- a/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs +++ b/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs @@ -13,7 +13,7 @@ namespace Titano.Data.Processing.Jobs; /// RenderJobParameters whose preset JSON is the export section of a project (preset, overrides, name template, /// folder, batch). The project is the embedded one or the sequence's own; frames flagged missing, corrupt, excluded or /// undecodable are left out. The preset and every batch preset render after one preparation pass into the sequence's -/// exports/ (or the export folder), named by the template, and each finished output is recorded at once. +/// exports/ (or the export folder, never on a server), named by the template, and each finished output is recorded at once. /// Checkpoint: the batch entries recorded and the output being written, so a crashed attempt's leftovers are removed and /// only unfinished entries render again. A failed entry fails the job after the others ran; retrying renders it alone. /// @@ -31,6 +31,8 @@ internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions opti var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); + // A server writes only inside its archive (BRIEF §14.2): an export folder chosen on a client means nothing there. + if (options.Host == EncoderHost.Server) export = export with { Folder = null }; var project = (parameters.SettingsJson is { } embedded ? ProjectStore.Parse(embedded) : ProjectStore.Read(sequence, folder)) with { Export = export }; var settingsJson = project.ToJson(); @@ -74,18 +76,12 @@ internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions opti } /// The job's preset, then every other preset of the batch with its own values. - internal static List Targets(ExportSettings export, string presetName) - { - var main = ExportResolver.Resolve(export); - var targets = new List { new(main, presetName) }; - foreach (var id in export.Batch.Distinct(StringComparer.OrdinalIgnoreCase)) - { - if (id.Equals(main.BasePresetId, StringComparison.OrdinalIgnoreCase)) continue; - var resolved = ExportResolver.Resolve(new ExportSettings { Preset = new Auto().WithManual(id) }); - targets.Add(new ExportTarget(resolved, ProcessingJobs.PresetName(resolved))); - } - return targets; - } + internal static List Targets(ExportSettings export, string presetName) => + [ + new(ExportResolver.Resolve(export), presetName), + .. ProcessingJobs.BatchPresets(export).Select(id => ExportResolver.Resolve(ProcessingJobs.BatchExport(export, id))) + .Select(resolved => new ExportTarget(resolved, ProcessingJobs.PresetName(resolved))), + ]; /// Archive-relative for outputs inside the archive (like the server's renders), absolute otherwise. private static string Stored(ArchivePaths paths, string output) diff --git a/src/Titano.Data/Processing/ProcessingJobs.cs b/src/Titano.Data/Processing/ProcessingJobs.cs index 2dac8db8..6d65076d 100644 --- a/src/Titano.Data/Processing/ProcessingJobs.cs +++ b/src/Titano.Data/Processing/ProcessingJobs.cs @@ -44,6 +44,32 @@ public static class ProcessingJobs JsonSerializer.Serialize(parameters, ProtocolJson.Default.RenderJobParameters), JobPriority.User); } + /// + /// One job per preset of : its preset, then each batch preset with its + /// own values. For a server: a render node uploads one file per job, so a batch must not travel as one job. + /// + public static IReadOnlyList RenderEach(Sequence sequence, ExportSettings export, int sourceWidth, int sourceHeight, + ProjectSettings? project = null, bool useProxies = false) + { + ArgumentNullException.ThrowIfNull(export); + return + [ + Render(sequence, export with { Batch = [] }, sourceWidth, sourceHeight, project, useProxies), + .. BatchPresets(export).Select(id => Render(sequence, BatchExport(export, id), sourceWidth, sourceHeight, project, useProxies)), + ]; + } + + /// The batch presets of an export besides its own, each once. + internal static IEnumerable BatchPresets(ExportSettings export) + { + var main = ExportResolver.Resolve(export).BasePresetId; + return export.Batch.Distinct(StringComparer.OrdinalIgnoreCase).Where(id => !id.Equals(main, StringComparison.OrdinalIgnoreCase)); + } + + /// A batch preset rendered with its own values, named and placed like the export it belongs to. + internal static ExportSettings BatchExport(ExportSettings export, string presetId) => + new() { Preset = new Auto().WithManual(presetId), NameTemplate = export.NameTemplate, Folder = export.Folder }; + /// Codec name in job parameters and render-node capabilities ("h264", "hevc", "mjpeg", "pngsequence"…). public static string CodecName(VideoCodec codec) => codec.ToString().ToLowerInvariant(); @@ -57,11 +83,20 @@ public static class ProcessingJobs /// The parameters of a render job and the export they select. /// The job has no valid render parameters. internal static (RenderJobParameters Parameters, ExportSettings Export) ReadRender(string? parametersJson) + { + var parameters = ReadParameters(parametersJson); + return (parameters, ReadExport(parameters.PresetJson)); + } + + /// The protocol record of a render job, without reading its export. + /// The job has no valid render parameters. + internal static RenderJobParameters ReadParameters(string? parametersJson) { try { - if (parametersJson is { Length: > 0 } && JsonSerializer.Deserialize(parametersJson, ProtocolJson.Default.RenderJobParameters) is { } parameters) - return (parameters, ReadExport(parameters.PresetJson)); + if (parametersJson is { Length: > 0 } && JsonSerializer.Deserialize(parametersJson, ProtocolJson.Default.RenderJobParameters) + is { PresetName.Length: > 0, PresetJson.Length: > 0, Codec.Length: > 0 } parameters) + return parameters; } catch (JsonException ex) { diff --git a/src/Titano.Data/Processing/ProcessingService.cs b/src/Titano.Data/Processing/ProcessingService.cs index 3a06bd42..b5033dfd 100644 --- a/src/Titano.Data/Processing/ProcessingService.cs +++ b/src/Titano.Data/Processing/ProcessingService.cs @@ -1,3 +1,4 @@ +using Titano.Core.Export; using Titano.Core.Model; using Titano.Data.Archiving; using Titano.Data.Jobs; @@ -8,8 +9,9 @@ namespace Titano.Data.Processing; /// /// The analysis and render engine of a host (BRIEF §4.3, §10, §11): job handlers for light analysis, deep analysis and /// renders over the archive in the index, built with the host's decoders and encoders (). -/// Register it on the next to the archive engine; a server engine leaves render jobs to render -/// nodes. Create jobs with . +/// Register it on the next to the archive engine. A client renders every job it is given; a +/// server takes only the renders its managed encoders produce and leaves the others to render nodes. Create jobs with +/// . /// public sealed class ProcessingService : IJobHandlerProvider { @@ -28,10 +30,14 @@ public sealed class ProcessingService : IJobHandlerProvider public ProcessingOptions Options { get; } /// Handlers for , and . - public IReadOnlyList CreateJobHandlers() => - [ - new AnalyseJobHandler(_db, _analyser, JobKind.LightAnalyse), - new AnalyseJobHandler(_db, _analyser, JobKind.DeepAnalyse), - new RenderJobHandler(_db, Options), - ]; + public IReadOnlyList CreateJobHandlers() + { + var render = new RenderJobHandler(_db, Options); + return + [ + new AnalyseJobHandler(_db, _analyser, JobKind.LightAnalyse), + new AnalyseJobHandler(_db, _analyser, JobKind.DeepAnalyse), + Options.Host == EncoderHost.Server ? new ManagedRenderJobHandler(render, Options.Encoders) : render, + ]; + } } diff --git a/src/Titano.Data/Processing/RenderNodeClient.cs b/src/Titano.Data/Processing/RenderNodeClient.cs index 8361caab..dbad0d8a 100644 --- a/src/Titano.Data/Processing/RenderNodeClient.cs +++ b/src/Titano.Data/Processing/RenderNodeClient.cs @@ -8,6 +8,7 @@ using Titano.Core.Localization; using Titano.Core.Model; using Titano.Core.Render; using Titano.Core.Video; +using Titano.Data.Processing.Jobs; using Titano.Protocol; using Titano.Protocol.Client; @@ -148,9 +149,9 @@ public sealed class RenderNodeClient : IAsyncDisposable { break; } - catch (TitanoApiException ex) when (ex.Status is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden) + catch (Exception ex) when (ex is TitanoApiException { Status: HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden } or UntrustedCertificateException) { - // Signed out, or the account lacks the render-node right: nothing to retry until the user acts. + // Signed out, no render-node right, or a changed certificate: nothing to retry until the user acts. Log.Warning(LogCategory, "The server refused this render node", ex); break; } @@ -193,8 +194,9 @@ public sealed class RenderNodeClient : IAsyncDisposable { _nodeId = null; } - catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException or UntrustedCertificateException) + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) { + // Unreachable or refusing: the claim loop decides whether to go on; beating must not stop the node. Log.Warning(LogCategory, "Heartbeat failed", ex); } } @@ -211,6 +213,9 @@ public sealed class RenderNodeClient : IAsyncDisposable var target = new ExportTarget(resolved, ProcessingJobs.PresetName(resolved)); if (target.Export.Container == ContainerFormat.ImageSequence) throw new InvalidOperationException(Loc.L("Errors.Render.NodeSequence")); var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export }; + // One output per job: batch presets were meant for separate jobs (ProcessingJobs.RenderEach). + if (RenderJobHandler.Targets(export, target.PresetName).Skip(1).Select(t => t.PresetName).ToList() is { Count: > 0 } skipped) + await reporter.ReportAsync(0, claimed.Frames.Count, Loc.L("Export.Log.NodeBatch", string.Join(", ", skipped))).ConfigureAwait(false); var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false)); var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false); @@ -257,21 +262,28 @@ public sealed class RenderNodeClient : IAsyncDisposable /// /// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused - /// across jobs), or proxies into the job folder when the job asks for them and the server has them. + /// across jobs, fetched once per content), or proxies into the job folder when the job asks for them and the server + /// has them. /// private async Task> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken) { var files = new Dictionary(); var done = 0; - await reporter.ReportAsync(0, claimed.Frames.Count, Loc.L("Export.Job.Downloading")).ConfigureAwait(false); - await Parallel.ForEachAsync(claimed.Frames, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken }, - async (frame, ct) => + var total = claimed.Frames.Count; + await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false); + var downloads = claimed.Frames.GroupBy(f => claimed.UseProxies && f.HasProxy ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList(); + await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken }, + async (frames, ct) => { + var frame = frames.First(); var path = claimed.UseProxies && frame.HasProxy ? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false) : await DownloadOriginalAsync(frame, ct).ConfigureAwait(false); - lock (files) files[frame.Id] = path; - await reporter.ReportAsync(Interlocked.Increment(ref done), claimed.Frames.Count, null).ConfigureAwait(false); + lock (files) + { + foreach (var f in frames) files[f.Id] = path; + } + await reporter.ReportAsync(Interlocked.Add(ref done, frames.Count()), total, null).ConfigureAwait(false); }).ConfigureAwait(false); return files; } @@ -309,8 +321,12 @@ public sealed class RenderNodeClient : IAsyncDisposable var part = path + ".part"; for (var attempt = 1; ; attempt++) { - await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + // A part as long as the file was complete when the last attempt stopped: only its hash is left to check. + if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) + { + await using var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); await _client.DownloadFrameAsync(frame.Id, "original", file, null, cancellationToken).ConfigureAwait(false); + } if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha) { File.Move(part, path, overwrite: true); @@ -406,7 +422,7 @@ public sealed class RenderNodeClient : IAsyncDisposable /// /// Progress of the job in flight to the server, at most every interval (and at the end). A refusal (409) means the - /// job was cancelled or given to another node: is set and the job's token cancelled. + /// job was cancelled or given to another node: the job's token is cancelled. /// private sealed class NodeReporter(TitanoClient client, Guid nodeId, Guid jobId, TimeSpan interval, CancellationTokenSource job) { @@ -418,8 +434,6 @@ public sealed class RenderNodeClient : IAsyncDisposable public Guid NodeId => nodeId; - public bool Lost { get; private set; } - public async ValueTask ReportAsync(long done, long total, string? message) { await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false); @@ -435,7 +449,6 @@ public sealed class RenderNodeClient : IAsyncDisposable } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict) { - Lost = true; await job.CancelAsync().ConfigureAwait(false); } catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException) diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index ef663e28..056d01d1 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -44,13 +44,15 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti ? await AnalyseDeepAsync(frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false) : ReadReport(folder, frames.Count); var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs); - var advice = Advisor.Recommend(light, report, context); - var warnings = advice.Findings.Count(f => f.Severity >= FindingSeverity.Warning); + var warnings = Advisor.Recommend(light, report, context).Findings.Count(f => f.Severity >= FindingSeverity.Warning); bool changed, unreadable = false; try { - changed = await projects.UpdateAsync(paths, sequence, settings => Advisor.Apply(settings, advice), cancellationToken).ConfigureAwait(false); + // The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3). + changed = await projects.UpdateAsync(paths, sequence, + settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })), + cancellationToken).ConfigureAwait(false); } catch (InvalidDataException ex) { diff --git a/src/Titano.Data/Repositories/JobRepository.cs b/src/Titano.Data/Repositories/JobRepository.cs index 0a663325..1226f9dc 100644 --- a/src/Titano.Data/Repositories/JobRepository.cs +++ b/src/Titano.Data/Repositories/JobRepository.cs @@ -39,9 +39,7 @@ public sealed class JobRepository /// public Task ClaimNextAsync(string workerId, IReadOnlyCollection? kinds = null, CancellationToken cancellationToken = default) { - var kindFilter = kinds is { Count: > 0 } - ? " AND kind IN (" + string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture))) + ")" - : string.Empty; + var kindFilter = kinds is { Count: > 0 } ? $" AND kind IN ({KindList(kinds)})" : string.Empty; var pick = $"SELECT id FROM jobs WHERE state = {(int)JobState.Queued}{kindFilter} ORDER BY priority DESC, created, id LIMIT 1"; var claim = $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $1, started = $2, finished = NULL, error = NULL, attempts = attempts + 1"; DbParam[] args = [workerId, DateTimeOffset.UtcNow]; @@ -130,9 +128,13 @@ public sealed class JobRepository /// Queued jobs of one kind in claim order (highest priority, then oldest). public Task> ListQueuedAsync(JobKind kind, int limit = 50, CancellationToken cancellationToken = default) => - _db.ListAsync( - $"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind = $1 ORDER BY priority DESC, created, id LIMIT $2", - [(int)kind, Cursor.ClampLimit(limit)], Map, cancellationToken); + ListQueuedAsync([kind], limit, cancellationToken); + + /// Queued jobs of the given kinds in claim order (highest priority, then oldest). + public Task> ListQueuedAsync(IReadOnlyCollection kinds, int limit = 50, CancellationToken cancellationToken = default) => + kinds.Count == 0 ? Task.FromResult>([]) : _db.ListAsync( + $"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY priority DESC, created, id LIMIT $1", + [Cursor.ClampLimit(limit)], Map, cancellationToken); /// Running jobs claimed by . public Task> ListRunningByNodeAsync(string nodeId, CancellationToken cancellationToken = default) => @@ -193,6 +195,8 @@ public sealed class JobRepository return new JobCounts(row.GetInt32(0), row.GetInt32(1), row.GetInt32(2), row.GetInt32(3), row.GetInt32(4)); }, cancellationToken); + private static string KindList(IEnumerable kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture))); + private static Job Map(DbRow r) => new() { Id = r.GetGuid(0), diff --git a/src/Titano.Server/Jobs/JobEngineHost.cs b/src/Titano.Server/Jobs/JobEngineHost.cs index e73b989b..2c5d5faf 100644 --- a/src/Titano.Server/Jobs/JobEngineHost.cs +++ b/src/Titano.Server/Jobs/JobEngineHost.cs @@ -12,8 +12,9 @@ namespace Titano.Server.Jobs; /// /// Hosts the server's (BRIEF §14.6) with the s registered in DI (directly or /// through s) and -/// relays its state changes and progress as WebSocket events. Render jobs are never run here: render nodes claim them -/// (BRIEF §14.7). On shutdown running jobs stay marked running and are requeued from their checkpoints at the next start. +/// relays its state changes and progress as WebSocket events. Render jobs run here only when the registered render +/// handler accepts them (managed formats, ); render nodes claim the others (BRIEF +/// §14.5, §14.7). On shutdown running jobs stay marked running and are requeued from their checkpoints at the next start. /// public sealed class JobEngineHost : IHostedService { @@ -30,7 +31,8 @@ public sealed class JobEngineHost : IHostedService { _db = db; _events = events; - var local = handlers.Concat(providers.SelectMany(p => p.CreateJobHandlers())).Where(h => h.Kind != JobKind.Render).ToList(); + // A render handler must be selective: a plain one would take the jobs that need a render node's encoder. + var local = handlers.Concat(providers.SelectMany(p => p.CreateJobHandlers())).Where(h => h.Kind != JobKind.Render || h is ISelectiveJobHandler).ToList(); _kinds = [.. local.Select(h => h.Kind)]; // Without handlers no worker starts: an engine with an empty kind list would claim every kind. Workers = _kinds.Count == 0 ? 0 : options.Workers; diff --git a/src/Titano.Server/Program.cs b/src/Titano.Server/Program.cs index 06d2e08d..caed2df6 100644 --- a/src/Titano.Server/Program.cs +++ b/src/Titano.Server/Program.cs @@ -26,15 +26,22 @@ switch (command.Command) case ServerCommand.UninstallService when OperatingSystem.IsWindows(): return ServiceAction(ServiceInstaller.Uninstall, $"Removed the {WindowsServiceHost.ServiceName} service."); case ServerCommand.Service when OperatingSystem.IsWindows(): - if (WindowsServiceHost.Run(() => ServerHost.Build(ServerOptions.Load(), [.. command.HostArguments], ArchiveModule.Register))) return 0; + if (WindowsServiceHost.Run(() => ServerHost.Build(ServerOptions.Load(), [.. command.HostArguments], RegisterModules))) return 0; await Console.Error.WriteLineAsync("--service is used by the Service Control Manager; start the service with 'sc start TitanoServer'."); return 1; default: - var app = ServerHost.Build(ServerOptions.Load(), [.. command.HostArguments], ArchiveModule.Register); + var app = ServerHost.Build(ServerOptions.Load(), [.. command.HostArguments], RegisterModules); await app.RunAsync(); return 0; } +// The archive engine, then analysis and managed rendering over it. +static void RegisterModules(IServiceCollection services) +{ + ArchiveModule.Register(services); + ProcessingModule.Register(services); +} + static int ServiceAction(Action action, string done) { try diff --git a/src/Titano.Server/Storage/ArchiveModule.cs b/src/Titano.Server/Storage/ArchiveModule.cs index d1738a91..3002c9d4 100644 --- a/src/Titano.Server/Storage/ArchiveModule.cs +++ b/src/Titano.Server/Storage/ArchiveModule.cs @@ -7,8 +7,8 @@ namespace Titano.Server.Storage; /// /// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and -/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance), plus the -/// analysis handlers of . Decoding is managed only (BRIEF §4.2). +/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance). Decoding is +/// managed only (BRIEF §4.2). /// public static class ArchiveModule { @@ -22,6 +22,5 @@ public static class ArchiveModule services.AddSingleton(sp => sp.GetRequiredService().Ingest); services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); - ProcessingModule.Register(services); } } diff --git a/src/Titano.Server/Storage/ProcessingModule.cs b/src/Titano.Server/Storage/ProcessingModule.cs index 591ef564..1a241bbe 100644 --- a/src/Titano.Server/Storage/ProcessingModule.cs +++ b/src/Titano.Server/Storage/ProcessingModule.cs @@ -10,9 +10,10 @@ using Titano.Server.Configuration; namespace Titano.Server.Storage; /// -/// Analysis and render engine of the server (BRIEF §4.2, §10, §14.6): light and deep analysis job handlers with managed -/// decoders, so the light analysis queued after uploads runs here. Its render handler is registered too, but the job -/// engine host never runs render jobs: render nodes claim them (BRIEF §14.7). +/// Analysis and render engine of the server (BRIEF §4.2, §10, §14.5, §14.6): light and deep analysis with managed +/// decoders (the light analysis queued after uploads runs here) and renders with managed encoders only. The render +/// handler takes just the jobs whose every output a managed encoder produces (AVI MJPEG, image sequences, GIF, APNG); +/// MP4 jobs stay queued for render nodes (BRIEF §14.7). /// public static class ProcessingModule { diff --git a/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs index 4820dc60..df22bc0b 100644 --- a/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs +++ b/tests/Titano.Data.Tests/Jobs/JobEngineTests.cs @@ -27,6 +27,25 @@ public sealed class JobEngineTests(TestContext context) } } + /// Takes only jobs whose parameters say they are its own. + private sealed class SelectiveHandler(JobKind kind) : ISelectiveJobHandler + { + public const string Mine = """{"mine":true}"""; + + public JobKind Kind => kind; + + public bool Accepts(Job job) => job.ParametersJson == Mine; + + public Task RunAsync(JobContext c) => Task.CompletedTask; + } + + private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler + { + public JobKind Kind => kind; + + public Task RunAsync(JobContext context) => run(context); + } + private async Task OpenAsync() => await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation); @@ -159,6 +178,50 @@ public sealed class JobEngineTests(TestContext context) Assert.Equal(JobKind.Render, pending[0].Kind); } + [Test] + public async Task SelectiveHandlersLeaveDeclinedJobsQueuedWithoutHoldingUpOthers() + { + await using var db = await OpenAsync(); + await using var engine = new JobEngine(db, [new SelectiveHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)], + new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) }); + var declined = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: """{"mine":false}""", priority: JobPriority.User); + var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.Background); + var accepted = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: SelectiveHandler.Mine, priority: JobPriority.User); + await engine.StartAsync(); + + var render = await WaitForStateAsync(db, accepted.Id, JobState.Done, TimeSpan.FromSeconds(10)); + var other = await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10)); + Assert.True(render.Started <= other.Started, "claim order: priority first, across selective and other kinds"); + await Task.Delay(200); + var waiting = (await db.Jobs.GetAsync(declined.Id))!; + Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId)); + } + + [Test] + public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns() + { + await using var db = await OpenAsync(); + var target = Guid.CreateVersion7(); + Job? thumbs = null, light = null; + var scan = new CallbackHandler(JobKind.Scan, async c => + { + thumbs = await c.EnqueueFollowUpAsync(JobKind.Thumbs, [target], "Roof", null, JobPriority.Background); + light = await c.EnqueueFollowUpAsync(JobKind.LightAnalyse, [target], "Roof", null, JobPriority.Background); + }); + await using var engine = new JobEngine(db, [scan, new CountingHandler(JobKind.Thumbs, 1)], + new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) }); + await engine.StartAsync(); + var user = Guid.CreateVersion7(); + var job = await engine.EnqueueAsync(JobKind.Scan, [], createdBy: user); + await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10)); + + Assert.Null(light); + var followUp = await WaitForStateAsync(db, Assert.NotNull(thumbs).Id, JobState.Done, TimeSpan.FromSeconds(10)); + Assert.Equal((user, JobPriority.Background, "Roof"), (followUp.CreatedBy!.Value, followUp.Priority, followUp.TargetLabel!)); + Assert.SequenceEqual([target], followUp.TargetIds); + Assert.Empty((await db.Jobs.ListAsync(null, JobKind.LightAnalyse)).Items); + } + [Test] public async Task HandlerFailureMarksJobFailedWithMessage() { diff --git a/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs new file mode 100644 index 00000000..0f9cad44 --- /dev/null +++ b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs @@ -0,0 +1,65 @@ +using Titano.Core.Archiving; +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Import; +using Titano.Core.Model; +using Titano.Core.Project; +using Titano.Data.Archiving; +using Titano.Data.Tests.Archiving; +using Titano.Testing; + +namespace Titano.Data.Tests.Processing; + +/// New sequences get their light analysis (and so their Auto values) without any user action (BRIEF §1.1). +[Category("processing")] +public sealed class AnalysisFollowUpTests(TestContext context) +{ + [Test] + public async Task AnAdoptingScanQueuesLightAnalysisOfTheNewSequencesOnly() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof"); + await fx.StartEngineAsync(withArchive: true); + + var scan = await fx.EnqueueAsync(ArchiveJobs.Scan(fx.Archive.Archive, full: true)); + await fx.WaitForAsync(scan.Id, JobState.Done); + var light = Assert.NotNull(await SingleLightJobAsync(fx)); + await fx.WaitForAsync(light.Id, JobState.Done); + + var sequence = (await fx.Db.Sequences.GetAsync(light.TargetIds.Single(), fx.Ct))!; + Assert.Equal("Roof", sequence.Folder); + Assert.True(sequence.Status.HasFlag(SequenceStatus.AnalyzedLight), "analysed"); + Assert.True(ProjectSettings.FromJson(await File.ReadAllTextAsync(fx.Folder(sequence).ProjectPath, fx.Ct)).Timing.Mode.HasAdvice, "advised"); + + // Scans that find nothing new queue nothing more. + foreach (var full in new[] { false, true }) + { + var again = await fx.EnqueueAsync(ArchiveJobs.Scan(fx.Archive.Archive, full)); + await fx.WaitForAsync(again.Id, JobState.Done); + } + Assert.Equal(light.Id, (await SingleLightJobAsync(fx))?.Id); + } + + [Test] + public async Task AnImportQueuesLightAnalysisOfTheSequencesItFilled() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var card = context.CreateTempDirectory("card"); + Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), ArchiveFixture.Start, 10); + var plan = await fx.Archive.Service.PlanImportAsync(fx.Archive.Archive.Id, card, new ImportOptions(), null, null, fx.Ct); + await fx.StartEngineAsync(withArchive: true); + + var import = await fx.Engine.EnqueueAsync(ArchiveJobs.Import(fx.Archive.Archive, plan, null), createdBy: Guid.CreateVersion7(), cancellationToken: fx.Ct); + await fx.WaitForAsync(import.Id, JobState.Done); + var light = Assert.NotNull(await SingleLightJobAsync(fx)); + await fx.WaitForAsync(light.Id, JobState.Done); + + var sequenceId = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Archive.Id, plan.Sessions.Single().SessionKey, fx.Ct); + Assert.SequenceEqual([Assert.NotNull(sequenceId)], light.TargetIds); + Assert.Equal(import.CreatedBy, light.CreatedBy); + Assert.Equal(JobPriority.Background, light.Priority); + Assert.True((await fx.Db.Sequences.GetAsync(sequenceId.Value, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedLight), "analysed"); + } + + private static async Task SingleLightJobAsync(ProcessingFixture fx) => + (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.SingleOrDefault(); +} diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs index 7673915d..1f03927c 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs @@ -5,6 +5,7 @@ using Titano.Core.Localization; using Titano.Core.Model; using Titano.Core.Project; using Titano.Data.Processing; +using Titano.Data.Processing.Jobs; using Titano.Data.Repositories; using Titano.Testing; @@ -105,6 +106,55 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Contains(Loc.L("Advisor.Log.Presets", string.Empty), done.LogTail); } + [Test] + public async Task ASequenceThatFailsDoesNotStopTheOthersAndARetryRedoesOnlyIt() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var a = await fx.AddAsync(Flickering(10), "A"); + var b = await fx.AddAsync(Flickering(10), "B"); + var c = await fx.AddAsync(Flickering(10), "C"); + // B's sidecar is gone for a while: its status cannot be recorded. + var sidecar = fx.Folder(b).ManifestPath; + var saved = await File.ReadAllBytesAsync(sidecar, fx.Ct); + File.Delete(sidecar); + await fx.StartEngineAsync(); + + var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([a.Id, b.Id, c.Id])); + var failed = await fx.WaitForAsync(job.Id, JobState.Failed); + Assert.Equal(Loc.L("Errors.Analysis.Failed", 1, 3), failed.Error); + Assert.Equal("3;1", failed.Checkpoint); + Assert.SequenceEqual([true, false, true], await AnalysedAsync(fx, a, b, c)); + + await File.WriteAllBytesAsync(sidecar, saved, fx.Ct); + await fx.Engine.ResumeAsync(job.Id, fx.Ct); + var done = await fx.WaitForAsync(job.Id, JobState.Done); + Assert.SequenceEqual([true, true, true], await AnalysedAsync(fx, a, b, c)); + Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal)); + // The retry analysed B alone. + var lightOfA = Loc.L("Advisor.Log.LightDone", a.DisplayTitle, 10, 0).Split(',')[0]; + Assert.Equal(1, done.LogTail!.Split('\n').Count(line => line.Contains(lightOfA, StringComparison.Ordinal))); + } + + [Test] + public void AnalysisCheckpointsKeepTheFailedTargets() + { + Assert.Equal("3", AnalyseJobHandler.Format(3, [])); + Assert.Equal("4;0,2", AnalyseJobHandler.Format(4, [2, 0])); + var (next, retry) = AnalyseJobHandler.Parse("4;0,2", 5); + Assert.Equal(4, next); + Assert.SequenceEqual([0, 2], retry); + Assert.Equal(2, AnalyseJobHandler.Parse("2", 5).Next); + // Anything else starts over: no checkpoint, beyond the targets, a failed target not yet reached, garbage. + foreach (var text in new[] { null, "6", "2;3", "2;x", "a" }) Assert.Equal(0, AnalyseJobHandler.Parse(text, 5).Next); + } + + private static async Task> AnalysedAsync(ProcessingFixture fx, params Sequence[] sequences) + { + var analysed = new List(); + foreach (var s in sequences) analysed.Add((await fx.Db.Sequences.GetAsync(s.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedLight)); + return analysed; + } + [Test] public async Task LightAnalysisAfterNewFramesDropsTheStaleDeepStatus() { diff --git a/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs b/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs index 39939b2e..78ed3a47 100644 --- a/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs +++ b/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs @@ -45,10 +45,10 @@ internal sealed class ProcessingFixture : IAsyncDisposable public static async Task CreateAsync(TestContext context) => new(await ArchiveFixture.CreateAsync(context), new GateDecoder()); - /// A started engine with the processing handlers (one worker, fast checkpoints). - public async Task StartEngineAsync() + /// A started engine with the processing handlers, and the archive's when asked (one worker, fast checkpoints). + public async Task StartEngineAsync(bool withArchive = false) { - _engine = new JobEngine(Db, Processing.CreateJobHandlers(), + _engine = new JobEngine(Db, [.. withArchive ? Archive.Service.CreateJobHandlers() : [], .. Processing.CreateJobHandlers()], new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); await _engine.StartAsync(Ct); return _engine; @@ -61,6 +61,15 @@ internal sealed class ProcessingFixture : IAsyncDisposable /// Writes frames … of as JPEGs with their Exif into /// (adopted in place) and indexes the archive. public async Task AddAsync(SyntheticSequence sequence, string folder = "Synthetic", int first = 0) + { + Write(sequence, folder, first); + await Archive.ScanAsync(ScanMode.Full); + var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder); + return (await Db.Sequences.GetAsync(location.Id, Ct))!; + } + + /// Writes frames … of as JPEGs with their Exif into . + public void Write(SyntheticSequence sequence, string folder = "Synthetic", int first = 0) { var directory = Path.Combine(Archive.Paths.Root, folder); Directory.CreateDirectory(directory); @@ -70,9 +79,6 @@ internal sealed class ProcessingFixture : IAsyncDisposable var exif = ExifWriter.BuildJpegApp1(sequence.Metadata[i]); File.WriteAllBytes(Path.Combine(directory, FileName(i)), JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 95, App1Segments = [exif[4..]] })); } - await Archive.ScanAsync(ScanMode.Full); - var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder); - return (await Db.Sequences.GetAsync(location.Id, Ct))!; } public static string FileName(int index) => $"IMG_{index + 1:0000}.JPG"; diff --git a/tests/Titano.Data.Tests/Processing/RenderJobTests.cs b/tests/Titano.Data.Tests/Processing/RenderJobTests.cs index bf1bc8a7..81307a33 100644 --- a/tests/Titano.Data.Tests/Processing/RenderJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/RenderJobTests.cs @@ -6,6 +6,7 @@ using Titano.Core.Imaging.Decoders; using Titano.Core.Localization; using Titano.Core.Model; using Titano.Core.Project; +using Titano.Data.Jobs; using Titano.Data.Processing; using Titano.Testing; @@ -145,6 +146,37 @@ public sealed class RenderJobTests(TestContext context) Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)); } + [Test] + public async Task AServerTakesOnlyTheRendersItsManagedEncodersProduce() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var server = new ProcessingService(fx.Db, new ProcessingOptions { Host = EncoderHost.Server }); + var handler = Assert.NotNull(server.CreateJobHandlers().Single(h => h.Kind == JobKind.Render) as ISelectiveJobHandler); + var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic" }; + bool Accepts(ExportSettings export) => + handler.Accepts(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, ParametersJson = ProcessingJobs.Render(sequence, export, 96, 64).ParametersJson }); + + Assert.True(Accepts(Mjpeg with { Batch = ["png-sequence", "gif-preview"] }), "managed outputs only"); + Assert.False(Accepts(new ExportSettings { Preset = new Auto().WithManual("4k-hevc") }), "HEVC needs a render node"); + Assert.False(Accepts(Mjpeg with { Batch = ["1080p-h264"] }), "one MP4 in the batch"); + Assert.True(handler.Accepts(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, ParametersJson = "{}" }), "unreadable: fails with its reason"); + + var client = new ProcessingService(fx.Db).CreateJobHandlers().Single(h => h.Kind == JobKind.Render); + Assert.False(client is ISelectiveJobHandler, "a client renders whatever it is given"); + } + + [Test] + public void ABatchForAServerBecomesOneJobPerPreset() + { + var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic", Title = "Roof" }; + var jobs = ProcessingJobs.RenderEach(sequence, Mjpeg with { Batch = ["mjpeg-avi", "gif-preview", "GIF-preview", "4k-hevc"] }, 96, 64); + var parameters = jobs.Select(j => System.Text.Json.JsonSerializer.Deserialize(j.ParametersJson!, Titano.Protocol.ProtocolJson.Default.RenderJobParameters)!).ToList(); + Assert.SequenceEqual(["mjpeg", "gif", "hevc"], parameters.Select(p => p.Codec)); + Assert.True(jobs.All(j => j.Kind == JobKind.Render && j.Targets.SequenceEqual([sequence.Id])), "render jobs of the sequence"); + Assert.True(parameters.All(p => ProcessingJobs.ReadExport(p.PresetJson).Batch.Count == 0), "no job carries a batch"); + Assert.Equal((3840, 2160), (parameters[2].Width, parameters[2].Height)); + } + private static List Entries(string folder) => Directory.Exists(folder) ? [.. Directory.EnumerateFileSystemEntries(folder).Select(e => Path.GetFileName(e)).Order(StringComparer.Ordinal)] : []; diff --git a/tests/Titano.Server.Tests/Storage/ProcessingModuleTests.cs b/tests/Titano.Server.Tests/Storage/ProcessingModuleTests.cs index 821c4155..61307753 100644 --- a/tests/Titano.Server.Tests/Storage/ProcessingModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ProcessingModuleTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.DependencyInjection; using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; using Titano.Core.Diagnostics.Synthetic; @@ -13,10 +14,17 @@ namespace Titano.Server.Tests.Storage; [Category("processing")] public sealed class ProcessingModuleTests(TestContext context) { + /// The registrations of Program.cs. + private static void Production(IServiceCollection services) + { + ArchiveModule.Register(services); + ProcessingModule.Register(services); + } + [Test] public async Task UploadedFramesAreLightAnalysedAndAdvised() { - await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + await using var server = await TestServer.StartAsync(context, services: Production); var (editor, _, _) = await server.UserAsync("alberto", "editor"); var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 10, StartEv = 0.5, EndEv = -0.5 }); Guid sequenceId = default;