feat(processing): selective render claims, analysis follow-ups, per-sequence failures

- JobEngine: ISelectiveJobHandler lets a host decline jobs of a kind it runs;
  declined jobs stay queued for other claimers without holding up the rest.
  JobContext.EnqueueFollowUpAsync queues follow-up work on the same engine.
- Server renders the jobs whose every output a managed encoder produces
  (ManagedRenderJobHandler); MP4 jobs stay for render nodes. Program.cs
  registers ProcessingModule next to ArchiveModule. Server renders ignore a
  client's export folder.
- Light analysis is queued after imports and after scans that indexed new
  sequences, so the advisor's Auto values exist without user action.
- Analysis jobs log a failing sequence and go on; the job then fails and a
  retry analyses only the failed ones. The advised frame rate follows the
  user's fit-to duration.
- ProcessingJobs.RenderEach splits a batch into one job per preset for
  servers; render nodes say which batch presets they did not render, fetch
  each content once and stop on a changed certificate.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 17:01:50 +00:00
1 parent d729d2d09c
commit 14cdbf1f7d
30 files changed
+572 -85

No files matched your search

+1
View File
@@ -33,6 +33,7 @@
<data name="Advisor.Finding.WhiteFrames" xml:space="preserve"><value>{0} frames are blown out to white.</value><comment>{0} frame count</comment></data>
<data name="Advisor.Log.AutoUpdated" xml:space="preserve"><value>Automatic values of {0} updated.</value><comment>job log line; {0} sequence title</comment></data>
<data name="Advisor.Log.DeepDone" xml:space="preserve"><value>Deep analysis of {0}: {1} frames, {2} warnings.</value><comment>job log line; {0} sequence title; {1} frame count; {2} warning count</comment></data>
<data name="Advisor.Log.Failed" xml:space="preserve"><value>{0} could not be analysed: {1}</value><comment>long: job log line; {0} sequence title; {1} reason</comment></data>
<data name="Advisor.Log.LightDone" xml:space="preserve"><value>Light analysis of {0}: {1} frames, {2} warnings.</value><comment>job log line; {0} sequence title; {1} frame count; {2} warning count</comment></data>
<data name="Advisor.Log.Presets" xml:space="preserve"><value>Suggested presets: {0}</value><comment>job log line; {0} preset names separated by dots</comment></data>
<data name="Advisor.Log.ProjectUnreadable" xml:space="preserve"><value>The project settings of {0} cannot be read; their automatic values were left as they were.</value><comment>job log line; {0} sequence title</comment></data>
+1
View File
@@ -4,6 +4,7 @@
<resheader name="version"><value>2.0</value></resheader>
<resheader name="reader"><value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
<resheader name="writer"><value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
<data name="Errors.Analysis.Failed" xml:space="preserve"><value>{0} of {1} sequences could not be analysed; see the job log.</value><comment>long; {0} failed count; {1} total count</comment></data>
<data name="Errors.Archive.Open" xml:space="preserve"><value>The archive could not be opened.</value><comment>long</comment></data>
<data name="Errors.Number.Range" xml:space="preserve"><value>Enter a whole number from {0} to {1}.</value></data>
<data name="Errors.Path.Empty" xml:space="preserve"><value>Enter a folder.</value></data>
+1
View File
@@ -16,6 +16,7 @@
<data name="Export.Job.Downloading" xml:space="preserve"><value>Downloading frames</value><comment>render node progress stage</comment></data>
<data name="Export.Job.Preparing" xml:space="preserve"><value>Measuring frames</value><comment>job progress stage</comment></data>
<data name="Export.Log.Failed" xml:space="preserve"><value>{0} failed: {1}</value><comment>long: job log line; {0} preset name; {1} reason</comment></data>
<data name="Export.Log.NodeBatch" xml:space="preserve"><value>A render node renders one preset per job; not rendered: {0}</value><comment>long: job log line; {0} preset names separated by commas</comment></data>
<data name="Export.Log.Rendered" xml:space="preserve"><value>{0} saved as {1}</value><comment>long: job log line; {0} preset name; {1} output file name</comment></data>
<data name="Export.Plan.FrameRateIgnored" xml:space="preserve"><value>Playback rate not understood; the export frame rate is used.</value><comment>long</comment></data>
<data name="Export.Plan.NoCaptureTimes" xml:space="preserve"><value>No capture times: frames are spaced evenly.</value><comment>long</comment></data>
+1
View File
@@ -33,6 +33,7 @@
<data name="Advisor.Finding.WhiteFrames" xml:space="preserve"><value>{0} fotogrammi sono completamente bruciati.</value></data>
<data name="Advisor.Log.AutoUpdated" xml:space="preserve"><value>Valori automatici di {0} aggiornati.</value></data>
<data name="Advisor.Log.DeepDone" xml:space="preserve"><value>Analisi approfondita di {0}: {1} fotogrammi, {2} avvisi.</value></data>
<data name="Advisor.Log.Failed" xml:space="preserve"><value>Impossibile analizzare {0}: {1}</value></data>
<data name="Advisor.Log.LightDone" xml:space="preserve"><value>Analisi rapida di {0}: {1} fotogrammi, {2} avvisi.</value></data>
<data name="Advisor.Log.Presets" xml:space="preserve"><value>Preset suggeriti: {0}</value></data>
<data name="Advisor.Log.ProjectUnreadable" xml:space="preserve"><value>Le impostazioni del progetto di {0} non sono leggibili; i valori automatici sono rimasti invariati.</value></data>
+1
View File
@@ -4,6 +4,7 @@
<resheader name="version"><value>2.0</value></resheader>
<resheader name="reader"><value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
<resheader name="writer"><value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
<data name="Errors.Analysis.Failed" xml:space="preserve"><value>{0} sequenze su {1} non sono state analizzate; vedi il registro del job.</value></data>
<data name="Errors.Archive.Open" xml:space="preserve"><value>Impossibile aprire l'archivio.</value></data>
<data name="Errors.Number.Range" xml:space="preserve"><value>Inserisci un numero intero da {0} a {1}.</value></data>
<data name="Errors.Path.Empty" xml:space="preserve"><value>Inserisci una cartella.</value></data>
+1
View File
@@ -16,6 +16,7 @@
<data name="Export.Job.Downloading" xml:space="preserve"><value>Download dei fotogrammi</value></data>
<data name="Export.Job.Preparing" xml:space="preserve"><value>Misura dei fotogrammi</value></data>
<data name="Export.Log.Failed" xml:space="preserve"><value>{0} non riuscito: {1}</value></data>
<data name="Export.Log.NodeBatch" xml:space="preserve"><value>Un nodo di rendering esegue un preset per job; non renderizzati: {0}</value></data>
<data name="Export.Log.Rendered" xml:space="preserve"><value>{0} salvato come {1}</value></data>
<data name="Export.Plan.FrameRateIgnored" xml:space="preserve"><value>Velocità di riproduzione non valida; viene usata la frequenza di esportazione.</value></data>
<data name="Export.Plan.NoCaptureTimes" xml:space="preserve"><value>Nessun orario di scatto: fotogrammi distribuiti in modo uniforme.</value></data>
@@ -143,6 +143,18 @@ public sealed class ImportService
}, progress, cancellationToken).ConfigureAwait(false);
}
/// <summary>The sequences the sessions of an import went into (new or continued), as far as they exist.</summary>
internal async Task<List<Guid>> SequencesOfAsync(ImportJobParameters job, CancellationToken cancellationToken)
{
var ids = new List<Guid>();
foreach (var session in job.Plan.Sessions)
{
if (await _db.Sequences.FindBySessionKeyAsync(job.ArchiveId, session.SessionKey, cancellationToken).ConfigureAwait(false) is { } id
&& !ids.Contains(id)) ids.Add(id);
}
return ids;
}
/// <summary>Dry run of "delete imported": every card file with a recorded import, checked against its archive copy.</summary>
public async Task<ErasePlan> PlanEraseAsync(Archive archive, Guid cardId, string sourceRoot, IProgress<int>? progress = null,
CancellationToken cancellationToken = default)
@@ -4,13 +4,14 @@ using Titano.Core.Import;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
using Titano.Data.Processing;
namespace Titano.Data.Archiving.Jobs;
/// <summary>
/// <see cref="JobKind.Import"/>: executes the confirmed plan in its parameters; progress in bytes. Checkpoint: the
/// number of plan items done. A removed card fails the job at a batch boundary; resuming it after re-insertion
/// continues there.
/// continues there. The sequences the import went into are then queued for light analysis.
/// </summary>
internal sealed class ImportJobHandler(ImportService imports) : IJobHandler
{
@@ -29,5 +30,6 @@ internal sealed class ImportJobHandler(ImportService imports) : IJobHandler
.ConfigureAwait(false);
}, p => context.ReportAsync(p.BytesDone, p.BytesTotal, p.Current), context.Cancellation).ConfigureAwait(false);
context.Log(Loc.L("Archive.Log.ImportDone", final.Copied, final.Skipped, final.Failed));
await AnalysisFollowUp.QueueAsync(context, await imports.SequencesOfAsync(parameters, context.Cancellation).ConfigureAwait(false)).ConfigureAwait(false);
}
}
@@ -1,6 +1,7 @@
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
using Titano.Data.Processing;
namespace Titano.Data.Archiving.Jobs;
@@ -15,7 +16,8 @@ internal static class JobArchive
/// <summary>
/// <see cref="JobKind.Scan"/> (incremental, or full with <see cref="ScanParameters.Full"/>) and
/// <see cref="JobKind.Rebuild"/>. Checkpoint: <c>after:&lt;folder&gt;</c>, the last folder indexed.
/// <see cref="JobKind.Rebuild"/>. Checkpoint: <c>after:&lt;folder&gt;</c>, the last folder indexed. When anything was
/// indexed, the sequences never light-analysed (adopted folders, new sidecars) are queued for it.
/// </summary>
internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler
{
@@ -37,5 +39,9 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer,
}, context.Cancellation).ConfigureAwait(false);
foreach (var (folder, error) in summary.Failed) context.Log(Loc.L("Archive.Log.FolderFailed", folder, error));
context.Log(Loc.L("Archive.Log.ScanDone", summary.Folders, summary.Indexed, summary.Adopted, summary.Removed));
// A resumed scan may have indexed its folders before the interruption.
if (summary.Indexed > 0 || resume is not null)
await AnalysisFollowUp.QueueAsync(context, await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false))
.ConfigureAwait(false);
}
}
+39 -2
View File
@@ -28,13 +28,18 @@ public sealed record JobEngineOptions
/// <summary>
/// Persistent job queue runner (BRIEF §14.6): N workers claim queued jobs from the database by priority, run the
/// registered handler, checkpoint progress, and survive crashes (running jobs are requeued at start with their
/// checkpoints). Used by the desktop client (local mode) and the server.
/// checkpoints). Used by the desktop client (local mode) and the server. Jobs an <see cref="ISelectiveJobHandler"/>
/// declines stay queued for other claimers.
/// </summary>
public sealed class JobEngine : IAsyncDisposable
{
/// <summary>Queued jobs of selective kinds examined per claim; jobs declined beyond it wait until those ahead leave the queue.</summary>
private const int SelectiveWindow = 100;
private readonly TitanoDatabase _db;
private readonly Dictionary<JobKind, IJobHandler> _handlers;
private IReadOnlyCollection<JobKind> _kinds = [];
private Dictionary<JobKind, ISelectiveJobHandler> _selective = [];
private readonly ConcurrentDictionary<Guid, CancellationTokenSource> _running = new();
private readonly Channel<bool> _wake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
private readonly CancellationTokenSource _shutdown = new();
@@ -65,6 +70,7 @@ public sealed class JobEngine : IAsyncDisposable
await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false);
// Only kinds this engine can run: an empty filter would claim (and fail) every job, render-node work included.
_kinds = [.. (Options.Kinds ?? _handlers.Keys).Where(_handlers.ContainsKey).Distinct()];
_selective = _kinds.Select(k => _handlers[k]).OfType<ISelectiveJobHandler>().ToDictionary(h => h.Kind);
if (_kinds.Count == 0)
{
Log.Info("jobs", "No job handlers; this engine claims nothing");
@@ -150,6 +156,9 @@ public sealed class JobEngine : IAsyncDisposable
}
}
/// <summary>True when this engine claims jobs of <paramref name="kind"/> (it has a handler and the kind is not filtered out).</summary>
internal bool Runs(JobKind kind) => _kinds.Contains(kind);
internal void StopJob(Guid id)
{
if (_running.TryGetValue(id, out var cts)) cts.Cancel();
@@ -174,7 +183,7 @@ public sealed class JobEngine : IAsyncDisposable
Job? job;
try
{
job = await _db.Jobs.ClaimNextAsync(workerId, _kinds, shutdown).ConfigureAwait(false);
job = await ClaimAsync(workerId, shutdown).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
@@ -205,6 +214,34 @@ public sealed class JobEngine : IAsyncDisposable
}
}
/// <summary>
/// The next job in claim order (priority, then age). Without selective handlers the database picks it atomically;
/// with them, the head of the other kinds competes with the first job a selective handler accepts among the first
/// <see cref="SelectiveWindow"/> of its kinds, and the winner is claimed if it is still queued.
/// </summary>
private async Task<Job?> ClaimAsync(string workerId, CancellationToken cancellationToken)
{
if (_selective.Count == 0) return await _db.Jobs.ClaimNextAsync(workerId, _kinds, cancellationToken).ConfigureAwait(false);
var others = _kinds.Where(k => !_selective.ContainsKey(k)).ToList();
for (var attempt = 0; attempt < 3; attempt++)
{
var best = others.Count > 0 ? (await _db.Jobs.ListQueuedAsync(others, 1, cancellationToken).ConfigureAwait(false)).FirstOrDefault() : null;
foreach (var job in await _db.Jobs.ListQueuedAsync(_selective.Keys, SelectiveWindow, cancellationToken).ConfigureAwait(false))
{
if (!_selective[job.Kind].Accepts(job)) continue;
if (best is null || ClaimsBefore(job, best)) best = job;
break;
}
if (best is null) return null;
// Another claimer may have been faster: look again.
if (await _db.Jobs.TryClaimAsync(best.Id, workerId, cancellationToken).ConfigureAwait(false) is { } claimed) return claimed;
}
return null;
}
private static bool ClaimsBefore(Job a, Job b) =>
a.Priority != b.Priority ? a.Priority > b.Priority : a.Created != b.Created ? a.Created < b.Created : a.Id.CompareTo(b.Id) < 0;
private async Task RunAsync(Job job, CancellationToken shutdown)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
+19
View File
@@ -13,6 +13,16 @@ public interface IJobHandler
Task RunAsync(JobContext context);
}
/// <summary>
/// A handler that runs only some jobs of its kind, decided from the job alone (its parameters never change). The engine
/// leaves the others queued for other claimers: the server renders what its managed encoders produce and leaves the
/// rest to render nodes (BRIEF §14.5, §14.7).
/// </summary>
public interface ISelectiveJobHandler : IJobHandler
{
bool Accepts(Job job);
}
/// <summary>A component that brings several handlers (the archive engine: scan, rebuild, verify, import, images, maintenance).</summary>
public interface IJobHandlerProvider
{
@@ -75,6 +85,15 @@ public sealed class JobContext
return ReportCoreAsync(done, total, Message, force: true);
}
/// <summary>
/// Queues follow-up work on this job's engine for the same user (light analysis after an import); null when the
/// engine has no handler for <paramref name="kind"/>, since nothing here would run it.
/// </summary>
public async Task<Job?> EnqueueFollowUpAsync(JobKind kind, IReadOnlyList<Guid> targets, string? label, string? parametersJson, JobPriority priority) =>
_engine.Runs(kind)
? await _engine.EnqueueAsync(kind, targets, label, parametersJson, priority, Job.CreatedBy, Cancellation).ConfigureAwait(false)
: null;
/// <summary>Appends a line to the job log (the last lines are kept with the job).</summary>
public void Log(string line)
{
@@ -0,0 +1,36 @@
using Titano.Core.Model;
using Titano.Data.Jobs;
using Titano.Data.Repositories;
namespace Titano.Data.Processing;
/// <summary>
/// Light analysis queued after work that brings sequences or frames into the index (card import, adoption, scan), so the
/// advisor's Auto values and recommended preset exist without any user action (BRIEF §1.1, §4.3). Nothing is queued
/// when the engine cannot run light analysis.
/// </summary>
internal static class AnalysisFollowUp
{
/// <summary>Queues one light analysis of <paramref name="sequenceIds"/> (labelled like the job that found them).</summary>
public static async Task QueueAsync(JobContext context, IReadOnlyCollection<Guid> sequenceIds)
{
if (sequenceIds.Count == 0) return;
var request = ProcessingJobs.LightAnalysis([.. sequenceIds], context.Job.TargetLabel);
await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false);
}
/// <summary>Sequences of <paramref name="archiveId"/> that were never light-analysed (new or adopted ones).</summary>
public static async Task<List<Guid>> UnanalysedAsync(TitanoDatabase db, Guid archiveId, CancellationToken cancellationToken)
{
var ids = new List<Guid>();
var query = new SequenceQuery { ArchiveId = archiveId, WithoutStatus = SequenceStatus.AnalyzedLight, Sort = SequenceSort.Oldest, Limit = 1000 };
do
{
var page = await db.Sequences.ListAsync(query, cancellationToken).ConfigureAwait(false);
ids.AddRange(page.Items.Select(s => s.Id));
query = query with { Cursor = page.NextCursor };
}
while (query.Cursor is not null);
return ids;
}
}
@@ -1,5 +1,6 @@
using System.Globalization;
using Titano.Core.Analysis.Deep;
using Titano.Core.Archiving;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
@@ -8,8 +9,10 @@ namespace Titano.Data.Processing.Jobs;
/// <summary>
/// <see cref="JobKind.LightAnalyse"/> and <see cref="JobKind.DeepAnalyse"/> over the target sequences, one after the
/// other. Progress counts sequences (light) or frames (deep); checkpoint: the number of sequences done. A deep pass
/// interrupted inside a sequence resumes from that sequence's <c>analysis.partial</c>.
/// other. Progress counts sequences (light) or frames (deep). A sequence that cannot be analysed is logged and the
/// others still run; the job then fails, and a retry analyses only the failed ones. An offline archive stops the job at
/// once. Checkpoint: the next target and the failed ones (<c>next[;i,j…]</c>). A deep pass interrupted inside a
/// sequence resumes from that sequence's <c>analysis.partial</c>.
/// </summary>
internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser analyser, JobKind kind) : IJobHandler
{
@@ -19,29 +22,47 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal
{
var deep = kind == JobKind.DeepAnalyse;
var targets = context.Job.TargetIds;
var first = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var next) ? next : 0;
var (next, retry) = Parse(context.Checkpoint, targets.Count);
// Deep progress is in frames across all targets, so the bar and the ETA cover the whole job.
var frames = new long[targets.Count];
if (deep)
{
for (var i = 0; i < targets.Count; i++)
frames[i] = (await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false))?.FrameCount ?? 0;
}
var total = deep ? frames.Sum() : targets.Count;
for (var i = first; i < targets.Count; i++)
var units = new long[targets.Count];
for (var i = 0; i < targets.Count; i++)
units[i] = deep ? (await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false))?.FrameCount ?? 0 : 1;
var total = units.Sum();
var done = Enumerable.Range(0, next).Where(i => !retry.Contains(i)).Sum(i => units[i]);
var failed = new List<int>();
int[] order = [.. retry, .. Enumerable.Range(next, targets.Count - next)];
for (var k = 0; k < order.Length; k++)
{
context.Cancellation.ThrowIfCancellationRequested();
var before = deep ? frames[..i].Sum() : i;
var i = order[k];
var before = done;
var chain = new ProgressChain(context.ReportAsync);
var progress = deep ? chain.For<AnalysisProgress>(p => (before + p.Done, total, null)) : null;
var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false);
await chain.FlushAsync().ConfigureAwait(false);
if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i]));
else Report(context, outcome, deep);
await context.CheckpointAsync((i + 1).ToString(CultureInfo.InvariantCulture), deep ? before + frames[i] : i + 1, total).ConfigureAwait(false);
try
{
var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false);
await chain.FlushAsync().ConfigureAwait(false);
if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i]));
else Report(context, outcome, deep);
}
catch (Exception ex) when (ex is not (OperationCanceledException or ArchiveOfflineException))
{
await chain.FlushAsync().ConfigureAwait(false);
Core.Diagnostics.Log.Warning("processing", $"Analysis of sequence {targets[i]} failed", ex);
context.Log(Loc.L("Advisor.Log.Failed", await TitleAsync(targets[i], context.Cancellation).ConfigureAwait(false), ex.Message));
failed.Add(i);
}
done += units[i];
next = int.Max(next, i + 1);
// Failed targets of an earlier attempt not retried yet stay listed.
await context.CheckpointAsync(Format(next, [.. failed, .. retry.Skip(k + 1)]), done, total).ConfigureAwait(false);
}
if (failed.Count > 0) throw new InvalidOperationException(Loc.L("Errors.Analysis.Failed", failed.Count, targets.Count));
}
private async Task<string> TitleAsync(Guid sequenceId, CancellationToken cancellationToken) =>
(await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false))?.DisplayTitle ?? sequenceId.ToString();
private static void Report(JobContext context, AnalysisOutcome outcome, bool deep)
{
var title = outcome.Sequence.DisplayTitle;
@@ -50,4 +71,23 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal
if (outcome.SettingsUnreadable) context.Log(Loc.L("Advisor.Log.ProjectUnreadable", title));
if (outcome.Presets.Count > 0) context.Log(Loc.L("Advisor.Log.Presets", string.Join(" · ", outcome.Presets.Select(p => p.DisplayName()))));
}
/// <summary>Next target and the failed targets to retry first; a missing or unreadable checkpoint starts over.</summary>
internal static (int Next, IReadOnlyList<int> Retry) Parse(string? checkpoint, int count)
{
var parts = checkpoint?.Split(';');
if (parts is not ([_] or [_, _]) || !int.TryParse(parts[0], NumberStyles.None, CultureInfo.InvariantCulture, out var next) || next > count) return (0, []);
var retry = new List<int>();
foreach (var item in parts.Length == 2 ? parts[1].Split(',') : [])
{
if (!int.TryParse(item, NumberStyles.None, CultureInfo.InvariantCulture, out var index) || index >= next) return (0, []);
if (!retry.Contains(index)) retry.Add(index);
}
return (next, retry);
}
internal static string Format(int next, IReadOnlyCollection<int> failed) =>
failed.Count == 0
? next.ToString(CultureInfo.InvariantCulture)
: string.Create(CultureInfo.InvariantCulture, $"{next};{string.Join(',', failed.Order())}");
}
@@ -0,0 +1,40 @@
using Titano.Core.Export;
using Titano.Core.Model;
using Titano.Core.Video;
using Titano.Data.Jobs;
namespace Titano.Data.Processing.Jobs;
/// <summary>
/// <see cref="JobKind.Render"/> on the server (BRIEF §4.4, §14.5): it takes only jobs whose every output (the preset and
/// each batch preset) its managed encoders produce, and runs them like a client does. Jobs that need a platform encoder
/// (MP4) stay queued for render nodes (§14.7). Jobs whose parameters cannot be read are taken, so they fail with their
/// reason instead of waiting for a claimer that never comes.
/// </summary>
internal sealed class ManagedRenderJobHandler(RenderJobHandler render, IReadOnlyList<IVideoEncoderProvider> encoders) : ISelectiveJobHandler
{
private readonly EncoderCapability[] _capabilities = [.. encoders.SelectMany(e => e.Capabilities)];
public JobKind Kind => JobKind.Render;
public bool Accepts(Job job)
{
ArgumentNullException.ThrowIfNull(job);
try
{
var parameters = ProcessingJobs.ReadParameters(job.ParametersJson);
// Most declined jobs are decided by the main codec, before the export is read.
return _capabilities.Any(c => string.Equals(ProcessingJobs.CodecName(c.Codec), parameters.Codec, StringComparison.OrdinalIgnoreCase))
&& RenderJobHandler.Targets(ProcessingJobs.ReadExport(parameters.PresetJson), parameters.PresetName).All(t => Produces(t.Export));
}
catch (InvalidDataException)
{
return true;
}
}
public Task RunAsync(JobContext context) => render.RunAsync(context);
private bool Produces(ResolvedExport export) =>
_capabilities.Any(c => c.Codec == export.Codec && export.Width <= c.MaxWidth && export.Height <= c.MaxHeight);
}
@@ -13,7 +13,7 @@ namespace Titano.Data.Processing.Jobs;
/// <c>RenderJobParameters</c> whose preset JSON is the export section of a project (preset, overrides, name template,
/// folder, batch). The project is the embedded one or the sequence's own; frames flagged missing, corrupt, excluded or
/// undecodable are left out. The preset and every batch preset render after one preparation pass into the sequence's
/// <c>exports/</c> (or the export folder), named by the template, and each finished output is recorded at once.
/// <c>exports/</c> (or the export folder, never on a server), named by the template, and each finished output is recorded at once.
/// Checkpoint: the batch entries recorded and the output being written, so a crashed attempt's leftovers are removed and
/// only unfinished entries render again. A failed entry fails the job after the others ran; retrying renders it alone.
/// </summary>
@@ -31,6 +31,8 @@ internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions opti
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
// A server writes only inside its archive (BRIEF §14.2): an export folder chosen on a client means nothing there.
if (options.Host == EncoderHost.Server) export = export with { Folder = null };
var project = (parameters.SettingsJson is { } embedded ? ProjectStore.Parse(embedded) : ProjectStore.Read(sequence, folder)) with { Export = export };
var settingsJson = project.ToJson();
@@ -74,18 +76,12 @@ internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions opti
}
/// <summary>The job's preset, then every other preset of the batch with its own values.</summary>
internal static List<ExportTarget> Targets(ExportSettings export, string presetName)
{
var main = ExportResolver.Resolve(export);
var targets = new List<ExportTarget> { new(main, presetName) };
foreach (var id in export.Batch.Distinct(StringComparer.OrdinalIgnoreCase))
{
if (id.Equals(main.BasePresetId, StringComparison.OrdinalIgnoreCase)) continue;
var resolved = ExportResolver.Resolve(new ExportSettings { Preset = new Auto<string>().WithManual(id) });
targets.Add(new ExportTarget(resolved, ProcessingJobs.PresetName(resolved)));
}
return targets;
}
internal static List<ExportTarget> Targets(ExportSettings export, string presetName) =>
[
new(ExportResolver.Resolve(export), presetName),
.. ProcessingJobs.BatchPresets(export).Select(id => ExportResolver.Resolve(ProcessingJobs.BatchExport(export, id)))
.Select(resolved => new ExportTarget(resolved, ProcessingJobs.PresetName(resolved))),
];
/// <summary>Archive-relative for outputs inside the archive (like the server's renders), absolute otherwise.</summary>
private static string Stored(ArchivePaths paths, string output)
+37 -2
View File
@@ -44,6 +44,32 @@ public static class ProcessingJobs
JsonSerializer.Serialize(parameters, ProtocolJson.Default.RenderJobParameters), JobPriority.User);
}
/// <summary>
/// One <see cref="Render"/> job per preset of <paramref name="export"/>: its preset, then each batch preset with its
/// own values. For a server: a render node uploads one file per job, so a batch must not travel as one job.
/// </summary>
public static IReadOnlyList<ArchiveJobRequest> RenderEach(Sequence sequence, ExportSettings export, int sourceWidth, int sourceHeight,
ProjectSettings? project = null, bool useProxies = false)
{
ArgumentNullException.ThrowIfNull(export);
return
[
Render(sequence, export with { Batch = [] }, sourceWidth, sourceHeight, project, useProxies),
.. BatchPresets(export).Select(id => Render(sequence, BatchExport(export, id), sourceWidth, sourceHeight, project, useProxies)),
];
}
/// <summary>The batch presets of an export besides its own, each once.</summary>
internal static IEnumerable<string> BatchPresets(ExportSettings export)
{
var main = ExportResolver.Resolve(export).BasePresetId;
return export.Batch.Distinct(StringComparer.OrdinalIgnoreCase).Where(id => !id.Equals(main, StringComparison.OrdinalIgnoreCase));
}
/// <summary>A batch preset rendered with its own values, named and placed like the export it belongs to.</summary>
internal static ExportSettings BatchExport(ExportSettings export, string presetId) =>
new() { Preset = new Auto<string>().WithManual(presetId), NameTemplate = export.NameTemplate, Folder = export.Folder };
/// <summary>Codec name in job parameters and render-node capabilities ("h264", "hevc", "mjpeg", "pngsequence"…).</summary>
public static string CodecName(VideoCodec codec) => codec.ToString().ToLowerInvariant();
@@ -57,11 +83,20 @@ public static class ProcessingJobs
/// <summary>The parameters of a render job and the export they select.</summary>
/// <exception cref="InvalidDataException">The job has no valid render parameters.</exception>
internal static (RenderJobParameters Parameters, ExportSettings Export) ReadRender(string? parametersJson)
{
var parameters = ReadParameters(parametersJson);
return (parameters, ReadExport(parameters.PresetJson));
}
/// <summary>The protocol record of a render job, without reading its export.</summary>
/// <exception cref="InvalidDataException">The job has no valid render parameters.</exception>
internal static RenderJobParameters ReadParameters(string? parametersJson)
{
try
{
if (parametersJson is { Length: > 0 } && JsonSerializer.Deserialize(parametersJson, ProtocolJson.Default.RenderJobParameters) is { } parameters)
return (parameters, ReadExport(parameters.PresetJson));
if (parametersJson is { Length: > 0 } && JsonSerializer.Deserialize(parametersJson, ProtocolJson.Default.RenderJobParameters)
is { PresetName.Length: > 0, PresetJson.Length: > 0, Codec.Length: > 0 } parameters)
return parameters;
}
catch (JsonException ex)
{
@@ -1,3 +1,4 @@
using Titano.Core.Export;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Jobs;
@@ -8,8 +9,9 @@ namespace Titano.Data.Processing;
/// <summary>
/// The analysis and render engine of a host (BRIEF §4.3, §10, §11): job handlers for light analysis, deep analysis and
/// renders over the archive in the index, built with the host's decoders and encoders (<see cref="ProcessingOptions"/>).
/// Register it on the <see cref="JobEngine"/> next to the archive engine; a server engine leaves render jobs to render
/// nodes. Create jobs with <see cref="ProcessingJobs"/>.
/// Register it on the <see cref="JobEngine"/> next to the archive engine. A client renders every job it is given; a
/// server takes only the renders its managed encoders produce and leaves the others to render nodes. Create jobs with
/// <see cref="ProcessingJobs"/>.
/// </summary>
public sealed class ProcessingService : IJobHandlerProvider
{
@@ -28,10 +30,14 @@ public sealed class ProcessingService : IJobHandlerProvider
public ProcessingOptions Options { get; }
/// <summary>Handlers for <see cref="JobKind.LightAnalyse"/>, <see cref="JobKind.DeepAnalyse"/> and <see cref="JobKind.Render"/>.</summary>
public IReadOnlyList<IJobHandler> CreateJobHandlers() =>
[
new AnalyseJobHandler(_db, _analyser, JobKind.LightAnalyse),
new AnalyseJobHandler(_db, _analyser, JobKind.DeepAnalyse),
new RenderJobHandler(_db, Options),
];
public IReadOnlyList<IJobHandler> CreateJobHandlers()
{
var render = new RenderJobHandler(_db, Options);
return
[
new AnalyseJobHandler(_db, _analyser, JobKind.LightAnalyse),
new AnalyseJobHandler(_db, _analyser, JobKind.DeepAnalyse),
Options.Host == EncoderHost.Server ? new ManagedRenderJobHandler(render, Options.Encoders) : render,
];
}
}
+27 -14
View File
@@ -8,6 +8,7 @@ using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Render;
using Titano.Core.Video;
using Titano.Data.Processing.Jobs;
using Titano.Protocol;
using Titano.Protocol.Client;
@@ -148,9 +149,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
{
break;
}
catch (TitanoApiException ex) when (ex.Status is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden)
catch (Exception ex) when (ex is TitanoApiException { Status: HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden } or UntrustedCertificateException)
{
// Signed out, or the account lacks the render-node right: nothing to retry until the user acts.
// Signed out, no render-node right, or a changed certificate: nothing to retry until the user acts.
Log.Warning(LogCategory, "The server refused this render node", ex);
break;
}
@@ -193,8 +194,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
{
_nodeId = null;
}
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException or UntrustedCertificateException)
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
// Unreachable or refusing: the claim loop decides whether to go on; beating must not stop the node.
Log.Warning(LogCategory, "Heartbeat failed", ex);
}
}
@@ -211,6 +213,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
var target = new ExportTarget(resolved, ProcessingJobs.PresetName(resolved));
if (target.Export.Container == ContainerFormat.ImageSequence) throw new InvalidOperationException(Loc.L("Errors.Render.NodeSequence"));
var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export };
// One output per job: batch presets were meant for separate jobs (ProcessingJobs.RenderEach).
if (RenderJobHandler.Targets(export, target.PresetName).Skip(1).Select(t => t.PresetName).ToList() is { Count: > 0 } skipped)
await reporter.ReportAsync(0, claimed.Frames.Count, Loc.L("Export.Log.NodeBatch", string.Join(", ", skipped))).ConfigureAwait(false);
var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false));
var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false);
@@ -257,21 +262,28 @@ public sealed class RenderNodeClient : IAsyncDisposable
/// <summary>
/// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused
/// across jobs), or proxies into the job folder when the job asks for them and the server has them.
/// across jobs, fetched once per content), or proxies into the job folder when the job asks for them and the server
/// has them.
/// </summary>
private async Task<Dictionary<Guid, string>> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
{
var files = new Dictionary<Guid, string>();
var done = 0;
await reporter.ReportAsync(0, claimed.Frames.Count, Loc.L("Export.Job.Downloading")).ConfigureAwait(false);
await Parallel.ForEachAsync(claimed.Frames, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken },
async (frame, ct) =>
var total = claimed.Frames.Count;
await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false);
var downloads = claimed.Frames.GroupBy(f => claimed.UseProxies && f.HasProxy ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken },
async (frames, ct) =>
{
var frame = frames.First();
var path = claimed.UseProxies && frame.HasProxy
? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false)
: await DownloadOriginalAsync(frame, ct).ConfigureAwait(false);
lock (files) files[frame.Id] = path;
await reporter.ReportAsync(Interlocked.Increment(ref done), claimed.Frames.Count, null).ConfigureAwait(false);
lock (files)
{
foreach (var f in frames) files[f.Id] = path;
}
await reporter.ReportAsync(Interlocked.Add(ref done, frames.Count()), total, null).ConfigureAwait(false);
}).ConfigureAwait(false);
return files;
}
@@ -309,8 +321,12 @@ public sealed class RenderNodeClient : IAsyncDisposable
var part = path + ".part";
for (var attempt = 1; ; attempt++)
{
await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
// A part as long as the file was complete when the last attempt stopped: only its hash is left to check.
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size)
{
await using var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
await _client.DownloadFrameAsync(frame.Id, "original", file, null, cancellationToken).ConfigureAwait(false);
}
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
{
File.Move(part, path, overwrite: true);
@@ -406,7 +422,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
/// <summary>
/// Progress of the job in flight to the server, at most every interval (and at the end). A refusal (409) means the
/// job was cancelled or given to another node: <see cref="Lost"/> is set and the job's token cancelled.
/// job was cancelled or given to another node: the job's token is cancelled.
/// </summary>
private sealed class NodeReporter(TitanoClient client, Guid nodeId, Guid jobId, TimeSpan interval, CancellationTokenSource job)
{
@@ -418,8 +434,6 @@ public sealed class RenderNodeClient : IAsyncDisposable
public Guid NodeId => nodeId;
public bool Lost { get; private set; }
public async ValueTask ReportAsync(long done, long total, string? message)
{
await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
@@ -435,7 +449,6 @@ public sealed class RenderNodeClient : IAsyncDisposable
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict)
{
Lost = true;
await job.CancelAsync().ConfigureAwait(false);
}
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or IOException)
@@ -44,13 +44,15 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
? await AnalyseDeepAsync(frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false)
: ReadReport(folder, frames.Count);
var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs);
var advice = Advisor.Recommend(light, report, context);
var warnings = advice.Findings.Count(f => f.Severity >= FindingSeverity.Warning);
var warnings = Advisor.Recommend(light, report, context).Findings.Count(f => f.Severity >= FindingSeverity.Warning);
bool changed, unreadable = false;
try
{
changed = await projects.UpdateAsync(paths, sequence, settings => Advisor.Apply(settings, advice), cancellationToken).ConfigureAwait(false);
// The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3).
changed = await projects.UpdateAsync(paths, sequence,
settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })),
cancellationToken).ConfigureAwait(false);
}
catch (InvalidDataException ex)
{
+10 -6
View File
@@ -39,9 +39,7 @@ public sealed class JobRepository
/// </summary>
public Task<Job?> ClaimNextAsync(string workerId, IReadOnlyCollection<JobKind>? kinds = null, CancellationToken cancellationToken = default)
{
var kindFilter = kinds is { Count: > 0 }
? " AND kind IN (" + string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture))) + ")"
: string.Empty;
var kindFilter = kinds is { Count: > 0 } ? $" AND kind IN ({KindList(kinds)})" : string.Empty;
var pick = $"SELECT id FROM jobs WHERE state = {(int)JobState.Queued}{kindFilter} ORDER BY priority DESC, created, id LIMIT 1";
var claim = $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $1, started = $2, finished = NULL, error = NULL, attempts = attempts + 1";
DbParam[] args = [workerId, DateTimeOffset.UtcNow];
@@ -130,9 +128,13 @@ public sealed class JobRepository
/// <summary>Queued jobs of one kind in claim order (highest priority, then oldest).</summary>
public Task<IReadOnlyList<Job>> ListQueuedAsync(JobKind kind, int limit = 50, CancellationToken cancellationToken = default) =>
_db.ListAsync(
$"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind = $1 ORDER BY priority DESC, created, id LIMIT $2",
[(int)kind, Cursor.ClampLimit(limit)], Map, cancellationToken);
ListQueuedAsync([kind], limit, cancellationToken);
/// <summary>Queued jobs of the given kinds in claim order (highest priority, then oldest).</summary>
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit = 50, CancellationToken cancellationToken = default) =>
kinds.Count == 0 ? Task.FromResult<IReadOnlyList<Job>>([]) : _db.ListAsync(
$"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY priority DESC, created, id LIMIT $1",
[Cursor.ClampLimit(limit)], Map, cancellationToken);
/// <summary>Running jobs claimed by <paramref name="nodeId"/>.</summary>
public Task<IReadOnlyList<Job>> ListRunningByNodeAsync(string nodeId, CancellationToken cancellationToken = default) =>
@@ -193,6 +195,8 @@ public sealed class JobRepository
return new JobCounts(row.GetInt32(0), row.GetInt32(1), row.GetInt32(2), row.GetInt32(3), row.GetInt32(4));
}, cancellationToken);
private static string KindList(IEnumerable<JobKind> kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture)));
private static Job Map(DbRow r) => new()
{
Id = r.GetGuid(0),
+5 -3
View File
@@ -12,8 +12,9 @@ namespace Titano.Server.Jobs;
/// <summary>
/// Hosts the server's <see cref="JobEngine"/> (BRIEF §14.6) with the <see cref="IJobHandler"/>s registered in DI (directly or
/// through <see cref="IJobHandlerProvider"/>s) and
/// relays its state changes and progress as WebSocket events. Render jobs are never run here: render nodes claim them
/// (BRIEF §14.7). On shutdown running jobs stay marked running and are requeued from their checkpoints at the next start.
/// relays its state changes and progress as WebSocket events. Render jobs run here only when the registered render
/// handler accepts them (managed formats, <see cref="ISelectiveJobHandler"/>); render nodes claim the others (BRIEF
/// §14.5, §14.7). On shutdown running jobs stay marked running and are requeued from their checkpoints at the next start.
/// </summary>
public sealed class JobEngineHost : IHostedService
{
@@ -30,7 +31,8 @@ public sealed class JobEngineHost : IHostedService
{
_db = db;
_events = events;
var local = handlers.Concat(providers.SelectMany(p => p.CreateJobHandlers())).Where(h => h.Kind != JobKind.Render).ToList();
// A render handler must be selective: a plain one would take the jobs that need a render node's encoder.
var local = handlers.Concat(providers.SelectMany(p => p.CreateJobHandlers())).Where(h => h.Kind != JobKind.Render || h is ISelectiveJobHandler).ToList();
_kinds = [.. local.Select(h => h.Kind)];
// Without handlers no worker starts: an engine with an empty kind list would claim every kind.
Workers = _kinds.Count == 0 ? 0 : options.Workers;
+9 -2
View File
@@ -26,15 +26,22 @@ switch (command.Command)
case ServerCommand.UninstallService when OperatingSystem.IsWindows():
return ServiceAction(ServiceInstaller.Uninstall, $"Removed the {WindowsServiceHost.ServiceName} service.");
case ServerCommand.Service when OperatingSystem.IsWindows():
if (WindowsServiceHost.Run(() => ServerHost.Build(ServerOptions.Load(), [.. command.HostArguments], ArchiveModule.Register))) return 0;
if (WindowsServiceHost.Run(() => ServerHost.Build(ServerOptions.Load(), [.. command.HostArguments], RegisterModules))) return 0;
await Console.Error.WriteLineAsync("--service is used by the Service Control Manager; start the service with 'sc start TitanoServer'.");
return 1;
default:
var app = ServerHost.Build(ServerOptions.Load(), [.. command.HostArguments], ArchiveModule.Register);
var app = ServerHost.Build(ServerOptions.Load(), [.. command.HostArguments], RegisterModules);
await app.RunAsync();
return 0;
}
// The archive engine, then analysis and managed rendering over it.
static void RegisterModules(IServiceCollection services)
{
ArchiveModule.Register(services);
ProcessingModule.Register(services);
}
static int ServiceAction(Action action, string done)
{
try
+2 -3
View File
@@ -7,8 +7,8 @@ namespace Titano.Server.Storage;
/// <summary>
/// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and
/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance), plus the
/// analysis handlers of <see cref="ProcessingModule"/>. Decoding is managed only (BRIEF §4.2).
/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance). Decoding is
/// managed only (BRIEF §4.2).
/// </summary>
public static class ArchiveModule
{
@@ -22,6 +22,5 @@ public static class ArchiveModule
services.AddSingleton<IArchiveIngest>(sp => sp.GetRequiredService<ArchiveService>().Ingest);
services.AddSingleton<IJobHandlerProvider>(sp => sp.GetRequiredService<ArchiveService>());
services.AddSingleton<IFrameAssetLocator, ArchiveAssetLocator>();
ProcessingModule.Register(services);
}
}
@@ -10,9 +10,10 @@ using Titano.Server.Configuration;
namespace Titano.Server.Storage;
/// <summary>
/// Analysis and render engine of the server (BRIEF §4.2, §10, §14.6): light and deep analysis job handlers with managed
/// decoders, so the light analysis queued after uploads runs here. Its render handler is registered too, but the job
/// engine host never runs render jobs: render nodes claim them (BRIEF §14.7).
/// Analysis and render engine of the server (BRIEF §4.2, §10, §14.5, §14.6): light and deep analysis with managed
/// decoders (the light analysis queued after uploads runs here) and renders with managed encoders only. The render
/// handler takes just the jobs whose every output a managed encoder produces (AVI MJPEG, image sequences, GIF, APNG);
/// MP4 jobs stay queued for render nodes (BRIEF §14.7).
/// </summary>
public static class ProcessingModule
{
@@ -27,6 +27,25 @@ public sealed class JobEngineTests(TestContext context)
}
}
/// <summary>Takes only jobs whose parameters say they are its own.</summary>
private sealed class SelectiveHandler(JobKind kind) : ISelectiveJobHandler
{
public const string Mine = """{"mine":true}""";
public JobKind Kind => kind;
public bool Accepts(Job job) => job.ParametersJson == Mine;
public Task RunAsync(JobContext c) => Task.CompletedTask;
}
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
{
public JobKind Kind => kind;
public Task RunAsync(JobContext context) => run(context);
}
private async Task<TitanoDatabase> OpenAsync() =>
await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
@@ -159,6 +178,50 @@ public sealed class JobEngineTests(TestContext context)
Assert.Equal(JobKind.Render, pending[0].Kind);
}
[Test]
public async Task SelectiveHandlersLeaveDeclinedJobsQueuedWithoutHoldingUpOthers()
{
await using var db = await OpenAsync();
await using var engine = new JobEngine(db, [new SelectiveHandler(JobKind.Render), new CountingHandler(JobKind.Thumbs, 1)],
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
var declined = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: """{"mine":false}""", priority: JobPriority.User);
var thumbs = await engine.EnqueueAsync(JobKind.Thumbs, [], priority: JobPriority.Background);
var accepted = await engine.EnqueueAsync(JobKind.Render, [], parametersJson: SelectiveHandler.Mine, priority: JobPriority.User);
await engine.StartAsync();
var render = await WaitForStateAsync(db, accepted.Id, JobState.Done, TimeSpan.FromSeconds(10));
var other = await WaitForStateAsync(db, thumbs.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.True(render.Started <= other.Started, "claim order: priority first, across selective and other kinds");
await Task.Delay(200);
var waiting = (await db.Jobs.GetAsync(declined.Id))!;
Assert.Equal((JobState.Queued, (string?)null), (waiting.State, waiting.NodeId));
}
[Test]
public async Task FollowUpsAreQueuedOnlyForKindsTheEngineRuns()
{
await using var db = await OpenAsync();
var target = Guid.CreateVersion7();
Job? thumbs = null, light = null;
var scan = new CallbackHandler(JobKind.Scan, async c =>
{
thumbs = await c.EnqueueFollowUpAsync(JobKind.Thumbs, [target], "Roof", null, JobPriority.Background);
light = await c.EnqueueFollowUpAsync(JobKind.LightAnalyse, [target], "Roof", null, JobPriority.Background);
});
await using var engine = new JobEngine(db, [scan, new CountingHandler(JobKind.Thumbs, 1)],
new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync();
var user = Guid.CreateVersion7();
var job = await engine.EnqueueAsync(JobKind.Scan, [], createdBy: user);
await WaitForStateAsync(db, job.Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Null(light);
var followUp = await WaitForStateAsync(db, Assert.NotNull(thumbs).Id, JobState.Done, TimeSpan.FromSeconds(10));
Assert.Equal((user, JobPriority.Background, "Roof"), (followUp.CreatedBy!.Value, followUp.Priority, followUp.TargetLabel!));
Assert.SequenceEqual([target], followUp.TargetIds);
Assert.Empty((await db.Jobs.ListAsync(null, JobKind.LightAnalyse)).Items);
}
[Test]
public async Task HandlerFailureMarksJobFailedWithMessage()
{
@@ -0,0 +1,65 @@
using Titano.Core.Archiving;
using Titano.Core.Diagnostics.Synthetic;
using Titano.Core.Import;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Archiving;
using Titano.Data.Tests.Archiving;
using Titano.Testing;
namespace Titano.Data.Tests.Processing;
/// <summary>New sequences get their light analysis (and so their Auto values) without any user action (BRIEF §1.1).</summary>
[Category("processing")]
public sealed class AnalysisFollowUpTests(TestContext context)
{
[Test]
public async Task AnAdoptingScanQueuesLightAnalysisOfTheNewSequencesOnly()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof");
await fx.StartEngineAsync(withArchive: true);
var scan = await fx.EnqueueAsync(ArchiveJobs.Scan(fx.Archive.Archive, full: true));
await fx.WaitForAsync(scan.Id, JobState.Done);
var light = Assert.NotNull(await SingleLightJobAsync(fx));
await fx.WaitForAsync(light.Id, JobState.Done);
var sequence = (await fx.Db.Sequences.GetAsync(light.TargetIds.Single(), fx.Ct))!;
Assert.Equal("Roof", sequence.Folder);
Assert.True(sequence.Status.HasFlag(SequenceStatus.AnalyzedLight), "analysed");
Assert.True(ProjectSettings.FromJson(await File.ReadAllTextAsync(fx.Folder(sequence).ProjectPath, fx.Ct)).Timing.Mode.HasAdvice, "advised");
// Scans that find nothing new queue nothing more.
foreach (var full in new[] { false, true })
{
var again = await fx.EnqueueAsync(ArchiveJobs.Scan(fx.Archive.Archive, full));
await fx.WaitForAsync(again.Id, JobState.Done);
}
Assert.Equal(light.Id, (await SingleLightJobAsync(fx))?.Id);
}
[Test]
public async Task AnImportQueuesLightAnalysisOfTheSequencesItFilled()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var card = context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), ArchiveFixture.Start, 10);
var plan = await fx.Archive.Service.PlanImportAsync(fx.Archive.Archive.Id, card, new ImportOptions(), null, null, fx.Ct);
await fx.StartEngineAsync(withArchive: true);
var import = await fx.Engine.EnqueueAsync(ArchiveJobs.Import(fx.Archive.Archive, plan, null), createdBy: Guid.CreateVersion7(), cancellationToken: fx.Ct);
await fx.WaitForAsync(import.Id, JobState.Done);
var light = Assert.NotNull(await SingleLightJobAsync(fx));
await fx.WaitForAsync(light.Id, JobState.Done);
var sequenceId = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Archive.Id, plan.Sessions.Single().SessionKey, fx.Ct);
Assert.SequenceEqual([Assert.NotNull(sequenceId)], light.TargetIds);
Assert.Equal(import.CreatedBy, light.CreatedBy);
Assert.Equal(JobPriority.Background, light.Priority);
Assert.True((await fx.Db.Sequences.GetAsync(sequenceId.Value, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedLight), "analysed");
}
private static async Task<Job?> SingleLightJobAsync(ProcessingFixture fx) =>
(await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.SingleOrDefault();
}
@@ -5,6 +5,7 @@ using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Processing;
using Titano.Data.Processing.Jobs;
using Titano.Data.Repositories;
using Titano.Testing;
@@ -105,6 +106,55 @@ public sealed class AnalysisJobTests(TestContext context)
Assert.Contains(Loc.L("Advisor.Log.Presets", string.Empty), done.LogTail);
}
[Test]
public async Task ASequenceThatFailsDoesNotStopTheOthersAndARetryRedoesOnlyIt()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var a = await fx.AddAsync(Flickering(10), "A");
var b = await fx.AddAsync(Flickering(10), "B");
var c = await fx.AddAsync(Flickering(10), "C");
// B's sidecar is gone for a while: its status cannot be recorded.
var sidecar = fx.Folder(b).ManifestPath;
var saved = await File.ReadAllBytesAsync(sidecar, fx.Ct);
File.Delete(sidecar);
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([a.Id, b.Id, c.Id]));
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Analysis.Failed", 1, 3), failed.Error);
Assert.Equal("3;1", failed.Checkpoint);
Assert.SequenceEqual([true, false, true], await AnalysedAsync(fx, a, b, c));
await File.WriteAllBytesAsync(sidecar, saved, fx.Ct);
await fx.Engine.ResumeAsync(job.Id, fx.Ct);
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.SequenceEqual([true, true, true], await AnalysedAsync(fx, a, b, c));
Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal));
// The retry analysed B alone.
var lightOfA = Loc.L("Advisor.Log.LightDone", a.DisplayTitle, 10, 0).Split(',')[0];
Assert.Equal(1, done.LogTail!.Split('\n').Count(line => line.Contains(lightOfA, StringComparison.Ordinal)));
}
[Test]
public void AnalysisCheckpointsKeepTheFailedTargets()
{
Assert.Equal("3", AnalyseJobHandler.Format(3, []));
Assert.Equal("4;0,2", AnalyseJobHandler.Format(4, [2, 0]));
var (next, retry) = AnalyseJobHandler.Parse("4;0,2", 5);
Assert.Equal(4, next);
Assert.SequenceEqual([0, 2], retry);
Assert.Equal(2, AnalyseJobHandler.Parse("2", 5).Next);
// Anything else starts over: no checkpoint, beyond the targets, a failed target not yet reached, garbage.
foreach (var text in new[] { null, "6", "2;3", "2;x", "a" }) Assert.Equal(0, AnalyseJobHandler.Parse(text, 5).Next);
}
private static async Task<List<bool>> AnalysedAsync(ProcessingFixture fx, params Sequence[] sequences)
{
var analysed = new List<bool>();
foreach (var s in sequences) analysed.Add((await fx.Db.Sequences.GetAsync(s.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedLight));
return analysed;
}
[Test]
public async Task LightAnalysisAfterNewFramesDropsTheStaleDeepStatus()
{
@@ -45,10 +45,10 @@ internal sealed class ProcessingFixture : IAsyncDisposable
public static async Task<ProcessingFixture> CreateAsync(TestContext context) => new(await ArchiveFixture.CreateAsync(context), new GateDecoder());
/// <summary>A started engine with the processing handlers (one worker, fast checkpoints).</summary>
public async Task<JobEngine> StartEngineAsync()
/// <summary>A started engine with the processing handlers, and the archive's when asked (one worker, fast checkpoints).</summary>
public async Task<JobEngine> StartEngineAsync(bool withArchive = false)
{
_engine = new JobEngine(Db, Processing.CreateJobHandlers(),
_engine = new JobEngine(Db, [.. withArchive ? Archive.Service.CreateJobHandlers() : [], .. Processing.CreateJobHandlers()],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await _engine.StartAsync(Ct);
return _engine;
@@ -61,6 +61,15 @@ internal sealed class ProcessingFixture : IAsyncDisposable
/// <summary>Writes frames <paramref name="first"/>… of <paramref name="sequence"/> as JPEGs with their Exif into
/// <paramref name="folder"/> (adopted in place) and indexes the archive.</summary>
public async Task<Sequence> AddAsync(SyntheticSequence sequence, string folder = "Synthetic", int first = 0)
{
Write(sequence, folder, first);
await Archive.ScanAsync(ScanMode.Full);
var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder);
return (await Db.Sequences.GetAsync(location.Id, Ct))!;
}
/// <summary>Writes frames <paramref name="first"/>… of <paramref name="sequence"/> as JPEGs with their Exif into <paramref name="folder"/>.</summary>
public void Write(SyntheticSequence sequence, string folder = "Synthetic", int first = 0)
{
var directory = Path.Combine(Archive.Paths.Root, folder);
Directory.CreateDirectory(directory);
@@ -70,9 +79,6 @@ internal sealed class ProcessingFixture : IAsyncDisposable
var exif = ExifWriter.BuildJpegApp1(sequence.Metadata[i]);
File.WriteAllBytes(Path.Combine(directory, FileName(i)), JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 95, App1Segments = [exif[4..]] }));
}
await Archive.ScanAsync(ScanMode.Full);
var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder);
return (await Db.Sequences.GetAsync(location.Id, Ct))!;
}
public static string FileName(int index) => $"IMG_{index + 1:0000}.JPG";
@@ -6,6 +6,7 @@ using Titano.Core.Imaging.Decoders;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Jobs;
using Titano.Data.Processing;
using Titano.Testing;
@@ -145,6 +146,37 @@ public sealed class RenderJobTests(TestContext context)
Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
}
[Test]
public async Task AServerTakesOnlyTheRendersItsManagedEncodersProduce()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var server = new ProcessingService(fx.Db, new ProcessingOptions { Host = EncoderHost.Server });
var handler = Assert.NotNull(server.CreateJobHandlers().Single(h => h.Kind == JobKind.Render) as ISelectiveJobHandler);
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic" };
bool Accepts(ExportSettings export) =>
handler.Accepts(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, ParametersJson = ProcessingJobs.Render(sequence, export, 96, 64).ParametersJson });
Assert.True(Accepts(Mjpeg with { Batch = ["png-sequence", "gif-preview"] }), "managed outputs only");
Assert.False(Accepts(new ExportSettings { Preset = new Auto<string>().WithManual("4k-hevc") }), "HEVC needs a render node");
Assert.False(Accepts(Mjpeg with { Batch = ["1080p-h264"] }), "one MP4 in the batch");
Assert.True(handler.Accepts(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, ParametersJson = "{}" }), "unreadable: fails with its reason");
var client = new ProcessingService(fx.Db).CreateJobHandlers().Single(h => h.Kind == JobKind.Render);
Assert.False(client is ISelectiveJobHandler, "a client renders whatever it is given");
}
[Test]
public void ABatchForAServerBecomesOneJobPerPreset()
{
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic", Title = "Roof" };
var jobs = ProcessingJobs.RenderEach(sequence, Mjpeg with { Batch = ["mjpeg-avi", "gif-preview", "GIF-preview", "4k-hevc"] }, 96, 64);
var parameters = jobs.Select(j => System.Text.Json.JsonSerializer.Deserialize(j.ParametersJson!, Titano.Protocol.ProtocolJson.Default.RenderJobParameters)!).ToList();
Assert.SequenceEqual(["mjpeg", "gif", "hevc"], parameters.Select(p => p.Codec));
Assert.True(jobs.All(j => j.Kind == JobKind.Render && j.Targets.SequenceEqual([sequence.Id])), "render jobs of the sequence");
Assert.True(parameters.All(p => ProcessingJobs.ReadExport(p.PresetJson).Batch.Count == 0), "no job carries a batch");
Assert.Equal((3840, 2160), (parameters[2].Width, parameters[2].Height));
}
private static List<string> Entries(string folder) =>
Directory.Exists(folder) ? [.. Directory.EnumerateFileSystemEntries(folder).Select(e => Path.GetFileName(e)).Order(StringComparer.Ordinal)] : [];
@@ -1,3 +1,4 @@
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Diagnostics.Synthetic;
@@ -13,10 +14,17 @@ namespace Titano.Server.Tests.Storage;
[Category("processing")]
public sealed class ProcessingModuleTests(TestContext context)
{
/// <summary>The registrations of Program.cs.</summary>
private static void Production(IServiceCollection services)
{
ArchiveModule.Register(services);
ProcessingModule.Register(services);
}
[Test]
public async Task UploadedFramesAreLightAnalysedAndAdvised()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
await using var server = await TestServer.StartAsync(context, services: Production);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 10, StartEv = 0.5, EndEv = -0.5 });
Guid sequenceId = default;