- JobEngine: ISelectiveJobHandler lets a host decline jobs of a kind it runs; declined jobs stay queued for other claimers without holding up the rest. JobContext.EnqueueFollowUpAsync queues follow-up work on the same engine. - Server renders the jobs whose every output a managed encoder produces (ManagedRenderJobHandler); MP4 jobs stay for render nodes. Program.cs registers ProcessingModule next to ArchiveModule. Server renders ignore a client's export folder. - Light analysis is queued after imports and after scans that indexed new sequences, so the advisor's Auto values exist without user action. - Analysis jobs log a failing sequence and go on; the job then fails and a retry analyses only the failed ones. The advised frame rate follows the user's fit-to duration. - ProcessingJobs.RenderEach splits a batch into one job per preset for servers; render nodes say which batch presets they did not render, fetch each content once and stop on a changed certificate. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
134 lines
7.8 KiB
C#
134 lines
7.8 KiB
C#
using System.Security.Cryptography;
|
||
using System.Text;
|
||
using Titano.Core.Advice;
|
||
using Titano.Core.Analysis;
|
||
using Titano.Core.Analysis.Deep;
|
||
using Titano.Core.Archiving;
|
||
using Titano.Core.Model;
|
||
using Titano.Data.Archiving;
|
||
using Titano.Data.Repositories;
|
||
|
||
namespace Titano.Data.Processing;
|
||
|
||
/// <summary>What an analysis of one sequence found and changed.</summary>
|
||
/// <param name="Warnings">Findings of warning severity or worse (light and, when current, deep).</param>
|
||
/// <param name="SettingsChanged">The advisor changed Auto values of the project.</param>
|
||
/// <param name="SettingsUnreadable">The project could not be read, so its Auto values were left alone.</param>
|
||
/// <param name="Presets">Sequence presets that fit the content (deep analysis only).</param>
|
||
internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable,
|
||
IReadOnlyList<SequencePreset> Presets);
|
||
|
||
/// <summary>
|
||
/// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the
|
||
/// Auto values of the project. The light pass reads the index only (plus one file for the GPS fix); the deep pass decodes
|
||
/// every frame, checkpointing into <c>.titano/analysis.partial</c> so a cancelled, paused or crashed run resumes, and
|
||
/// saves <c>analysis.json</c> atomically. Results land in the sidecar status, the index (status, warning count, compact
|
||
/// summary, per-frame light statistics) and <c>project.json</c>.
|
||
/// </summary>
|
||
internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions options, SequenceWriter writer, ArchiveIndexer indexer, ProjectStore projects)
|
||
{
|
||
/// <summary>Analyses one sequence; null when it is no longer in the index.</summary>
|
||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
|
||
public async Task<AnalysisOutcome?> AnalyseAsync(Guid sequenceId, bool deep, IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
|
||
{
|
||
if (await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } sequence
|
||
|| await db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive) return null;
|
||
var paths = new ArchivePaths(archive.RootPath);
|
||
ArchiveVolume.EnsureWritable(paths);
|
||
var folder = paths.Sequence(sequence.Folder);
|
||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, cancellationToken).ConfigureAwait(false);
|
||
var metadata = frames.LightFrames();
|
||
var light = LightAnalyzer.Analyze(metadata);
|
||
|
||
var report = deep
|
||
? await AnalyseDeepAsync(frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false)
|
||
: ReadReport(folder, frames.Count);
|
||
var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs);
|
||
var warnings = Advisor.Recommend(light, report, context).Findings.Count(f => f.Severity >= FindingSeverity.Warning);
|
||
|
||
bool changed, unreadable = false;
|
||
try
|
||
{
|
||
// The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3).
|
||
changed = await projects.UpdateAsync(paths, sequence,
|
||
settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })),
|
||
cancellationToken).ConfigureAwait(false);
|
||
}
|
||
catch (InvalidDataException ex)
|
||
{
|
||
Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex);
|
||
(changed, unreadable) = (false, true);
|
||
}
|
||
|
||
var set = SequenceStatus.AnalyzedLight | (report is null ? SequenceStatus.None : SequenceStatus.AnalyzedDeep)
|
||
| (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None);
|
||
// Without a current report (none, or made before frames were added) the deep status no longer holds.
|
||
var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings);
|
||
var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null;
|
||
await writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
|
||
{
|
||
var status = (m.Status | set) & ~clear;
|
||
return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
|
||
}, cancellationToken).ConfigureAwait(false);
|
||
await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false);
|
||
await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||
|
||
var presets = deep ? SequencePresets.Generate(light, report, context) : [];
|
||
return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets);
|
||
}
|
||
|
||
/// <summary>The saved deep report when it describes the current frames; null when absent, stale or unreadable.</summary>
|
||
public static DeepAnalysisReport? ReadReport(SequencePaths folder, int frameCount)
|
||
{
|
||
if (!File.Exists(folder.AnalysisPath)) return null;
|
||
try
|
||
{
|
||
var report = DeepAnalysisReport.Load(folder.AnalysisPath);
|
||
return report.FrameCount == frameCount ? report : null;
|
||
}
|
||
catch (Exception ex) when (ex is IOException or InvalidDataException or System.Text.Json.JsonException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("processing", $"{folder.AnalysisPath} cannot be read", ex);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private async Task<DeepAnalysisReport> AnalyseDeepAsync(SequenceFrames frames, IReadOnlyList<LightFrame> metadata, SequencePaths folder,
|
||
IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
|
||
{
|
||
var file = new DeepAnalysisCheckpointFile(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
|
||
DeepAnalysisReport report;
|
||
await using (var source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder))
|
||
{
|
||
report = await DeepAnalyzer.AnalyzeAsync(source, options.DeepAnalysis, new FramesCheckpoint(file, ContentKey(frames)), progress, cancellationToken)
|
||
.ConfigureAwait(false);
|
||
}
|
||
report.Save(folder.AnalysisPath);
|
||
file.Delete();
|
||
|
||
var columns = report.Frames;
|
||
var stats = new FrameLightStats[frames.Count];
|
||
for (var i = 0; i < stats.Length; i++)
|
||
stats[i] = new FrameLightStats(frames.Frames[i].Id, columns.LogLuminance[i], columns.ClipLow[i], columns.ClipHigh[i]);
|
||
await db.Frames.UpdateLightStatsAsync(stats, cancellationToken).ConfigureAwait(false);
|
||
return report;
|
||
}
|
||
|
||
/// <summary>Identity of the analysed content: frames replaced or renumbered since a checkpoint was written (with the
|
||
/// same count) make it stale, like changed options do.</summary>
|
||
private static string ContentKey(SequenceFrames frames)
|
||
{
|
||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||
foreach (var f in frames.Frames) hash.AppendData(Encoding.UTF8.GetBytes($"{f.Id:N}/{f.Sha256}/{f.Size}/{f.Flags & SequenceFrames.NotAnalysed};"));
|
||
return Convert.ToHexStringLower(hash.GetHashAndReset())[..16];
|
||
}
|
||
|
||
private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey) : IDeepAnalysisCheckpoint
|
||
{
|
||
public ValueTask<IReadOnlyList<DeepFrame>> LoadAsync(string fingerprint, CancellationToken cancellationToken) =>
|
||
file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken);
|
||
|
||
public ValueTask AppendAsync(IReadOnlyList<DeepFrame> frames, CancellationToken cancellationToken) => file.AppendAsync(frames, cancellationToken);
|
||
}
|
||
}
|