The server decodes with managed decoders only, so HEIF originals had no
proxies, thumbnails from their small previews and no pixels for analysis or
managed renders, and nothing could send it proxies (ENG-05). Manifests now
carry the archive's proxy spec and, per frame, its flags and whether the
server wants its proxy. PUT /api/v1/frames/{id}/proxy (Edit or RenderNode)
stores a JPEG of at most 64 MiB that decodes and whose Exif capture time is
the frame's, then queues thumbnails and the proxy pass. An upload from the
desktop makes and sends the wanted proxies after the originals and counts
them as local changes; a render node sends those it can make from the
originals it fetched once the job is done. The server never makes such a
proxy from a preview, drops one left by a renumbered frame, takes thumbnails
from a proxy, and analyses and renders those frames from their proxies.
Render nodes fetched every frame of the manifest: frames missing, corrupt or
excluded on the server failed the download, and originals not hashed yet
failed the SHA-256 check (ENG-07). Nodes now skip those frames, fetch unhashed
originals into the job folder with a size check, judge decodability with their
own decoders, and use the proxy of a frame they cannot decode.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
141 lines
8.4 KiB
C#
141 lines
8.4 KiB
C#
using System.Security.Cryptography;
|
||
using System.Text;
|
||
using Titano.Core.Advice;
|
||
using Titano.Core.Analysis;
|
||
using Titano.Core.Analysis.Deep;
|
||
using Titano.Core.Archiving;
|
||
using Titano.Core.Model;
|
||
using Titano.Data.Archiving;
|
||
using Titano.Data.Repositories;
|
||
|
||
namespace Titano.Data.Processing;
|
||
|
||
/// <summary>What an analysis of one sequence found and changed.</summary>
|
||
/// <param name="Warnings">Findings of warning severity or worse (light and, when current, deep).</param>
|
||
/// <param name="SettingsChanged">The advisor changed Auto values of the project.</param>
|
||
/// <param name="SettingsUnreadable">The project could not be read, so its Auto values were left alone.</param>
|
||
/// <param name="Presets">Sequence presets that fit the content (deep analysis only).</param>
|
||
internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable,
|
||
IReadOnlyList<SequencePreset> Presets);
|
||
|
||
/// <summary>
|
||
/// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the
|
||
/// Auto values of the project. The light pass reads the index only (plus one file for the GPS fix); the deep pass decodes
|
||
/// every frame, checkpointing into <c>.titano/analysis.partial</c> so a cancelled, paused or crashed run resumes, and
|
||
/// saves <c>analysis.json</c> atomically. Results land in the sidecar status, the index (status, warning count, compact
|
||
/// summary, per-frame light statistics) and <c>project.json</c>.
|
||
/// </summary>
|
||
internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions options, SequenceWriter writer, ArchiveIndexer indexer, ProjectStore projects)
|
||
{
|
||
/// <summary>Analyses one sequence; null when it is no longer in the index.</summary>
|
||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
|
||
public async Task<AnalysisOutcome?> AnalyseAsync(Guid sequenceId, bool deep, IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
|
||
{
|
||
if (await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } sequence
|
||
|| await db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive) return null;
|
||
var paths = new ArchivePaths(archive.RootPath);
|
||
ArchiveVolume.EnsureWritable(paths);
|
||
var folder = paths.Sequence(sequence.Folder);
|
||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
|
||
var metadata = frames.LightFrames();
|
||
var light = LightAnalyzer.Analyze(metadata);
|
||
|
||
var report = deep
|
||
? await AnalyseDeepAsync(frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false)
|
||
: ReadReport(folder, frames.Count);
|
||
var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs);
|
||
|
||
bool changed, unreadable = false;
|
||
try
|
||
{
|
||
// The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3).
|
||
changed = await projects.UpdateAsync(paths, sequence,
|
||
settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })),
|
||
cancellationToken).ConfigureAwait(false);
|
||
}
|
||
catch (InvalidDataException ex)
|
||
{
|
||
Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex);
|
||
(changed, unreadable) = (false, true);
|
||
}
|
||
|
||
if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false);
|
||
|
||
int warnings;
|
||
// Under the sequence lock, which a deep pass of the same sequence also takes to record its report: a light pass
|
||
// then either sees that report or runs before it is recorded, and never clears what it recorded.
|
||
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||
{
|
||
report ??= ReadReport(folder, frames.Count);
|
||
warnings = Advisor.Recommend(light, report, context).Findings.Count(f => f.Severity >= FindingSeverity.Warning);
|
||
var set = SequenceStatus.AnalyzedLight | (report is null ? SequenceStatus.None : SequenceStatus.AnalyzedDeep)
|
||
| (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None);
|
||
// Without a current report (none, or made before frames were added) the deep status no longer holds.
|
||
var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings);
|
||
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
|
||
var status = (manifest.Status | set) & ~clear;
|
||
if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow });
|
||
var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null;
|
||
await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false);
|
||
await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||
}
|
||
|
||
var presets = deep ? SequencePresets.Generate(light, report, context) : [];
|
||
return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets);
|
||
}
|
||
|
||
/// <summary>The saved deep report when it describes the current frames; null when absent, stale or unreadable.</summary>
|
||
public static DeepAnalysisReport? ReadReport(SequencePaths folder, int frameCount)
|
||
{
|
||
if (!File.Exists(folder.AnalysisPath)) return null;
|
||
try
|
||
{
|
||
var report = DeepAnalysisReport.Load(folder.AnalysisPath);
|
||
return report.FrameCount == frameCount ? report : null;
|
||
}
|
||
catch (Exception ex) when (ex is IOException or InvalidDataException or System.Text.Json.JsonException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("processing", $"{folder.AnalysisPath} cannot be read", ex);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private async Task<DeepAnalysisReport> AnalyseDeepAsync(SequenceFrames frames, IReadOnlyList<LightFrame> metadata, SequencePaths folder,
|
||
IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
|
||
{
|
||
var file = new DeepAnalysisCheckpointFile(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
|
||
DeepAnalysisReport report;
|
||
await using (var source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder))
|
||
{
|
||
report = await DeepAnalyzer.AnalyzeAsync(source, options.DeepAnalysis, new FramesCheckpoint(file, ContentKey(frames)), progress, cancellationToken)
|
||
.ConfigureAwait(false);
|
||
}
|
||
report.Save(folder.AnalysisPath);
|
||
file.Delete();
|
||
|
||
var columns = report.Frames;
|
||
var stats = new FrameLightStats[frames.Count];
|
||
for (var i = 0; i < stats.Length; i++)
|
||
stats[i] = new FrameLightStats(frames.Frames[i].Id, columns.LogLuminance[i], columns.ClipLow[i], columns.ClipHigh[i]);
|
||
await db.Frames.UpdateLightStatsAsync(stats, cancellationToken).ConfigureAwait(false);
|
||
return report;
|
||
}
|
||
|
||
/// <summary>Identity of the analysed content: frames replaced or renumbered since a checkpoint was written (with the
|
||
/// same count) make it stale, like changed options do.</summary>
|
||
private static string ContentKey(SequenceFrames frames)
|
||
{
|
||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||
foreach (var f in frames.Frames) hash.AppendData(Encoding.UTF8.GetBytes($"{f.Id:N}/{f.Sha256}/{f.Size}/{f.Flags & SequenceFrames.NotAnalysed};"));
|
||
return Convert.ToHexStringLower(hash.GetHashAndReset())[..16];
|
||
}
|
||
|
||
private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey) : IDeepAnalysisCheckpoint
|
||
{
|
||
public ValueTask<IReadOnlyList<DeepFrame>> LoadAsync(string fingerprint, CancellationToken cancellationToken) =>
|
||
file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken);
|
||
|
||
public ValueTask AppendAsync(IReadOnlyList<DeepFrame> frames, CancellationToken cancellationToken) => file.AppendAsync(frames, cancellationToken);
|
||
}
|
||
}
|