Files
Chronos/src/Titano.Data/Processing/SequenceAnalyser.cs
T
Claude 0085765a12 fix(archive,analysis): an unreadable photo changes nothing; a report stale by proxies never replaces a current one
The orphans fix and scans treated a folder whose only new photo could not be
read yet (held open, no permission, an offline placeholder) as changed: the
sidecar was rewritten, the covering flags cleared and analyses queued again at
every click. SequenceAdopter.Reconcile now returns null when it indexed nothing
and changed no flag, as its contract says.

A deep pass counted as current by frames version alone, so a pass whose verdict
a proxy made stale overwrote the report a later pass made from the proxies,
then dropped the deep status. It now counts as current only when its report
describes the frames as they are, and otherwise replaces only a saved report
that does not describe them either.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-13 00:31:30 +00:00

335 lines
23 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Security.Cryptography;
using System.Text;
using Titano.Core.Advice;
using Titano.Core.Analysis;
using Titano.Core.Analysis.Deep;
using Titano.Core.Archiving;
using Titano.Core.Imaging;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Repositories;
namespace Titano.Data.Processing;
/// <summary>What an analysis of one sequence found and changed.</summary>
/// <param name="Warnings">Findings of warning severity or worse (light and, when current, deep).</param>
/// <param name="SettingsChanged">The advisor changed Auto values of the project.</param>
/// <param name="SettingsUnreadable">The project could not be read, so its Auto values were left alone.</param>
/// <param name="Presets">Sequence presets that fit the content (deep analysis only).</param>
/// <param name="FramesChanged">How the frames it read were no longer the sequence's when it recorded: anything but
/// <see cref="FramesChange.None"/> means nothing was recorded from them.</param>
internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable,
IReadOnlyList<SequencePreset> Presets, FramesChange FramesChanged = FramesChange.None);
/// <summary>How the frames an analysis read had changed by the time it recorded, under the sequence lock.</summary>
internal enum FramesChange
{
/// <summary>Still the sequence's frames, or changed in this index with a light analysis queued to replace what it records.</summary>
None,
/// <summary>
/// Added, removed or renumbered in this index (an upload, an import, a scan), with no light analysis queued: one must
/// run here for the frames as they are.
/// </summary>
Indexed,
/// <summary>
/// Changed in the sidecar by another process (a desktop sharing a server's archive, or that server), which analyses
/// them; this index holds them only after its next scan, which on a server queues their analysis here too.
/// </summary>
NotIndexed,
}
/// <summary>
/// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the
/// Auto values of the project. The light pass reads the index, probes the decode tier (§8.2) of frames whose tier is not
/// known yet (headers only, once per frame) and reads one file for the GPS fix; the deep pass decodes every frame,
/// checkpointing into <c>.titano/analysis.partial</c> so a cancelled, paused or crashed run resumes, records the tier
/// each frame decoded at (or that it did not decode) and saves <c>analysis.json</c> atomically under the sequence lock.
/// Results land in the sidecar (status, tiers), the index (status, warning count, compact summary, tier flags,
/// per-frame light statistics) and <c>project.json</c>.
/// </summary>
internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions options, SequenceWriter writer, ArchiveIndexer indexer, ProjectStore projects)
{
/// <summary>Long edge of the decode that confirms a frame an interrupted earlier run found undecodable.</summary>
private const int ConfirmLongEdge = 64;
private readonly FrameTiers _tiers = new(db, writer, indexer);
/// <summary>Analyses one sequence; null when it is no longer in the index.</summary>
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
public async Task<AnalysisOutcome?> AnalyseAsync(Guid sequenceId, bool deep, IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
{
if (await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } sequence
|| await db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive) return null;
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
// Format tiers (BRIEF §10.1): frames indexed since the last analysis are probed with this host's decoders. A frame
// read from its proxy is not preview-only whatever its original's tier, so it caps nothing (BRIEF §8.1, §8.2); the
// flags are taken as the sidecar has them once recorded, so ones another worker recorded meanwhile count too.
var probed = await _tiers.ProbeUnknownAsync(archive, folder, ManifestStore.ReadSequence(folder.ManifestPath), cancellationToken).ConfigureAwait(false);
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, probed, proxies: true, cancellationToken).ConfigureAwait(false));
var metadata = frames.LightFrames();
DeepAnalysisReport? report = null;
if (deep)
{
// Proxies may have arrived during the pass: the flags below follow the frames as they are at its end, and the
// report is checked against them under the lock.
(report, var decoded) = await AnalyseDeepAsync(archive, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false);
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxies: true, cancellationToken).ConfigureAwait(false));
metadata = [.. metadata.Select((m, i) => m with { Flags = frames.Frames[i].Flags })];
}
var light = LightAnalyzer.Analyze(metadata);
var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs);
bool changed, unreadable = false;
int warnings;
// Under the sequence lock, which a deep pass of the same sequence also takes to record its report: a light pass
// then either sees that report or runs before it is recorded, and never clears what it recorded. The report counts
// only while it describes the frames as they are under the lock (ReadReport), so a deep pass loads them again: a
// proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis
// the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it
// left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records
// nothing (FramesChangeAsync), and a light analysis of the frames as they are runs instead, here or where they
// were changed; its report is saved only over none of the frames as they are (SaveReport). What a deep pass records
// then follows the report analysis.json holds: its own, or the one it left in place.
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false);
// The frames as this host reads them now: a proxy that arrived while the pass ran moves no frames version but
// makes the pass's verdict on that frame's original stale, so only a report that describes them is current.
var now = deep && change == FramesChange.None
? await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false)
: null;
var saved = deep && SaveReport(folder, report!,
change == FramesChange.None && manifest.Versions.Frames == sequence.FramesVersion && Describes(report!, now!),
now, manifest.Frames.Count);
if (change != FramesChange.None)
return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], change);
if (deep)
{
// A report left in place is read as a light analysis reads it, so the deep status, summary, warnings and
// Auto values recorded here describe the file that readers and the report endpoint open.
if (!saved) report = ReadReport(folder, now!);
else if (!Describes(report!, now!)) report = null;
}
else
{
report = ReadReport(folder, frames);
}
try
{
// The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3).
changed = await projects.UpdateLockedAsync(paths, sequence,
settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })),
cancellationToken).ConfigureAwait(false);
}
catch (InvalidDataException ex)
{
Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex);
(changed, unreadable) = (false, true);
}
warnings = Advisor.Recommend(light, report, context).Findings.Count(f => f.Severity >= FindingSeverity.Warning);
var set = SequenceStatus.AnalyzedLight | (report is null ? SequenceStatus.None : SequenceStatus.AnalyzedDeep)
| (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None);
// Without a current report (none, or made before frames were added) the deep status no longer holds.
var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings);
var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var status = (manifest.Status | set) & ~clear;
if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow });
var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null;
await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false);
if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false);
var presets = deep ? SequencePresets.Generate(light, report, context) : [];
return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets);
}
/// <summary>
/// How the frames this pass read had changed by now, under the sequence lock. Every writer of this index's frames bumps
/// the row's frames version with them under the same lock (an upload or import, a scan reconciling photos copied in by
/// hand or indexing frames another process listed in the sidecar), and every writer of the sidecar's frames bumps its
/// version (<paramref name="manifest"/>, read under the lock). What the pass found then describes other frames:
/// recorded, it would mark the new ones analysed, with Auto values and warnings counted on the old ones, over what an
/// analysis of the frames as they are may have recorded already, in the shared sidecar and <c>project.json</c> too.
/// A sidecar at another version than the row lists frames another process changed and this index does not hold yet
/// (<see cref="FramesChange.NotIndexed"/>): nothing is queued here, since a light analysis would read this index as
/// it is and find it behind the sidecar again until the next scan indexes them. A row whose version moved since the
/// pass read it holds the frames as they are, and a light analysis here covers them (<see cref="FramesChange.Indexed"/>).
/// One already queued for the sequence runs after this on the frames as they are then, so the pass still records
/// meanwhile, as an image pass keeps its flag (<see cref="ArchiveImages"/>): during an upload burst every pass is
/// overtaken, and a sequence being shot would otherwise show no analysis until uploads paused.
/// </summary>
private async Task<FramesChange> FramesChangeAsync(Sequence sequence, SequenceManifest manifest, CancellationToken cancellationToken)
{
if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not { } row) return FramesChange.None;
if (manifest.Versions.Frames != row.FramesVersion) return FramesChange.NotIndexed;
return row.FramesVersion == sequence.FramesVersion
|| await db.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequence.Id, cancellationToken).ConfigureAwait(false)
? FramesChange.None
: FramesChange.Indexed;
}
/// <summary>
/// The saved deep report when it describes <paramref name="frames"/> as this host reads them now: also stale when a
/// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode,
/// BRIEF §2.3), so that verdict and the frame's missing measurements no longer hold.
/// </summary>
public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames) =>
ReadReport(folder, frames.Count) is { } report && Describes(report, frames) ? report : null;
/// <summary>Whether <paramref name="report"/> describes <paramref name="frames"/> as this host reads them now (<see cref="ReadReport(SequencePaths, SequenceFrames)"/>).</summary>
private static bool Describes(DeepAnalysisReport report, SequenceFrames frames)
{
if (report.FrameCount != frames.Count) return false;
if (frames.Proxied is not { } proxied) return true;
HashSet<int> measured = report.ProxyFrames is { } read ? [.. read] : [];
foreach (var index in report.Issues.Undecodable)
{
if (index < frames.Count && proxied[index] && !measured.Contains(index) && (frames.Frames[index].Flags & SequenceFrames.NotAnalysed) == 0)
return false;
}
return true;
}
/// <summary>The saved deep report when it covers <paramref name="frameCount"/> frames; null when absent, stale or unreadable.</summary>
public static DeepAnalysisReport? ReadReport(SequencePaths folder, int frameCount)
{
if (!File.Exists(folder.AnalysisPath)) return null;
try
{
var report = DeepAnalysisReport.Load(folder.AnalysisPath);
return report.FrameCount == frameCount ? report : null;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or System.Text.Json.JsonException)
{
Core.Diagnostics.Log.Warning("processing", $"{folder.AnalysisPath} cannot be read", ex);
return null;
}
}
/// <summary>The frame rows with the flags and preview sizes of the <paramref name="current"/> sidecar frames.</summary>
private static SequenceFrames WithTiers(SequenceFrames frames, IReadOnlyList<SidecarFrame> current)
{
if (current.Count == 0) return frames;
var byId = current.ToDictionary(f => f.Id);
var rows = frames.Frames.Select(f => byId.TryGetValue(f.Id, out var s)
? f with { Flags = s.Flags, PreviewWidth = s.PreviewW ?? 0, PreviewHeight = s.PreviewH ?? 0 }
: f).ToList();
return frames with { Frames = rows };
}
/// <summary>
/// What the deep pass learnt about each frame's tier, by frame id: the tier this run decoded it at. A frame this run
/// tried to read and recorded nothing for vanished or could not be read, which says nothing about its content. A frame
/// only an interrupted earlier run analysed and found undecodable is decoded again (small) to confirm it: that run's
/// failure may have been a read error too. Missing and corrupt frames are not decoded at all.
/// </summary>
private Dictionary<Guid, FileTier> Decoded(SequenceFrames frames, DeepAnalysisReport report, SequenceFrameSource source,
IReadOnlyDictionary<Guid, string> companions, CancellationToken cancellationToken)
{
var tiers = new Dictionary<Guid, FileTier>();
foreach (var (index, tier) in source.Observed) tiers[frames.Frames[index].Id] = tier;
var read = source.Read.ToHashSet();
foreach (var index in report.Issues.Undecodable)
{
// A frame read from its proxy says nothing about its original's tier.
if (index >= frames.Count || read.Contains(index) || frames.Files[index] is not { } path
|| (frames.Frames[index].Flags & SequenceFrames.NotAnalysed) != 0 || frames.Proxied is { } proxied && proxied[index]) continue;
var frame = frames.Frames[index];
try
{
using var decoded = options.Archive.Decoder.Decode(
new DecodeRequest(path, ConfirmLongEdge, DecodePurpose.Analysis, Companion: companions.GetValueOrDefault(frame.Id)), cancellationToken);
if (decoded is not null) tiers[frame.Id] = new FileTier(frame.Format, decoded.Tier, decoded.Decoder, null, 0, 0);
else if (File.Exists(path)) tiers[frame.Id] = new FileTier(frame.Format, DecodeTier.Unavailable, null, "no decoder produced pixels", 0, 0);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("processing", $"Frame {path} could not be read", ex);
}
}
return tiers;
}
/// <summary>
/// Saves a deep pass's report as <c>analysis.json</c>, under the sequence lock, then drops its checkpoint; returns
/// whether it saved it. The report replaces the saved one when it describes the frames as they are
/// (<paramref name="current"/>), or when the saved one describes them no better: none, unreadable, or not describing
/// <paramref name="now"/> (the frames as this host reads them, proxies included), or, when they were not read, of
/// another frame count than <paramref name="frameCount"/>, the sidecar's. A pass overtaken by an upload, by a proxy,
/// or by another process sharing the archive, would otherwise replace the report a later pass made of the frames as
/// they are, and the next light analysis would drop that one's deep status.
/// </summary>
private static bool SaveReport(SequencePaths folder, DeepAnalysisReport report, bool current, SequenceFrames? now, int frameCount)
{
var save = current || (now is not null ? ReadReport(folder, now) : ReadReport(folder, frameCount)) is null;
if (save) report.Save(folder.AnalysisPath);
Checkpoint(folder).Delete();
return save;
}
/// <summary>The deep pass's checkpoint, <c>.titano/analysis.partial</c>: kept until the pass has saved its report.</summary>
private static DeepAnalysisCheckpointFile Checkpoint(SequencePaths folder) =>
new(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
private async Task<(DeepAnalysisReport Report, Dictionary<Guid, FileTier> Decoded)> AnalyseDeepAsync(Archive archive, SequenceFrames frames,
IReadOnlyList<LightFrame> metadata, SequencePaths folder, IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
{
var file = Checkpoint(folder);
// The host's analysis parameters as they are now (Settings › Analysis on the desktop). Those that change per-frame
// results are part of the checkpoint's fingerprint, so a partial run made with other values starts again.
var analysis = options.Current.DeepAnalysis;
var companions = await GprCompanions.FindAsync(db, archive, frames.Frames, cancellationToken).ConfigureAwait(false);
DeepAnalysisReport report;
Dictionary<Guid, FileTier> decoded;
await using (var source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder) { Originals = true, Companions = companions })
{
report = await DeepAnalyzer.AnalyzeAsync(source, analysis, new FramesCheckpoint(file, ContentKey(frames)), progress, cancellationToken)
.ConfigureAwait(false);
decoded = Decoded(frames, report, source, companions, cancellationToken);
}
// An unreadable verdict on a frame read from its original goes stale once the frame has a proxy (ReadReport). The
// report is saved under the sequence lock, once the frames it measured are known to be current (SaveReport).
int[] fromProxies = [.. Enumerable.Range(0, frames.Count).Where(i => frames.Proxied?[i] == true)];
report = report with { ProxyFrames = fromProxies.Length > 0 ? fromProxies : null };
var columns = report.Frames;
var stats = new FrameLightStats[frames.Count];
for (var i = 0; i < stats.Length; i++)
stats[i] = new FrameLightStats(frames.Frames[i].Id, columns.LogLuminance[i], columns.ClipLow[i], columns.ClipHigh[i]);
await db.Frames.UpdateLightStatsAsync(stats, cancellationToken).ConfigureAwait(false);
return (report, decoded);
}
/// <summary>Identity of the analysed content: frames replaced or renumbered since a checkpoint was written (with the
/// same count), or now read from their proxies where they were read from their originals (or the other way round),
/// make it stale, like changed options do. Only frames read from proxies are marked, so the key of a sequence read
/// from its originals is what it was before proxies counted, and checkpoints written then still resume.</summary>
internal static string ContentKey(SequenceFrames frames)
{
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
for (var i = 0; i < frames.Count; i++)
{
var f = frames.Frames[i];
var source = frames.Proxied is { } proxied && proxied[i] ? "/proxy" : "";
hash.AppendData(Encoding.UTF8.GetBytes($"{f.Id:N}/{f.Sha256}/{f.Size}/{f.Flags & SequenceFrames.NotAnalysed}{source};"));
}
return Convert.ToHexStringLower(hash.GetHashAndReset())[..16];
}
private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey) : IDeepAnalysisCheckpoint
{
public ValueTask<IReadOnlyList<DeepFrame>> LoadAsync(string fingerprint, CancellationToken cancellationToken) =>
file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken);
public ValueTask AppendAsync(IReadOnlyList<DeepFrame> frames, CancellationToken cancellationToken) => file.AppendAsync(frames, cancellationToken);
}
}