using System.Security.Cryptography; using System.Text; using Titano.Core.Advice; using Titano.Core.Analysis; using Titano.Core.Analysis.Deep; using Titano.Core.Archiving; using Titano.Core.Imaging; using Titano.Core.Model; using Titano.Data.Archiving; using Titano.Data.Repositories; namespace Titano.Data.Processing; /// What an analysis of one sequence found and changed. /// Findings of warning severity or worse (light and, when current, deep). /// The advisor changed Auto values of the project. /// The project could not be read, so its Auto values were left alone. /// Sequence presets that fit the content (deep analysis only). internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable, IReadOnlyList Presets); /// /// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the /// Auto values of the project. The light pass reads the index, probes the decode tier (§8.2) of frames whose tier is not /// known yet (headers only, once per frame) and reads one file for the GPS fix; the deep pass decodes every frame, /// checkpointing into .titano/analysis.partial so a cancelled, paused or crashed run resumes, records the tier /// each frame decoded at (or that it did not decode) and saves analysis.json atomically. Results land in the /// sidecar (status, tiers), the index (status, warning count, compact summary, tier flags, per-frame light statistics) /// and project.json. /// internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions options, SequenceWriter writer, ArchiveIndexer indexer, ProjectStore projects) { /// Long edge of the decode that confirms a frame an interrupted earlier run found undecodable. private const int ConfirmLongEdge = 64; private readonly FrameTiers _tiers = new(db, writer, indexer); /// Analyses one sequence; null when it is no longer in the index. /// The archive is offline or read-only. public async Task AnalyseAsync(Guid sequenceId, bool deep, IProgress? progress, CancellationToken cancellationToken) { if (await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } sequence || await db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive) return null; var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false); // Format tiers (BRIEF §10.1): frames indexed since the last analysis are probed with this host's decoders. A frame // read from its proxy is not preview-only whatever its original's tier, so it caps nothing (BRIEF §8.1, §8.2); the // flags are taken as the sidecar has them once recorded, so ones another worker recorded meanwhile count too. var probed = await _tiers.ProbeUnknownAsync(archive, folder, ManifestStore.ReadSequence(folder.ManifestPath), cancellationToken).ConfigureAwait(false); frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, probed, proxies: true, cancellationToken).ConfigureAwait(false)); var metadata = frames.LightFrames(); DeepAnalysisReport? report; if (deep) { // Proxies may have arrived during the pass: the advice below follows the frames as they are at its end. (report, var decoded) = await AnalyseDeepAsync(archive, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false); frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxies: true, cancellationToken).ConfigureAwait(false)); metadata = [.. metadata.Select((m, i) => m with { Flags = frames.Frames[i].Flags })]; } else { report = ReadReport(folder, frames); } var light = LightAnalyzer.Analyze(metadata); var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs); bool changed, unreadable = false; try { // The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3). changed = await projects.UpdateAsync(paths, sequence, settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })), cancellationToken).ConfigureAwait(false); } catch (InvalidDataException ex) { Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex); (changed, unreadable) = (false, true); } if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false); int warnings; // Under the sequence lock, which a deep pass of the same sequence also takes to record its report: a light pass // then either sees that report or runs before it is recorded, and never clears what it recorded. await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { report ??= ReadReport(folder, frames); warnings = Advisor.Recommend(light, report, context).Findings.Count(f => f.Severity >= FindingSeverity.Warning); var set = SequenceStatus.AnalyzedLight | (report is null ? SequenceStatus.None : SequenceStatus.AnalyzedDeep) | (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None); // Without a current report (none, or made before frames were added) the deep status no longer holds. var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings); var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); var status = (manifest.Status | set) & ~clear; if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }); var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null; await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false); await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); } var presets = deep ? SequencePresets.Generate(light, report, context) : []; return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets); } /// /// The saved deep report when it describes as this host reads them now: also stale when a /// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode, /// BRIEF §2.3), so that verdict and the frame's missing measurements no longer hold. /// public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames) { var report = ReadReport(folder, frames.Count); if (report is null || frames.Proxied is not { } proxied) return report; HashSet measured = report.ProxyFrames is { } read ? [.. read] : []; foreach (var index in report.Issues.Undecodable) { if (index < frames.Count && proxied[index] && !measured.Contains(index) && (frames.Frames[index].Flags & SequenceFrames.NotAnalysed) == 0) return null; } return report; } /// The saved deep report when it covers frames; null when absent, stale or unreadable. public static DeepAnalysisReport? ReadReport(SequencePaths folder, int frameCount) { if (!File.Exists(folder.AnalysisPath)) return null; try { var report = DeepAnalysisReport.Load(folder.AnalysisPath); return report.FrameCount == frameCount ? report : null; } catch (Exception ex) when (ex is IOException or InvalidDataException or System.Text.Json.JsonException) { Core.Diagnostics.Log.Warning("processing", $"{folder.AnalysisPath} cannot be read", ex); return null; } } /// The frame rows with the flags and preview sizes of the sidecar frames. private static SequenceFrames WithTiers(SequenceFrames frames, IReadOnlyList current) { if (current.Count == 0) return frames; var byId = current.ToDictionary(f => f.Id); var rows = frames.Frames.Select(f => byId.TryGetValue(f.Id, out var s) ? f with { Flags = s.Flags, PreviewWidth = s.PreviewW ?? 0, PreviewHeight = s.PreviewH ?? 0 } : f).ToList(); return frames with { Frames = rows }; } /// /// What the deep pass learnt about each frame's tier, by frame id: the tier this run decoded it at. A frame this run /// tried to read and recorded nothing for vanished or could not be read, which says nothing about its content. A frame /// only an interrupted earlier run analysed and found undecodable is decoded again (small) to confirm it: that run's /// failure may have been a read error too. Missing and corrupt frames are not decoded at all. /// private Dictionary Decoded(SequenceFrames frames, DeepAnalysisReport report, SequenceFrameSource source, IReadOnlyDictionary companions, CancellationToken cancellationToken) { var tiers = new Dictionary(); foreach (var (index, tier) in source.Observed) tiers[frames.Frames[index].Id] = tier; var read = source.Read.ToHashSet(); foreach (var index in report.Issues.Undecodable) { // A frame read from its proxy says nothing about its original's tier. if (index >= frames.Count || read.Contains(index) || frames.Files[index] is not { } path || (frames.Frames[index].Flags & SequenceFrames.NotAnalysed) != 0 || frames.Proxied is { } proxied && proxied[index]) continue; var frame = frames.Frames[index]; try { using var decoded = options.Archive.Decoder.Decode( new DecodeRequest(path, ConfirmLongEdge, DecodePurpose.Analysis, Companion: companions.GetValueOrDefault(frame.Id)), cancellationToken); if (decoded is not null) tiers[frame.Id] = new FileTier(frame.Format, decoded.Tier, decoded.Decoder, null, 0, 0); else if (File.Exists(path)) tiers[frame.Id] = new FileTier(frame.Format, DecodeTier.Unavailable, null, "no decoder produced pixels", 0, 0); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { Core.Diagnostics.Log.Warning("processing", $"Frame {path} could not be read", ex); } } return tiers; } private async Task<(DeepAnalysisReport Report, Dictionary Decoded)> AnalyseDeepAsync(Archive archive, SequenceFrames frames, IReadOnlyList metadata, SequencePaths folder, IProgress? progress, CancellationToken cancellationToken) { var file = new DeepAnalysisCheckpointFile(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName)); // The host's analysis parameters as they are now (Settings › Analysis on the desktop). Those that change per-frame // results are part of the checkpoint's fingerprint, so a partial run made with other values starts again. var analysis = options.Current.DeepAnalysis; var companions = await GprCompanions.FindAsync(db, archive, frames.Frames, cancellationToken).ConfigureAwait(false); DeepAnalysisReport report; Dictionary decoded; await using (var source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder) { Originals = true, Companions = companions }) { report = await DeepAnalyzer.AnalyzeAsync(source, analysis, new FramesCheckpoint(file, ContentKey(frames)), progress, cancellationToken) .ConfigureAwait(false); decoded = Decoded(frames, report, source, companions, cancellationToken); } // An unreadable verdict on a frame read from its original goes stale once the frame has a proxy (ReadReport). int[] fromProxies = [.. Enumerable.Range(0, frames.Count).Where(i => frames.Proxied?[i] == true)]; report = report with { ProxyFrames = fromProxies.Length > 0 ? fromProxies : null }; report.Save(folder.AnalysisPath); file.Delete(); var columns = report.Frames; var stats = new FrameLightStats[frames.Count]; for (var i = 0; i < stats.Length; i++) stats[i] = new FrameLightStats(frames.Frames[i].Id, columns.LogLuminance[i], columns.ClipLow[i], columns.ClipHigh[i]); await db.Frames.UpdateLightStatsAsync(stats, cancellationToken).ConfigureAwait(false); return (report, decoded); } /// Identity of the analysed content: frames replaced or renumbered since a checkpoint was written (with the /// same count), or now read from their proxies where they were read from their originals (or the other way round), /// make it stale, like changed options do. Only frames read from proxies are marked, so the key of a sequence read /// from its originals is what it was before proxies counted, and checkpoints written then still resume. internal static string ContentKey(SequenceFrames frames) { using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); for (var i = 0; i < frames.Count; i++) { var f = frames.Frames[i]; var source = frames.Proxied is { } proxied && proxied[i] ? "/proxy" : ""; hash.AppendData(Encoding.UTF8.GetBytes($"{f.Id:N}/{f.Sha256}/{f.Size}/{f.Flags & SequenceFrames.NotAnalysed}{source};")); } return Convert.ToHexStringLower(hash.GetHashAndReset())[..16]; } private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey) : IDeepAnalysisCheckpoint { public ValueTask> LoadAsync(string fingerprint, CancellationToken cancellationToken) => file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken); public ValueTask AppendAsync(IReadOnlyList frames, CancellationToken cancellationToken) => file.AppendAsync(frames, cancellationToken); } }