From 34b7c2736de95f90a7d906d5f607e33cb4fae48e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 10:40:19 +0000 Subject: [PATCH] feat(analysis): add streaming deep analysis with resumable checkpoints Per-frame statistics, nine-patch shift, Draft flow on sampled pairs, stars, clouds, sky/land luminance and exposure-aligned scene distance, then sequence-level flicker, received exposure steps, white balance, shake, day/night, hero frame and findings in a versioned analysis.json. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../Analysis/Deep/CloudCoverage.cs | 84 ++++ .../Analysis/Deep/DeepAnalysisCheckpoint.cs | 108 +++++ .../Analysis/Deep/DeepAnalysisJson.cs | 25 + .../Analysis/Deep/DeepAnalysisOptions.cs | 69 +++ .../Analysis/Deep/DeepAnalysisReport.cs | 226 +++++++++ src/Titano.Core/Analysis/Deep/DeepAnalyzer.cs | 210 ++++++++ src/Titano.Core/Analysis/Deep/DeepFrame.cs | 82 ++++ .../Analysis/Deep/DeepSequenceAnalysis.cs | 450 ++++++++++++++++++ src/Titano.Core/Analysis/Deep/FrameProbe.cs | 147 ++++++ .../Analysis/Deep/HistogramDistance.cs | 61 +++ src/Titano.Core/Analysis/Deep/IFrameSource.cs | 25 + .../Analysis/Deep/StarFieldDetector.cs | 57 +++ .../Analysis/Deep/DeepAnalyzerTests.cs | 311 ++++++++++++ .../Analysis/Deep/DeepFixtures.cs | 125 +++++ 14 files changed, 1980 insertions(+) create mode 100644 src/Titano.Core/Analysis/Deep/CloudCoverage.cs create mode 100644 src/Titano.Core/Analysis/Deep/DeepAnalysisCheckpoint.cs create mode 100644 src/Titano.Core/Analysis/Deep/DeepAnalysisJson.cs create mode 100644 src/Titano.Core/Analysis/Deep/DeepAnalysisOptions.cs create mode 100644 src/Titano.Core/Analysis/Deep/DeepAnalysisReport.cs create mode 100644 src/Titano.Core/Analysis/Deep/DeepAnalyzer.cs create mode 100644 src/Titano.Core/Analysis/Deep/DeepFrame.cs create mode 100644 src/Titano.Core/Analysis/Deep/DeepSequenceAnalysis.cs create mode 100644 src/Titano.Core/Analysis/Deep/FrameProbe.cs create mode 100644 src/Titano.Core/Analysis/Deep/HistogramDistance.cs create mode 100644 src/Titano.Core/Analysis/Deep/IFrameSource.cs create mode 100644 src/Titano.Core/Analysis/Deep/StarFieldDetector.cs create mode 100644 tests/Titano.Core.Tests/Analysis/Deep/DeepAnalyzerTests.cs create mode 100644 tests/Titano.Core.Tests/Analysis/Deep/DeepFixtures.cs diff --git a/src/Titano.Core/Analysis/Deep/CloudCoverage.cs b/src/Titano.Core/Analysis/Deep/CloudCoverage.cs new file mode 100644 index 00000000..62138d12 --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/CloudCoverage.cs @@ -0,0 +1,84 @@ +using Titano.Core.Imaging; +using Titano.Core.Mathematics; + +namespace Titano.Core.Analysis.Deep; + +/// +/// Cloud coverage estimate (BRIEF §10.2) from sky texture: the sky (weight ≥ 0.9 of a ) is +/// cut into square blocks and a block is cloudy when the 30th percentile of |Laplacian| of its sRGB-encoded luminance +/// exceeds max(·σ, ). σ is the 10th percentile of the blocks' +/// Immerkær (1996) noise estimates, so the clouds' own texture does not raise the noise floor. Cloud texture lifts nearly every pixel of a block while stars touch a few, so the low percentile ignores +/// star fields; the Laplacian ignores smooth sky gradients. A uniform overcast reads as clear sky. +/// +public static class CloudCoverage +{ + /// The 30th percentile of |Laplacian| of pure Gaussian noise is ≈ 1.7σ, so this is 3.5 × the noise level. + public const double NoiseFactor = 6; + + public const double MinimumTexture = 0.004; + + /// Fraction of sky blocks that are cloudy; NaN when no block is sky. + public static double Estimate(ImageBuffer image, ReadOnlySpan skyWeights, int block = 8) + { + ArgumentNullException.ThrowIfNull(image); + ArgumentOutOfRangeException.ThrowIfLessThan(block, 3); + if (skyWeights.Length != image.PlaneLength) throw new ArgumentException("One sky weight per pixel is required.", nameof(skyWeights)); + int w = image.Width, h = image.Height; + if (w < 3 || h < 3) return double.NaN; + var e = new float[image.PlaneLength]; + ColorSpace.Luminance(image, e); + for (var i = 0; i < e.Length; i++) e[i] = ColorSpace.EncodeFast(e[i]); + var threshold = double.Max(NoiseFactor * Noise(e, w, h, block), MinimumTexture); + + var values = new double[block * block]; + int sky = 0, cloudy = 0; + for (var by = 0; by + block <= h; by += block) + { + for (var bx = 0; bx + block <= w; bx += block) + { + double weight = 0; + var count = 0; + for (var y = by; y < by + block; y++) + { + for (var x = bx; x < bx + block; x++) + { + var i = y * w + x; + weight += skyWeights[i]; + if (x == 0 || y == 0 || x == w - 1 || y == h - 1) continue; + values[count++] = double.Abs(e[i - 1] + e[i + 1] + e[i - w] + e[i + w] - 4.0 * e[i]); + } + } + if (weight < 0.9 * block * block || count == 0) continue; + sky++; + if (Statistics.Select(values.AsSpan(0, count), (int)(0.3 * (count - 1))) > threshold) cloudy++; + } + } + return sky > 0 ? (double)cloudy / sky : double.NaN; + } + + /// 10th percentile over whole blocks of the Immerkær fast noise σ (interior pixels only). + private static double Noise(float[] e, int w, int h, int block) + { + var estimates = new List(); + for (var by = 0; by + block <= h; by += block) + { + for (var bx = 0; bx + block <= w; bx += block) + { + double sum = 0; + var count = 0; + for (var y = int.Max(1, by); y < int.Min(h - 1, by + block); y++) + { + for (var x = int.Max(1, bx); x < int.Min(w - 1, bx + block); x++) + { + var i = y * w + x; + sum += double.Abs(e[i - w - 1] + e[i - w + 1] + e[i + w - 1] + e[i + w + 1] + - 2.0 * (e[i - w] + e[i + w] + e[i - 1] + e[i + 1]) + 4.0 * e[i]); + count++; + } + } + if (count > 0) estimates.Add(double.Sqrt(double.Pi / 2) / 6 * sum / count); + } + } + return estimates.Count > 0 ? Statistics.Percentile(estimates.ToArray(), 0.1) : 0; + } +} diff --git a/src/Titano.Core/Analysis/Deep/DeepAnalysisCheckpoint.cs b/src/Titano.Core/Analysis/Deep/DeepAnalysisCheckpoint.cs new file mode 100644 index 00000000..4faad2e1 --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/DeepAnalysisCheckpoint.cs @@ -0,0 +1,108 @@ +using System.Text; +using System.Text.Json; + +namespace Titano.Core.Analysis.Deep; + +/// +/// Where persists per-frame results while it runs, so a crashed or cancelled analysis +/// resumes after the last persisted frame. +/// +public interface IDeepAnalysisCheckpoint +{ + /// + /// Frames persisted by an earlier run with the same , in index order from 0. A store + /// holding another fingerprint discards its content and starts over. + /// + ValueTask> LoadAsync(string fingerprint, CancellationToken cancellationToken); + + /// Persists the next frames (consecutive indices) durably before returning. + ValueTask AppendAsync(IReadOnlyList frames, CancellationToken cancellationToken); +} + +/// +/// Checkpoint as a JSON-lines file (e.g. .titano/analysis.partial): a header line with the fingerprint, then +/// one per line, flushed to disk after every batch. A torn last line from a crash is dropped +/// on load. Delete the file once the final report is saved. +/// +public sealed class DeepAnalysisCheckpointFile(string path) : IDeepAnalysisCheckpoint +{ + /// Conventional file name next to . + public const string FileName = "analysis.partial"; + + private static readonly UTF8Encoding Utf8 = new(false); + + public string Path { get; } = path; + + public async ValueTask> LoadAsync(string fingerprint, CancellationToken cancellationToken) + { + var frames = new List(); + if (File.Exists(Path)) + { + var bytes = await File.ReadAllBytesAsync(Path, cancellationToken).ConfigureAwait(false); + var valid = Parse(bytes, fingerprint, frames); + if (valid > 0) + { + // Cut a torn or foreign tail so later appends continue a well-formed file. + await using var stream = new FileStream(Path, FileMode.Open, FileAccess.Write, FileShare.None); + stream.SetLength(valid); + return frames; + } + } + var header = JsonSerializer.Serialize(new CheckpointHeader(fingerprint), DeepCheckpointJson.Default.CheckpointHeader) + "\n"; + Directory.CreateDirectory(System.IO.Path.GetDirectoryName(System.IO.Path.GetFullPath(Path))!); + await File.WriteAllBytesAsync(Path, Utf8.GetBytes(header), cancellationToken).ConfigureAwait(false); + return frames; + } + + public async ValueTask AppendAsync(IReadOnlyList frames, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(frames); + var text = new StringBuilder(); + foreach (var frame in frames) text.Append(JsonSerializer.Serialize(frame, DeepCheckpointJson.Default.DeepFrame)).Append('\n'); + await using var stream = new FileStream(Path, FileMode.Append, FileAccess.Write, FileShare.None); + await stream.WriteAsync(Utf8.GetBytes(text.ToString()), cancellationToken).ConfigureAwait(false); + stream.Flush(flushToDisk: true); + } + + public void Delete() + { + if (File.Exists(Path)) File.Delete(Path); + } + + /// Reads the header and the contiguous frames after it; returns the byte length of the valid prefix + /// (0 when the header is missing or belongs to another fingerprint). + private static int Parse(byte[] bytes, string fingerprint, List frames) + { + var offset = 0; + var valid = 0; + var line = 0; + while (offset < bytes.Length) + { + var end = Array.IndexOf(bytes, (byte)'\n', offset); + if (end < 0) break; + var span = bytes.AsSpan(offset, end - offset); + try + { + if (line == 0) + { + if (JsonSerializer.Deserialize(span, DeepCheckpointJson.Default.CheckpointHeader)?.Fingerprint != fingerprint) return 0; + } + else + { + var frame = JsonSerializer.Deserialize(span, DeepCheckpointJson.Default.DeepFrame); + if (frame is null || frame.Index != frames.Count) break; + frames.Add(frame); + } + } + catch (JsonException) + { + if (line == 0) return 0; + break; + } + line++; + offset = end + 1; + valid = offset; + } + return valid; + } +} diff --git a/src/Titano.Core/Analysis/Deep/DeepAnalysisJson.cs b/src/Titano.Core/Analysis/Deep/DeepAnalysisJson.cs new file mode 100644 index 00000000..7a61cc52 --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/DeepAnalysisJson.cs @@ -0,0 +1,25 @@ +using System.Text.Json.Serialization; + +namespace Titano.Core.Analysis.Deep; + +/// Source-generated JSON for analysis.json and the database summary: compact, camelCase, enums as names. +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + UseStringEnumConverter = true, + PropertyNameCaseInsensitive = true)] +[JsonSerializable(typeof(DeepAnalysisReport))] +[JsonSerializable(typeof(AnalysisSummary))] +internal sealed partial class DeepAnalysisJson : JsonSerializerContext; + +/// Checkpoint lines: exact doubles, NaN written as a literal so unmeasured values survive a resume. +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals, + UseStringEnumConverter = true)] +[JsonSerializable(typeof(DeepFrame))] +[JsonSerializable(typeof(CheckpointHeader))] +internal sealed partial class DeepCheckpointJson : JsonSerializerContext; + +/// First line of a checkpoint file. +internal sealed record CheckpointHeader(string Fingerprint); diff --git a/src/Titano.Core/Analysis/Deep/DeepAnalysisOptions.cs b/src/Titano.Core/Analysis/Deep/DeepAnalysisOptions.cs new file mode 100644 index 00000000..79c5e537 --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/DeepAnalysisOptions.cs @@ -0,0 +1,69 @@ +using System.Globalization; + +namespace Titano.Core.Analysis.Deep; + +/// Parameters of ; every value enters the checkpoint fingerprint. +public sealed record DeepAnalysisOptions +{ + public static DeepAnalysisOptions Default { get; } = new(); + + /// Upper bound of the analysis long edge (BRIEF §10.2: half native, capped at 2048). + public int MaxAnalysisLongEdge { get; init; } = 2048; + + /// Sources smaller than twice this are analysed at this size (or native when smaller). + public int MinAnalysisLongEdge { get; init; } = 512; + + /// Long edge of the plane the per-frame shift is measured on (nine phase-correlation patches). + public int CorrelationLongEdge { get; init; } = 1024; + + /// Long edge of the optical-flow images (Draft quality). + public int FlowLongEdge { get; init; } = 512; + + /// Optical flow runs on the pair (k − 1, k) for every k divisible by this. + public int FlowEvery { get; init; } = 10; + + /// Long edge of the thumbnails used for regions and cloud coverage. + public int ThumbnailLongEdge { get; init; } = 256; + + /// Frames sampled evenly for the sky/land mask. + public int RegionSampleFrames { get; init; } = 15; + + /// Frames whose trimmed mean log2 luminance is at or below this are dark (night) frames. + public double DarkLogLuminance { get; init; } = -4; + + /// Point sources per megapixel from which a dark frame shows a star field. + public double StarFieldDensity { get; init; } = 30; + + /// Smallest exposure-aligned histogram distance that can be a scene change. + public double SceneChangeMinimum { get; init; } = 0.25; + + /// σ (frames) of the camera-path smoothing used for the shake measure (as StabilizerOptions). + public double ShakeSmoothingSigma { get; init; } = 12; + + /// Frames measured concurrently; also the batch size between checkpoints. + public int MaxDegreeOfParallelism { get; init; } = int.Clamp(Environment.ProcessorCount, 1, 4); + + /// Analysis long edge for a source whose oriented long edge is (0 = unknown). + public int AnalysisLongEdge(int sourceLongEdge) + { + if (sourceLongEdge <= 0) return MaxAnalysisLongEdge; + var half = (sourceLongEdge + 1) / 2; + return int.Min(sourceLongEdge, int.Min(MaxAnalysisLongEdge, int.Max(MinAnalysisLongEdge, half))); + } + + /// Stable text of every value that changes per-frame results (parallelism excluded). + public string Fingerprint() => string.Create(CultureInfo.InvariantCulture, + $"edge={MinAnalysisLongEdge}-{MaxAnalysisLongEdge};corr={CorrelationLongEdge};flow={FlowLongEdge}/{FlowEvery};thumb={ThumbnailLongEdge};regions={RegionSampleFrames};dark={DarkLogLuminance}"); + + internal void Validate() + { + ArgumentOutOfRangeException.ThrowIfLessThan(MinAnalysisLongEdge, 16); + ArgumentOutOfRangeException.ThrowIfLessThan(MaxAnalysisLongEdge, MinAnalysisLongEdge); + ArgumentOutOfRangeException.ThrowIfLessThan(CorrelationLongEdge, 16); + ArgumentOutOfRangeException.ThrowIfLessThan(FlowLongEdge, 16); + ArgumentOutOfRangeException.ThrowIfLessThan(FlowEvery, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(ThumbnailLongEdge, 16); + ArgumentOutOfRangeException.ThrowIfLessThan(RegionSampleFrames, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(MaxDegreeOfParallelism, 1); + } +} diff --git a/src/Titano.Core/Analysis/Deep/DeepAnalysisReport.cs b/src/Titano.Core/Analysis/Deep/DeepAnalysisReport.cs new file mode 100644 index 00000000..78c9810f --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/DeepAnalysisReport.cs @@ -0,0 +1,226 @@ +using System.Text.Json; +using Titano.Core.IO; + +namespace Titano.Core.Analysis.Deep; + +/// Light condition of a whole sequence. +public enum DayNight +{ + Unknown, + Day, + Night, + DayToNight, + NightToDay, +} + +/// +/// Per-frame series of the report as compact parallel arrays (one entry per frame, null = not measured), rounded to +/// what charts and render rules need (~100 bytes per frame in JSON). Shifts and motion are in source pixels. +/// +public sealed record DeepFrameColumns +{ + /// Trimmed mean log2 luminance, 0.001 EV. + public double?[] LogLuminance { get; init; } = []; + + public double?[] ClipLow { get; init; } = []; + public double?[] ClipHigh { get; init; } = []; + + /// log2(R/G) and log2(B/G) of the mean colour, 0.001. + public double?[] RedRatio { get; init; } = []; + + public double?[] BlueRatio { get; init; } = []; + + /// Correlated colour temperature in kelvin, null when not near-Planckian. + public double?[] Kelvin { get; init; } = []; + + /// Laplacian variance and noise σ of sRGB-encoded luminance, two significant digits. + public double?[] Sharpness { get; init; } = []; + + public double?[] Noise { get; init; } = []; + + /// Content motion from the previous usable frame, 0.01 px. + public double?[] ShiftX { get; init; } = []; + + public double?[] ShiftY { get; init; } = []; + + /// Stars per megapixel of the analysis image (dark frames only). + public double?[] StarDensity { get; init; } = []; + + public double?[] CloudCoverage { get; init; } = []; + public double?[] SkyLogLuminance { get; init; } = []; + public double?[] LandLogLuminance { get; init; } = []; + public double?[] SceneDistance { get; init; } = []; + + /// per frame. + public int[] Flags { get; init; } = []; + + /// Frames k whose optical flow from k − 1 was sampled, with median and 95th-percentile magnitude (px per frame). + public int[] MotionFrames { get; init; } = []; + + public double[] MotionMedian { get; init; } = []; + public double[] MotionP95 { get; init; } = []; + + public DeepFrameFlags FlagsOf(int index) => (DeepFrameFlags)Flags[index]; +} + +/// RMS deviation of the step-free luminance from its smooth curve. +/// RMS frame-to-frame flicker relative to the curve (). +/// Period of periodic flicker (). +/// Deflicker window: 4 × period or correlation length, odd, bounded. +public sealed record FlickerSummary(double AmplitudeEv, double FrameToFrameEv, double? PeriodFrames, double PeakCorrelation, double CorrelationLength, int RecommendedWindow); + +/// Horizon row / mask height at up to 64 evenly spaced columns (horizon method only). +/// RMS of the difference between the sky and land residuals: how independently they flicker. +public sealed record RegionSummary(RegionMethod Method, double Confidence, double SkyFraction, int MaskWidth, int MaskHeight, double[]? Horizon, double DivergenceEv); + +/// One metadata exposure step as the pixels received it (). +public sealed record ExposureStepSummary(int Index, double ExpectedEv, double? MeasuredEv, bool Received, double Length); + +/// Median frames between received steps (two or more received). +/// Half the typical spacing, 12 for a single step, null without received steps. +/// 5–95 % range of the step-free luminance: how much the light itself changed. +public sealed record ExposureSummary(int MetadataSteps, int ReceivedSteps, double? TypicalSpacing, int? TransitionLength, double SceneRangeEv, IReadOnlyList Steps); + +/// RMS frame-to-frame colour-ratio deviation from the smoothed ratios (log2 units). +/// Largest range of the smoothed colour ratios (slow, deliberate change). +public sealed record WhiteBalanceSummary(double JitterLog2, double DriftLog2, double? KelvinLow, double? KelvinHigh); + +/// RMS distance of the camera path from its smoothed path, source pixels. +/// Area fraction a centred crop must give up to hide every correction. +/// Average drift of the smoothed path. +/// Share of frame pairs whose shift could be measured (textured patches). +public sealed record ShakeSummary(double RmsPx, double MaxPx, double CropCost, double PanPxPerFrame, bool PanDetected, double MeasuredFraction); + +/// Median of the sampled pairs' median flow magnitude, source px per frame. +/// Median of the sampled pairs' 95th-percentile magnitude: the typical speed of moving content. +public sealed record MotionSummary(int Pairs, double? MedianPx, double? P95Px, double? MaxP95Px); + +/// Median stars per megapixel over dark frames. +/// Mean cloudy fraction of the sky. +public sealed record SkySummary(DayNight Light, double DarkFraction, int DarkFrames, int StarFrames, double? StarDensity, bool StarField, double? CloudCoverage); + +public sealed record FrameIssues(int[] Undecodable, int[] Black, int[] White, int[] Duplicate, int[] SceneChanges) +{ + /// Every frame that would flash or stutter in a render, ascending. + public int[] Excludable() => [.. Undecodable.Concat(Black).Concat(White).Concat(Duplicate).Distinct().Order()]; +} + +/// A with numeric arguments, so the report round-trips through JSON. +public sealed record DeepFinding(string Key, FindingSeverity Severity, double[] Arguments, int[] Frames) +{ + public Finding ToFinding() => new(Key, Severity, [.. Arguments.Cast()], Frames); +} + +/// Resource keys of the findings produced by the deep analysis; arguments are numbers. +public static class DeepFindingKeys +{ + public const string UnreadableFrames = "Advisor.Finding.UnreadableFrames"; + public const string BlackFrames = "Advisor.Finding.BlackFrames"; + public const string WhiteFrames = "Advisor.Finding.WhiteFrames"; + public const string DuplicateFrames = "Advisor.Finding.DuplicateFrames"; + public const string SceneChanges = "Advisor.Finding.SceneChanges"; + public const string Flicker = "Advisor.Finding.Flicker"; + public const string Shake = "Advisor.Finding.Shake"; + public const string Panning = "Advisor.Finding.Panning"; + public const string StarField = "Advisor.Finding.StarField"; + public const string Clouds = "Advisor.Finding.Clouds"; + public const string WhiteBalanceJitter = "Advisor.Finding.WhiteBalanceJitter"; + public const string StepsNotReceived = "Advisor.Finding.StepsNotReceived"; +} + +/// +/// Result of the deep analysis (BRIEF §10.2), stored as .titano/analysis.json. Versioned; per-frame data is +/// columnar (), sequence-level results are small records. +/// +public sealed record DeepAnalysisReport +{ + public const int CurrentVersion = 1; + + /// File name inside a sequence's .titano folder. + public const string FileName = "analysis.json"; + + public int Version { get; init; } = CurrentVersion; + public int FrameCount { get; init; } + + /// Dominant oriented source size from metadata (0 when unknown): the pixel unit of shifts and motion. + public int SourceWidth { get; init; } + + public int SourceHeight { get; init; } + public int AnalysisLongEdge { get; init; } + public DeepFrameColumns Frames { get; init; } = new(); + public required FlickerSummary Flicker { get; init; } + public RegionSummary? Regions { get; init; } + public required ExposureSummary Exposure { get; init; } + public required WhiteBalanceSummary WhiteBalance { get; init; } + public required ShakeSummary Shake { get; init; } + public required MotionSummary Motion { get; init; } + public required SkySummary Sky { get; init; } + public required FrameIssues Issues { get; init; } + + /// Suggested hero frame: sharp, well exposed, near golden hour or mid-sequence, not excluded. + public int? HeroIndex { get; init; } + + public IReadOnlyList Findings { get; init; } = []; + + public string ToJson() => JsonSerializer.Serialize(this, DeepAnalysisJson.Default.DeepAnalysisReport); + + public static DeepAnalysisReport FromJson(string json) + { + var report = JsonSerializer.Deserialize(json, DeepAnalysisJson.Default.DeepAnalysisReport) + ?? throw new InvalidDataException("analysis.json is empty."); + if (report.Version > CurrentVersion) + throw new InvalidDataException($"analysis.json version {report.Version} is newer than this Titano ({CurrentVersion})."); + return report; + } + + /// Writes the report atomically (temp file, flush, rename). + public void Save(string path) => AtomicFile.WriteAllText(path, ToJson()); + + public static DeepAnalysisReport Load(string path) => FromJson(File.ReadAllText(path)); +} + +/// Compact digest of a for the sequence row of the database (BRIEF §5). +public sealed record AnalysisSummary +{ + public int Version { get; init; } = DeepAnalysisReport.CurrentVersion; + public int FrameCount { get; init; } + public DayNight Light { get; init; } + public double FlickerEv { get; init; } + public double? FlickerPeriod { get; init; } + + /// Shake RMS in source pixels. + public double ShakePx { get; init; } + + /// Typical speed of moving content (). + public double? MotionPx { get; init; } + + public bool StarField { get; init; } + public double? StarDensity { get; init; } + public double? CloudCoverage { get; init; } + public int WarningCount { get; init; } + public int? HeroIndex { get; init; } + + public static AnalysisSummary From(DeepAnalysisReport report) + { + ArgumentNullException.ThrowIfNull(report); + return new AnalysisSummary + { + FrameCount = report.FrameCount, + Light = report.Sky.Light, + FlickerEv = report.Flicker.AmplitudeEv, + FlickerPeriod = report.Flicker.PeriodFrames, + ShakePx = report.Shake.RmsPx, + MotionPx = report.Motion.P95Px, + StarField = report.Sky.StarField, + StarDensity = report.Sky.StarDensity, + CloudCoverage = report.Sky.CloudCoverage, + WarningCount = report.Findings.Count(f => f.Severity >= FindingSeverity.Warning), + HeroIndex = report.HeroIndex, + }; + } + + public string ToJson() => JsonSerializer.Serialize(this, DeepAnalysisJson.Default.AnalysisSummary); + + public static AnalysisSummary FromJson(string json) => + JsonSerializer.Deserialize(json, DeepAnalysisJson.Default.AnalysisSummary) ?? throw new InvalidDataException("Analysis summary is empty."); +} diff --git a/src/Titano.Core/Analysis/Deep/DeepAnalyzer.cs b/src/Titano.Core/Analysis/Deep/DeepAnalyzer.cs new file mode 100644 index 00000000..e0e55593 --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/DeepAnalyzer.cs @@ -0,0 +1,210 @@ +using System.Diagnostics; +using Titano.Core.Imaging; +using Titano.Core.Motion; + +namespace Titano.Core.Analysis.Deep; + +/// +/// Deep analysis (BRIEF §10.2): one streaming pass over every frame. A pre-pass segments sky and land on a sample of +/// thumbnails; then frames are probed in parallel batches (: statistics, stars, region +/// luminance, clouds) and compared in order with the previous usable frame (nine-patch phase-correlation shift, duplicate +/// test, exposure-aligned histogram distance, Draft optical flow on every -th +/// pair). Memory is one batch of analysis images plus the previous frame's reduced planes. After every batch the new +/// s go to the checkpoint, so an interrupted run resumes after the last persisted frame with +/// identical results. Sequence-level results are computed by . +/// +public static class DeepAnalyzer +{ + public static async Task AnalyzeAsync( + IFrameSource source, + DeepAnalysisOptions? options = null, + IDeepAnalysisCheckpoint? checkpoint = null, + IProgress? progress = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(source); + options ??= DeepAnalysisOptions.Default; + options.Validate(); + var n = source.Count; + var metadata = new LightFrame[n]; + for (var i = 0; i < n; i++) metadata[i] = source.Metadata(i); + + var frames = new DeepFrame[n]; + var start = 0; + if (checkpoint is not null) + { + var saved = await checkpoint.LoadAsync(Fingerprint(n, options), cancellationToken).ConfigureAwait(false); + while (start < saved.Count && start < n && saved[start].Index == start) frames[start] = saved[start++]; + } + + var mask = n > 0 ? await FrameProbe.SegmentRegionsAsync(source, options, cancellationToken).ConfigureAwait(false) : null; + using var state = new SequentialState(options); + await PrimeAsync(source, frames, start, options, mask, state, cancellationToken).ConfigureAwait(false); + + var clock = Stopwatch.StartNew(); + var batch = options.MaxDegreeOfParallelism; + for (var first = start; first < n; first += batch) + { + cancellationToken.ThrowIfCancellationRequested(); + var count = int.Min(batch, n - first); + var tasks = new Task[count]; + for (var k = 0; k < count; k++) + { + var index = first + k; + tasks[k] = Task.Run(() => FrameProbe.RunAsync(source, index, options, mask, cancellationToken), cancellationToken); + } + var produced = new DeepFrame[count]; + try + { + await Task.WhenAll(tasks).ConfigureAwait(false); + for (var k = 0; k < count; k++) produced[k] = frames[first + k] = state.Next(tasks[k].Result, cancellationToken); + } + finally + { + foreach (var task in tasks) + { + if (task.IsCompletedSuccessfully) task.Result.Dispose(); + } + } + if (checkpoint is not null) await checkpoint.AppendAsync(produced, cancellationToken).ConfigureAwait(false); + if (progress is not null) + { + var done = first + count; + var rate = (done - start) / double.Max(clock.Elapsed.TotalSeconds, 1e-3); + progress.Report(new AnalysisProgress(done, n, rate, TimeSpan.FromSeconds((n - done) / rate))); + } + } + + return DeepSequenceAnalysis.Build(frames, metadata, mask, options); + } + + /// Identifies checkpoints written by this version for this frame count and these options. + public static string Fingerprint(int frameCount, DeepAnalysisOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return $"titano-deep/{DeepAnalysisReport.CurrentVersion};frames={frameCount};{options.Fingerprint()}"; + } + + /// Restores the comparison state of a resumed run from the last usable frame before . + private static async Task PrimeAsync(IFrameSource source, DeepFrame[] frames, int start, DeepAnalysisOptions options, RegionMask? mask, SequentialState state, CancellationToken cancellationToken) + { + var last = start - 1; + while (last >= 0 && !frames[last].IsUsable) last--; + if (last < 0) return; + using var probe = await FrameProbe.RunAsync(source, last, options, mask, cancellationToken).ConfigureAwait(false); + state.Next(probe, cancellationToken); + } + + /// + /// The order-dependent comparisons: each usable frame is compared with the previous usable frame and becomes the + /// new reference; undecodable, black and white frames leave the state untouched. + /// + private sealed class SequentialState(DeepAnalysisOptions options) : IDisposable + { + private const int MinimumStabilizerSize = 16; + private readonly OpticalFlowEngine _flow = new(FlowQuality.Draft); + private Stabilizer? _stabilizer; + private FrameStatistics? _previous; + private ImageBuffer? _previousFlow; + private int _previousIndex = -1; + + public DeepFrame Next(FrameProbe probe, CancellationToken cancellationToken) + { + if (probe.Statistics is not { } stats) return new DeepFrame { Index = probe.Index, Flags = DeepFrameFlags.Undecodable }; + var frame = Measured(probe.Index, stats); + if (!probe.IsUsable) return frame; + + var (shiftX, shiftY) = Shift(probe); + var duplicate = _previous is not null && FrameStatistics.AreLikelyDuplicates(_previous, stats); + var scene = _previous is null ? double.NaN : HistogramDistance.ExposureAligned(_previous, stats); + var (motionMedian, motionP95) = Motion(probe, stats, cancellationToken); + frame = frame with + { + Flags = frame.Flags | (duplicate ? DeepFrameFlags.Duplicate : DeepFrameFlags.None), + ShiftX = shiftX, + ShiftY = shiftY, + MotionMedian = motionMedian, + MotionP95 = motionP95, + StarDensity = probe.StarDensity, + CloudCoverage = probe.CloudCoverage, + SkyLogLuminance = probe.SkyLogLuminance, + LandLogLuminance = probe.LandLogLuminance, + SceneDistance = scene, + }; + + _previous = stats; + _previousIndex = probe.Index; + _previousFlow?.Dispose(); + _previousFlow = null; + _previousFlow = probe.TakeFlow(); + return frame; + } + + public void Dispose() + { + _flow.Dispose(); + _stabilizer?.Dispose(); + _previousFlow?.Dispose(); + } + + private DeepFrame Measured(int index, FrameStatistics stats) + { + var flags = DeepFrameFlags.None; + if (stats.IsBlack()) flags |= DeepFrameFlags.Black; + if (stats.IsWhite()) flags |= DeepFrameFlags.White; + if (stats.MeanLogLuminance <= options.DarkLogLuminance) flags |= DeepFrameFlags.Dark; + var green = stats.MeanGreen; + return new DeepFrame + { + Index = index, + Flags = flags, + LogLuminance = stats.MeanLogLuminance, + ClipLow = stats.ClipLowFraction, + ClipHigh = stats.ClipHighFraction, + RedRatio = stats.MeanRed > 0 && green > 0 ? double.Log2(stats.MeanRed / green) : double.NaN, + BlueRatio = stats.MeanBlue > 0 && green > 0 ? double.Log2(stats.MeanBlue / green) : double.NaN, + Kelvin = stats.ColorTemperature?.Kelvin ?? double.NaN, + Sharpness = stats.Sharpness, + Noise = stats.Noise, + }; + } + + /// Content motion at the centre of the correlation plane, in source pixels; NaN for the first frame of a + /// size, too small planes and frames without textured patches. + private (double X, double Y) Shift(FrameProbe probe) + { + int w = probe.CorrelationWidth, h = probe.CorrelationHeight; + if (w < MinimumStabilizerSize || h < MinimumStabilizerSize || probe.Correlation is null) return (double.NaN, double.NaN); + if (_stabilizer is null || _stabilizer.Width != w || _stabilizer.Height != h) + { + _stabilizer?.Dispose(); + _stabilizer = new Stabilizer(w, h); + _stabilizer.AddFrame(probe.Correlation); + return (double.NaN, double.NaN); + } + var motion = _stabilizer.AddFrame(probe.Correlation); + if (!motion.Estimated) return (double.NaN, double.NaN); + double cx = (w - 1) / 2.0, cy = (h - 1) / 2.0; + var (x, y) = motion.Transform.Apply(cx, cy); + return ((x - cx) * probe.CorrelationScale, (y - cy) * probe.CorrelationScale); + } + + /// Draft flow from the previous frame with the current frame brought to the previous mean level, so + /// flicker and exposure steps do not read as motion; magnitudes in source pixels per frame. + private (double Median, double P95) Motion(FrameProbe probe, FrameStatistics stats, CancellationToken cancellationToken) + { + if (probe.Index % options.FlowEvery != 0 || _previousIndex != probe.Index - 1) return (double.NaN, double.NaN); + if (probe.Flow is not { } current || _previousFlow is not { } previous || previous.Width != current.Width || previous.Height != current.Height) + return (double.NaN, double.NaN); + using var levelled = current.Clone(); + var gain = (float)double.Exp2(_previous!.MeanLogLuminance - stats.MeanLogLuminance); + if (float.IsFinite(gain)) + { + foreach (ref var v in levelled.Data) v *= gain; + } + using var field = _flow.Compute(previous, levelled, cancellationToken); + var statistics = field.Statistics(); + return (statistics.MedianMagnitude * probe.FlowScale, statistics.P95Magnitude * probe.FlowScale); + } + } +} diff --git a/src/Titano.Core/Analysis/Deep/DeepFrame.cs b/src/Titano.Core/Analysis/Deep/DeepFrame.cs new file mode 100644 index 00000000..f2884ef4 --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/DeepFrame.cs @@ -0,0 +1,82 @@ +namespace Titano.Core.Analysis.Deep; + +/// Per-frame conditions found by the deep analysis. +[Flags] +public enum DeepFrameFlags +{ + None = 0, + + /// The source could not decode the frame; every measurement is NaN. + Undecodable = 1, + + Black = 2, + White = 4, + + /// Statistically identical to the previous decoded frame. + Duplicate = 8, + + /// Exposure-aligned histogram jump from the previous frame (camera moved or bumped); set at sequence level. + SceneChange = 16, + + /// Trimmed mean log2 luminance at or below . + Dark = 32, + + /// Dark frame with at least stars per megapixel; set at sequence level. + StarField = 64, +} + +/// +/// Exact per-frame measurements of : what a checkpoint persists and what the +/// sequence-level results are computed from. NaN marks a value that was not measured. Shifts and motion are in +/// source pixels (oriented full resolution). +/// +public sealed record DeepFrame +{ + public int Index { get; init; } + public DeepFrameFlags Flags { get; init; } + + /// Trimmed mean log2 luminance (). + public double LogLuminance { get; init; } = double.NaN; + + public double ClipLow { get; init; } = double.NaN; + public double ClipHigh { get; init; } = double.NaN; + + /// log2(R/G) of the mean colour (white-balance drift signal). + public double RedRatio { get; init; } = double.NaN; + + /// log2(B/G) of the mean colour. + public double BlueRatio { get; init; } = double.NaN; + + /// Correlated colour temperature of the mean colour; NaN when not near-Planckian. + public double Kelvin { get; init; } = double.NaN; + + public double Sharpness { get; init; } = double.NaN; + public double Noise { get; init; } = double.NaN; + + /// Content motion from the previous decoded frame at the image centre (nine-patch similarity fit). + public double ShiftX { get; init; } = double.NaN; + + public double ShiftY { get; init; } = double.NaN; + + /// Median optical-flow magnitude from the previous frame, px per frame (sampled pairs only). + public double MotionMedian { get; init; } = double.NaN; + + /// 95th-percentile optical-flow magnitude from the previous frame (sampled pairs only). + public double MotionP95 { get; init; } = double.NaN; + + /// Point sources per megapixel of the analysis image (dark frames only). + public double StarDensity { get; init; } = double.NaN; + + /// Fraction of sky blocks with cloud texture (frames with a sky region only). + public double CloudCoverage { get; init; } = double.NaN; + + /// Trimmed mean log2 luminance of the sky region (region deflicker signal). + public double SkyLogLuminance { get; init; } = double.NaN; + + public double LandLogLuminance { get; init; } = double.NaN; + + /// Exposure-aligned chi-square histogram distance from the previous decoded frame. + public double SceneDistance { get; init; } = double.NaN; + + public bool IsUsable => (Flags & (DeepFrameFlags.Undecodable | DeepFrameFlags.Black | DeepFrameFlags.White)) == 0; +} diff --git a/src/Titano.Core/Analysis/Deep/DeepSequenceAnalysis.cs b/src/Titano.Core/Analysis/Deep/DeepSequenceAnalysis.cs new file mode 100644 index 00000000..bb1a4d77 --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/DeepSequenceAnalysis.cs @@ -0,0 +1,450 @@ +using Titano.Core.Mathematics; +using Titano.Core.Motion; + +namespace Titano.Core.Analysis.Deep; + +/// +/// Sequence-level results of the deep analysis, computed from the exact s and the frame +/// metadata: exposure steps as received (), flicker spectrum and amplitude on the +/// step-free luminance (, ), region divergence, white-balance +/// jitter (), shake from the integrated shifts (camera path minus its +/// smoothing, as ), motion, stars, clouds, day/night, +/// scene changes, the hero frame and the findings. +/// +public static class DeepSequenceAnalysis +{ + /// Flicker below this RMS is not visible (≈ 1.4 % brightness). + public const double VisibleFlickerEv = 0.02; + + /// Colour-ratio jitter (log2 RMS) above which white-balance smoothing helps (≈ 0.7 %). + public const double VisibleWhiteBalanceJitter = 0.01; + + /// Mid-grey (18 %) in log2 luminance, the hero frame's exposure target. + private const double MidGreyEv = -2.47; + + private const int HorizonSamples = 64; + + public static DeepAnalysisReport Build(IReadOnlyList frames, IReadOnlyList metadata, RegionMask? mask, DeepAnalysisOptions? options = null) + { + ArgumentNullException.ThrowIfNull(frames); + ArgumentNullException.ThrowIfNull(metadata); + options ??= DeepAnalysisOptions.Default; + var n = frames.Count; + if (metadata.Count != n) throw new ArgumentException("One metadata entry per frame is required.", nameof(metadata)); + var light = LightAnalyzer.Analyze(metadata); + var (sourceWidth, sourceHeight) = light.DominantSize ?? (0, 0); + var luminance = new double[n]; + for (var i = 0; i < n; i++) luminance[i] = frames[i].IsUsable ? frames[i].LogLuminance : double.NaN; + + var (exposure, offsets) = Exposure(luminance, light); + var flat = new double[n]; + for (var i = 0; i < n; i++) flat[i] = luminance[i] - offsets[i]; + exposure = exposure with { SceneRangeEv = Range(flat, 0.05, 0.95) }; + var flicker = Flicker(flat); + var sceneChanges = SceneChanges(frames, options); + var starFrames = new HashSet(); + var sky = Sky(frames, flat, light, options, starFrames); + var shake = Shake(frames, sourceWidth, sourceHeight, options); + var issues = new FrameIssues( + Where(frames, f => (f.Flags & DeepFrameFlags.Undecodable) != 0), + Where(frames, f => (f.Flags & DeepFrameFlags.Black) != 0), + Where(frames, f => (f.Flags & DeepFrameFlags.White) != 0), + Where(frames, f => (f.Flags & DeepFrameFlags.Duplicate) != 0), + [.. sceneChanges.Order()]); + var report = new DeepAnalysisReport + { + FrameCount = n, + SourceWidth = sourceWidth, + SourceHeight = sourceHeight, + AnalysisLongEdge = options.AnalysisLongEdge(int.Max(sourceWidth, sourceHeight)), + Frames = Columns(frames, sceneChanges, starFrames), + Flicker = flicker, + Regions = Regions(frames, mask, flicker.RecommendedWindow), + Exposure = exposure, + WhiteBalance = WhiteBalance(frames), + Shake = shake, + Motion = Motion(frames), + Sky = sky, + Issues = issues, + HeroIndex = Hero(frames, sceneChanges, light.DayPhases), + }; + return report with { Findings = Findings(report) }; + } + + private static (ExposureSummary Summary, double[] Offsets) Exposure(double[] luminance, LightAnalysisReport light) + { + var n = luminance.Length; + if (light.ExposureSteps.Count == 0 || n == 0) return (new ExposureSummary(0, 0, null, null, 0, []), new double[n]); + var result = HolyGrailEngine.Compute(luminance, light.ExposureValues); + var received = result.Transitions.Where(t => t.Received).Select(t => t.Index).ToArray(); + double? spacing = null; + int? length = null; + if (received.Length >= 2) + { + var gaps = new double[received.Length - 1]; + for (var i = 1; i < received.Length; i++) gaps[i - 1] = received[i] - received[i - 1]; + spacing = Statistics.Median(gaps); + length = int.Max(2, (int)double.Round(spacing.Value / 2)); + } + else if (received.Length == 1) + { + length = HolyGrailOptions.Default.SingleStepTransitionFrames; + } + var steps = result.Transitions + .Select(t => new ExposureStepSummary(t.Index, Round(t.ExpectedEv, 3), double.IsFinite(t.MeasuredEv) ? Round(t.MeasuredEv, 3) : null, t.Received, Round(t.Length, 2))) + .ToList(); + return (new ExposureSummary(result.Transitions.Count, received.Length, spacing, length, 0, steps), result.StepOffsets); + } + + private static FlickerSummary Flicker(double[] flat) + { + if (Count(flat) < 3) return new FlickerSummary(0, 0, null, 0, 0, FlickerSpectrumOptions.Default.MinimumWindow); + var spectrum = FlickerSpectrum.Analyze(flat); + var diagnostics = DeflickerEngine.Compute(flat, new DeflickerOptions { Window = spectrum.RecommendedWindow }).Diagnostics; + return new FlickerSummary( + Round(diagnostics.ResidualRms, 4), + Round(diagnostics.FlickerBefore, 4), + spectrum.DominantPeriod is { } p ? Round(p, 2) : null, + Round(spectrum.PeakCorrelation, 3), + Round(spectrum.CorrelationLength, 2), + spectrum.RecommendedWindow); + } + + private static RegionSummary? Regions(IReadOnlyList frames, RegionMask? mask, int window) + { + if (mask is null) return null; + double[]? horizon = null; + if (mask.Horizon is { } h) + { + var count = int.Min(HorizonSamples, mask.Width); + horizon = new double[count]; + for (var k = 0; k < count; k++) + { + var x = count > 1 ? (int)((long)k * (mask.Width - 1) / (count - 1)) : 0; + horizon[k] = Round(h[x] / mask.Height, 4); + } + } + double skyFraction = 0; + foreach (var w in mask.Sky) skyFraction += w; + skyFraction /= mask.Sky.Length; + + var n = frames.Count; + var sky = new double[n]; + var land = new double[n]; + for (var i = 0; i < n; i++) + { + sky[i] = frames[i].IsUsable ? frames[i].SkyLogLuminance : double.NaN; + land[i] = frames[i].IsUsable ? frames[i].LandLogLuminance : double.NaN; + } + var divergence = 0.0; + if (Count(sky) >= 3 && Count(land) >= 3) + { + var skyFit = LocalRegression.Smooth(sky, window, 1, 3).Fitted; + var landFit = LocalRegression.Smooth(land, window, 1, 3).Fitted; + var difference = new double[n]; + for (var i = 0; i < n; i++) difference[i] = sky[i] - skyFit[i] - (land[i] - landFit[i]); + divergence = Statistics.Rms(difference); + } + return new RegionSummary(mask.Method, Round(mask.Confidence, 3), Round(skyFraction, 3), mask.Width, mask.Height, horizon, + Round(double.IsFinite(divergence) ? divergence : 0, 4)); + } + + private static WhiteBalanceSummary WhiteBalance(IReadOnlyList frames) + { + var n = frames.Count; + var red = new double[n]; + var green = new double[n]; + var blue = new double[n]; + var kelvin = new double[n]; + for (var i = 0; i < n; i++) + { + var f = frames[i]; + var valid = f.IsUsable && double.IsFinite(f.RedRatio) && double.IsFinite(f.BlueRatio); + red[i] = valid ? double.Exp2(f.RedRatio) : double.NaN; + green[i] = valid ? 1 : double.NaN; + blue[i] = valid ? double.Exp2(f.BlueRatio) : double.NaN; + kelvin[i] = f.IsUsable ? f.Kelvin : double.NaN; + } + if (Count(red) < 3) return new WhiteBalanceSummary(0, 0, null, null); + var result = WhiteBalanceSmoother.Smooth(red, green, blue); + var deviation = new double[n]; + for (var i = 0; i < n; i++) + { + var dr = result.RedRatio[i] - result.SmoothedRedRatio[i]; + var db = result.BlueRatio[i] - result.SmoothedBlueRatio[i]; + deviation[i] = double.Sqrt(dr * dr + db * db); + } + var drift = double.Max(Range(result.SmoothedRedRatio, 0, 1), Range(result.SmoothedBlueRatio, 0, 1)); + var hasKelvin = Count(kelvin) > 0; + return new WhiteBalanceSummary( + Round(Statistics.Rms(deviation), 4), + Round(drift, 4), + hasKelvin ? double.Round(Statistics.Percentile(kelvin, 0.05)) : null, + hasKelvin ? double.Round(Statistics.Percentile(kelvin, 0.95)) : null); + } + + private static ShakeSummary Shake(IReadOnlyList frames, int sourceWidth, int sourceHeight, DeepAnalysisOptions options) + { + var n = frames.Count; + if (n < 2) return new ShakeSummary(0, 0, 0, 0, false, 0); + var x = new double[n]; + var y = new double[n]; + int measured = 0, pairs = 0; + for (var i = 1; i < n; i++) + { + var f = frames[i]; + var known = double.IsFinite(f.ShiftX) && double.IsFinite(f.ShiftY); + if (f.IsUsable) pairs++; + if (known) measured++; + x[i] = x[i - 1] + (known ? f.ShiftX : 0); + y[i] = y[i - 1] + (known ? f.ShiftY : 0); + } + var sx = PathSmoother.LocalLinear(x, options.ShakeSmoothingSigma); + var sy = PathSmoother.LocalLinear(y, options.ShakeSmoothingSigma); + double sum = 0, max = 0, maxX = 0, maxY = 0; + for (var i = 0; i < n; i++) + { + double rx = x[i] - sx[i], ry = y[i] - sy[i]; + var r2 = rx * rx + ry * ry; + sum += r2; + max = double.Max(max, double.Sqrt(r2)); + maxX = double.Max(maxX, double.Abs(rx)); + maxY = double.Max(maxY, double.Abs(ry)); + } + var width = sourceWidth > 0 ? sourceWidth : options.AnalysisLongEdge(0); + var height = sourceHeight > 0 ? sourceHeight : width; + var scale = double.Clamp(double.Min(1 - 2 * maxX / width, 1 - 2 * maxY / height), 0, 1); + var drift = double.Hypot(sx[n - 1] - sx[0], sy[n - 1] - sy[0]); + return new ShakeSummary( + Round(double.Sqrt(sum / n), 3), + Round(max, 3), + Round(1 - scale * scale, 4), + Round(drift / (n - 1), 3), + drift >= 0.02 * width, + Round(pairs > 1 ? double.Min(1, measured / (double)(pairs - 1)) : 0, 3)); + } + + private static MotionSummary Motion(IReadOnlyList frames) + { + var medians = frames.Where(f => double.IsFinite(f.MotionP95)).Select(f => f.MotionMedian).ToArray(); + var p95 = frames.Where(f => double.IsFinite(f.MotionP95)).Select(f => f.MotionP95).ToArray(); + if (p95.Length == 0) return new MotionSummary(0, null, null, null); + return new MotionSummary(p95.Length, Round(Statistics.Median(medians), 3), Round(Statistics.Median(p95), 3), Round(p95.Max(), 3)); + } + + private static SkySummary Sky(IReadOnlyList frames, double[] flat, LightAnalysisReport light, DeepAnalysisOptions options, HashSet starFrames) + { + int usable = 0, dark = 0; + var densities = new List(); + var clouds = new List(); + foreach (var f in frames) + { + if (!f.IsUsable) continue; + usable++; + if (double.IsFinite(f.CloudCoverage)) clouds.Add(f.CloudCoverage); + if ((f.Flags & DeepFrameFlags.Dark) == 0) continue; + dark++; + if (!double.IsFinite(f.StarDensity)) continue; + densities.Add(f.StarDensity); + if (f.StarDensity >= options.StarFieldDensity) starFrames.Add(f.Index); + } + var darkFraction = usable > 0 ? (double)dark / usable : 0; + return new SkySummary( + DayNightOf(flat, darkFraction, usable, light.DayPhases), + Round(darkFraction, 3), + dark, + starFrames.Count, + densities.Count > 0 ? double.Round(Statistics.Median(densities.ToArray())) : null, + starFrames.Count > 0 && starFrames.Count >= 0.25 * usable, + clouds.Count > 0 ? Round(clouds.Average(), 3) : null); + } + + /// + /// Solar phases when GPS gave them for most frames; otherwise the step-free luminance: a 10–90 % range of at + /// least 3 EV is a transition (direction from the first and last tenth), else the dark-frame share decides. + /// + private static DayNight DayNightOf(double[] flat, double darkFraction, int usable, DayPhase?[] phases) + { + int day = 0, night = 0, known = 0; + DayPhase? first = null, last = null; + foreach (var phase in phases) + { + if (phase is not { } p) continue; + known++; + first ??= p; + last = p; + if (p is DayPhase.Day or DayPhase.GoldenHour) day++; + else if (p is DayPhase.Night or DayPhase.AstronomicalTwilight or DayPhase.NauticalTwilight) night++; + } + if (known * 2 >= phases.Length && known > 0) + { + if (day > 0 && night > 0) return first >= DayPhase.CivilTwilight && last < DayPhase.CivilTwilight ? DayNight.DayToNight : DayNight.NightToDay; + return night > day ? DayNight.Night : DayNight.Day; + } + if (usable == 0) return DayNight.Unknown; + if (Range(flat, 0.1, 0.9) >= 3) + { + var tenth = int.Max(1, flat.Length / 10); + var start = Statistics.Mean(flat.AsSpan(0, tenth)); + var end = Statistics.Mean(flat.AsSpan(flat.Length - tenth)); + if (double.IsFinite(start) && double.IsFinite(end)) return start > end ? DayNight.DayToNight : DayNight.NightToDay; + } + return darkFraction >= 0.5 ? DayNight.Night : DayNight.Day; + } + + /// Spikes of the exposure-aligned histogram distance: above max(minimum, median + 10 robust σ). + private static HashSet SceneChanges(IReadOnlyList frames, DeepAnalysisOptions options) + { + var candidates = frames.Where(f => f.IsUsable && (f.Flags & DeepFrameFlags.Duplicate) == 0 && double.IsFinite(f.SceneDistance)).ToList(); + var result = new HashSet(); + if (candidates.Count == 0) return result; + var distances = candidates.Select(f => f.SceneDistance).ToArray(); + var median = Statistics.Median(distances); + var sigma = Statistics.MadToSigma * Statistics.MedianAbsoluteDeviation(distances, median); + var threshold = double.Max(options.SceneChangeMinimum, median + 10 * sigma); + foreach (var f in candidates) + { + if (f.SceneDistance > threshold) result.Add(f.Index); + } + return result; + } + + /// + /// Highest √(relative sharpness) × exposure × time score over usable, unique, non-scene-change frames. Exposure + /// favours mid-grey and penalises clipping; time favours golden hour (when GPS gave phases) or the middle of the + /// sequence. Ties go to the earliest frame. + /// + private static int? Hero(IReadOnlyList frames, HashSet sceneChanges, DayPhase?[] phases) + { + var candidates = frames.Where(f => f.IsUsable && (f.Flags & DeepFrameFlags.Duplicate) == 0 && !sceneChanges.Contains(f.Index) + && double.IsFinite(f.Sharpness) && double.IsFinite(f.LogLuminance)).ToList(); + if (candidates.Count == 0) return null; + var medianSharpness = Statistics.Median(candidates.Select(f => f.Sharpness).ToArray()); + var n = frames.Count; + int? best = null; + var bestScore = double.NegativeInfinity; + foreach (var f in candidates) + { + var sharp = medianSharpness > 0 ? double.Min(2, f.Sharpness / medianSharpness) : 1; + var level = (f.LogLuminance - MidGreyEv) / 1.5; + var exposure = double.Exp(-level * level) * double.Max(0, 1 - 5 * f.ClipHigh - 2 * f.ClipLow); + var phase = f.Index < phases.Length ? phases[f.Index] : null; + var time = phase switch + { + DayPhase.GoldenHour => 1.0, + DayPhase.CivilTwilight => 0.8, + DayPhase.Day => 0.6, + null => n > 1 ? 1 - 2 * double.Abs((double)f.Index / (n - 1) - 0.5) : 1, + _ => 0.4, + }; + var score = double.Sqrt(sharp) * exposure * (0.5 + 0.5 * time); + if (score > bestScore) + { + bestScore = score; + best = f.Index; + } + } + return best; + } + + private static List Findings(DeepAnalysisReport report) + { + var findings = new List(); + void Add(string key, FindingSeverity severity, int[] frames, params double[] args) => + findings.Add(new DeepFinding(key, severity, args, frames.Length > Finding.MaxFrames ? frames[..Finding.MaxFrames] : frames)); + + var issues = report.Issues; + if (issues.Undecodable.Length > 0) Add(DeepFindingKeys.UnreadableFrames, FindingSeverity.Warning, issues.Undecodable, issues.Undecodable.Length); + if (issues.Black.Length > 0) Add(DeepFindingKeys.BlackFrames, FindingSeverity.Warning, issues.Black, issues.Black.Length); + if (issues.White.Length > 0) Add(DeepFindingKeys.WhiteFrames, FindingSeverity.Warning, issues.White, issues.White.Length); + if (issues.Duplicate.Length > 0) Add(DeepFindingKeys.DuplicateFrames, FindingSeverity.Warning, issues.Duplicate, issues.Duplicate.Length); + if (issues.SceneChanges.Length > 0) Add(DeepFindingKeys.SceneChanges, FindingSeverity.Warning, issues.SceneChanges, issues.SceneChanges.Length); + var ignored = report.Exposure.Steps.Where(s => !s.Received).Select(s => s.Index).ToArray(); + if (ignored.Length > 0) Add(DeepFindingKeys.StepsNotReceived, FindingSeverity.Warning, ignored, ignored.Length); + if (report.Flicker.AmplitudeEv >= VisibleFlickerEv) Add(DeepFindingKeys.Flicker, FindingSeverity.Info, [], Round(report.Flicker.AmplitudeEv, 3)); + if (report.Shake.RmsPx >= 0.3) Add(DeepFindingKeys.Shake, FindingSeverity.Info, [], Round(report.Shake.RmsPx, 2), Round(100 * report.Shake.CropCost, 1)); + if (report.Shake.PanDetected) Add(DeepFindingKeys.Panning, FindingSeverity.Info, [], Round(report.Shake.PanPxPerFrame, 2)); + if (report.Sky.StarField) Add(DeepFindingKeys.StarField, FindingSeverity.Info, [], report.Sky.StarFrames); + if (report.Sky.CloudCoverage is { } cover && cover >= 0.1) Add(DeepFindingKeys.Clouds, FindingSeverity.Info, [], double.Round(100 * cover)); + if (report.WhiteBalance.JitterLog2 >= VisibleWhiteBalanceJitter) + Add(DeepFindingKeys.WhiteBalanceJitter, FindingSeverity.Info, [], Round(100 * (double.Exp2(report.WhiteBalance.JitterLog2) - 1), 1)); + return findings; + } + + private static DeepFrameColumns Columns(IReadOnlyList frames, HashSet sceneChanges, HashSet starFrames) + { + var n = frames.Count; + double?[] Column(Func value, Func quantise) + { + var column = new double?[n]; + for (var i = 0; i < n; i++) + { + var v = value(frames[i]); + column[i] = double.IsFinite(v) ? quantise(v) : null; + } + return column; + } + + var flags = new int[n]; + for (var i = 0; i < n; i++) + { + var f = frames[i].Flags; + if (sceneChanges.Contains(i)) f |= DeepFrameFlags.SceneChange; + if (starFrames.Contains(i)) f |= DeepFrameFlags.StarField; + flags[i] = (int)f; + } + var sampled = frames.Where(f => double.IsFinite(f.MotionP95)).ToArray(); + return new DeepFrameColumns + { + LogLuminance = Column(f => f.LogLuminance, v => Round(v, 3)), + ClipLow = Column(f => f.ClipLow, v => Round(v, 3)), + ClipHigh = Column(f => f.ClipHigh, v => Round(v, 3)), + RedRatio = Column(f => f.RedRatio, v => Round(v, 3)), + BlueRatio = Column(f => f.BlueRatio, v => Round(v, 3)), + Kelvin = Column(f => f.Kelvin, double.Round), + Sharpness = Column(f => f.Sharpness, v => Significant(v, 2)), + Noise = Column(f => f.Noise, v => Significant(v, 2)), + ShiftX = Column(f => f.ShiftX, v => Round(v, 2)), + ShiftY = Column(f => f.ShiftY, v => Round(v, 2)), + StarDensity = Column(f => f.StarDensity, double.Round), + CloudCoverage = Column(f => f.CloudCoverage, v => Round(v, 2)), + SkyLogLuminance = Column(f => f.SkyLogLuminance, v => Round(v, 3)), + LandLogLuminance = Column(f => f.LandLogLuminance, v => Round(v, 3)), + SceneDistance = Column(f => f.SceneDistance, v => Round(v, 3)), + Flags = flags, + MotionFrames = [.. sampled.Select(f => f.Index)], + MotionMedian = [.. sampled.Select(f => Round(f.MotionMedian, 2))], + MotionP95 = [.. sampled.Select(f => Round(f.MotionP95, 2))], + }; + } + + private static int[] Where(IReadOnlyList frames, Func predicate) => [.. frames.Where(predicate).Select(f => f.Index)]; + + private static int Count(double[] values) + { + var count = 0; + foreach (var v in values) + { + if (double.IsFinite(v)) count++; + } + return count; + } + + /// Spread between two percentiles of the finite values (0 when fewer than two). + private static double Range(double[] values, double low, double high) + { + if (Count(values) < 2) return 0; + return Statistics.Percentile(values, high) - Statistics.Percentile(values, low); + } + + /// Rounds half away from zero; adding +0 turns −0 into 0 so JSON never spends a byte on the sign. + private static double Round(double value, int decimals) => double.Round(value, decimals, MidpointRounding.AwayFromZero) + 0.0; + + private static double Significant(double value, int digits) + { + if (value == 0) return 0; + var magnitude = (int)double.Floor(double.Log10(double.Abs(value))); + var decimals = digits - 1 - magnitude; + return decimals is >= 0 and <= 15 ? Round(value, decimals) : value; + } +} diff --git a/src/Titano.Core/Analysis/Deep/FrameProbe.cs b/src/Titano.Core/Analysis/Deep/FrameProbe.cs new file mode 100644 index 00000000..b74fb9f9 --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/FrameProbe.cs @@ -0,0 +1,147 @@ +using Titano.Core.Imaging; +using Titano.Core.Motion; + +namespace Titano.Core.Analysis.Deep; + +/// +/// The measurements of one frame that need no other frame (statistics, stars, sky/land luminance, clouds) plus the +/// reduced planes the sequential stage of compares with the previous frame. Frames are +/// probed in parallel; the full analysis image is released before the probe is returned. +/// +internal sealed class FrameProbe : IDisposable +{ + private FrameProbe(int index) => Index = index; + + public int Index { get; } + + /// Null when the frame could not be decoded. + public FrameStatistics? Statistics { get; private set; } + + /// Perceptual luminance at the correlation size, row-major. + public float[]? Correlation { get; private set; } + + public int CorrelationWidth { get; private set; } + public int CorrelationHeight { get; private set; } + + /// Source pixels per correlation-plane pixel. + public double CorrelationScale { get; private set; } = 1; + + /// Flow-sized image; only for frames that take part in a sampled flow pair. + public ImageBuffer? Flow { get; private set; } + + /// Source pixels per flow-image pixel. + public double FlowScale { get; private set; } = 1; + + public double StarDensity { get; private set; } = double.NaN; + public double CloudCoverage { get; private set; } = double.NaN; + public double SkyLogLuminance { get; private set; } = double.NaN; + public double LandLogLuminance { get; private set; } = double.NaN; + + /// True when the frame is decoded and neither black nor white (it may update the sequential state). + public bool IsUsable => Statistics is { } s && !s.IsBlack() && !s.IsWhite(); + + /// Whether frame takes part in a sampled optical-flow pair (k − 1, k), k divisible by . + public static bool NeedsFlow(int index, int every) => index % every == 0 || (index + 1) % every == 0; + + public static async Task RunAsync(IFrameSource source, int index, DeepAnalysisOptions options, RegionMask? mask, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var (sourceWidth, sourceHeight) = source.Metadata(index).OrientedSize; + var edge = options.AnalysisLongEdge(int.Max(sourceWidth, sourceHeight)); + var loaded = await source.LoadAsync(index, edge, cancellationToken).ConfigureAwait(false); + var probe = new FrameProbe(index); + if (loaded is null) return probe; + cancellationToken.ThrowIfCancellationRequested(); + using var analysis = int.Max(loaded.Width, loaded.Height) > edge ? Reduce(loaded, edge) : loaded; + var sourceLong = sourceWidth > 0 && sourceHeight > 0 ? int.Max(sourceWidth, sourceHeight) : int.Max(analysis.Width, analysis.Height); + probe.Measure(analysis, sourceLong, options, mask); + return probe; + } + + /// Hands the flow image over to the caller, who then disposes it. + public ImageBuffer? TakeFlow() + { + var flow = Flow; + Flow = null; + return flow; + } + + public void Dispose() + { + Flow?.Dispose(); + Flow = null; + } + + private static ImageBuffer Reduce(ImageBuffer image, int edge) + { + using (image) return AnalysisImage.Downscale(image, edge); + } + + private void Measure(ImageBuffer analysis, int sourceLong, DeepAnalysisOptions options, RegionMask? mask) + { + var stats = FrameStatistics.Measure(analysis); + Statistics = stats; + if (!IsUsable) return; + + if (stats.MeanLogLuminance <= options.DarkLogLuminance) StarDensity = StarFieldDetector.Density(analysis, stats.Noise); + + if (mask is not null) + { + using var thumbnail = AnalysisImage.Downscale(analysis, options.ThumbnailLongEdge); + var fitted = mask.Resample(thumbnail.Width, thumbnail.Height); + (SkyLogLuminance, LandLogLuminance) = RegionDeflicker.Measure(thumbnail, fitted); + CloudCoverage = Deep.CloudCoverage.Estimate(thumbnail, fitted.Sky); + } + + using (var correlation = AnalysisImage.Downscale(analysis, options.CorrelationLongEdge)) + { + CorrelationWidth = correlation.Width; + CorrelationHeight = correlation.Height; + CorrelationScale = (double)sourceLong / int.Max(correlation.Width, correlation.Height); + var plane = new float[correlation.PlaneLength]; + PlaneOps.PerceptualLuminance(correlation, plane); + Correlation = plane; + } + + if (NeedsFlow(Index, options.FlowEvery)) + { + Flow = AnalysisImage.Downscale(analysis, options.FlowLongEdge); + FlowScale = (double)sourceLong / int.Max(Flow.Width, Flow.Height); + } + } + + /// Loads and measures the region-mask sample (thumbnails of usable frames that share the first one's size). + public static async Task SegmentRegionsAsync(IFrameSource source, DeepAnalysisOptions options, CancellationToken cancellationToken) + { + var n = source.Count; + var count = int.Min(options.RegionSampleFrames, n); + var thumbnails = new List(count); + try + { + for (var k = 0; k < count; k++) + { + var index = count > 1 ? (int)((long)k * (n - 1) / (count - 1)) : 0; + var (w, h) = source.Metadata(index).OrientedSize; + var edge = options.AnalysisLongEdge(int.Max(w, h)); + using var loaded = await source.LoadAsync(index, edge, cancellationToken).ConfigureAwait(false); + if (loaded is null) continue; + using var analysis = AnalysisImage.Downscale(loaded, edge); + var stats = FrameStatistics.Measure(analysis); + if (stats.IsBlack() || stats.IsWhite()) continue; + var thumbnail = AnalysisImage.Downscale(analysis, options.ThumbnailLongEdge); + if (thumbnails.Count > 0 && (thumbnail.Width != thumbnails[0].Width || thumbnail.Height != thumbnails[0].Height) || thumbnail.Height < 8) + { + thumbnail.Dispose(); + continue; + } + thumbnails.Add(thumbnail); + } + cancellationToken.ThrowIfCancellationRequested(); + return thumbnails.Count > 0 ? RegionSegmenter.Segment(thumbnails) : null; + } + finally + { + foreach (var t in thumbnails) t.Dispose(); + } + } +} diff --git a/src/Titano.Core/Analysis/Deep/HistogramDistance.cs b/src/Titano.Core/Analysis/Deep/HistogramDistance.cs new file mode 100644 index 00000000..257fb941 --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/HistogramDistance.cs @@ -0,0 +1,61 @@ +namespace Titano.Core.Analysis.Deep; + +/// Scene-change measure between two s. +internal static class HistogramDistance +{ + private const double SmoothingBins = 2; + private const double BinsPerEv = FrameStatistics.HistogramBins / (FrameStatistics.HistogramMaxEv - FrameStatistics.HistogramMinEv); + + /// + /// Symmetric chi-square distance (0 = same, 1 = disjoint) after both histograms are smoothed with a Gaussian of + /// σ = 2 bins (⅛ EV) and the current one is shifted by the change of mean log-luminance (linear interpolation), + /// so flicker, exposure steps and brightness ramps score near zero while a new view scores high. Mass shifted out + /// of range counts as disjoint. + /// + public static double ExposureAligned(FrameStatistics previous, FrameStatistics current) + { + var a = Smooth(previous.Histogram); + var b = Smooth(current.Histogram); + var delta = current.MeanLogLuminance - previous.MeanLogLuminance; + var shift = double.IsFinite(delta) ? delta * BinsPerEv : 0; + double sum = 0, matched = 0, total = 0; + foreach (var v in b) total += v; + for (var i = 0; i < a.Length; i++) + { + var x = i + shift; + var j = (int)double.Floor(x); + var t = x - j; + var q = (1 - t) * At(b, j) + t * At(b, j + 1); + matched += q; + var both = a[i] + q; + if (both > 0) sum += (a[i] - q) * (a[i] - q) / both; + } + sum += double.Max(0, total - matched); + return 0.5 * sum; + } + + private static double At(double[] values, int index) => index >= 0 && index < values.Length ? values[index] : 0; + + /// Gaussian smoothing that keeps the total mass (kernel renormalised at the edges). + private static double[] Smooth(double[] histogram) + { + var radius = (int)double.Ceiling(3 * SmoothingBins); + Span kernel = stackalloc double[2 * radius + 1]; + for (var d = -radius; d <= radius; d++) kernel[d + radius] = double.Exp(-0.5 * d * d / (SmoothingBins * SmoothingBins)); + var result = new double[histogram.Length]; + for (var i = 0; i < histogram.Length; i++) + { + if (histogram[i] == 0) continue; + double weight = 0; + for (var d = -radius; d <= radius; d++) + { + if (i + d >= 0 && i + d < histogram.Length) weight += kernel[d + radius]; + } + for (var d = -radius; d <= radius; d++) + { + if (i + d >= 0 && i + d < histogram.Length) result[i + d] += histogram[i] * kernel[d + radius] / weight; + } + } + return result; + } +} diff --git a/src/Titano.Core/Analysis/Deep/IFrameSource.cs b/src/Titano.Core/Analysis/Deep/IFrameSource.cs new file mode 100644 index 00000000..8a5492ba --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/IFrameSource.cs @@ -0,0 +1,25 @@ +using Titano.Core.Imaging; + +namespace Titano.Core.Analysis.Deep; + +/// +/// The frames of one sequence as reads them. Decoding is the implementer's job (archive +/// decoder on the client, proxies on the server, SyntheticSequence in tests). +/// +public interface IFrameSource +{ + int Count { get; } + + /// Metadata of frame (capture time, exposure, size, GPS); cheap, no decoding. + LightFrame Metadata(int index); + + /// + /// Decodes frame in linear light, oriented, with its long edge at most + /// (a larger result is downscaled by the caller). The caller owns and disposes the + /// buffer. Null when the frame cannot be decoded. + /// + ValueTask LoadAsync(int index, int maxLongEdge, CancellationToken cancellationToken); +} + +/// Progress of a deep analysis: frames done (including resumed ones), throughput of this run and its ETA. +public readonly record struct AnalysisProgress(int Done, int Total, double FramesPerSecond, TimeSpan? Eta); diff --git a/src/Titano.Core/Analysis/Deep/StarFieldDetector.cs b/src/Titano.Core/Analysis/Deep/StarFieldDetector.cs new file mode 100644 index 00000000..00cbe6f8 --- /dev/null +++ b/src/Titano.Core/Analysis/Deep/StarFieldDetector.cs @@ -0,0 +1,57 @@ +using Titano.Core.Imaging; + +namespace Titano.Core.Analysis.Deep; + +/// +/// Star-field detection (BRIEF §10.2): counts small bright points on sRGB-encoded luminance. A point is a strict +/// 3×3 local maximum that rises above the mean of the 5×5 ring around it by at least +/// max(·σ, ), whose 8 neighbours average less than half that +/// rise and whose ring stays below half of it everywhere (point-like: lit windows, the moon and textured land fail; +/// stars up to 2×2 pixels pass). +/// +public static class StarFieldDetector +{ + public const double NoiseFactor = 6; + public const double MinimumContrast = 0.05; + + /// Point sources per megapixel of ; is the encoded-luminance + /// noise σ (e.g. ). + public static double Density(ImageBuffer image, double noiseSigma) + { + ArgumentNullException.ThrowIfNull(image); + int w = image.Width, h = image.Height; + if (w < 5 || h < 5) return 0; + var e = new float[image.PlaneLength]; + ColorSpace.Luminance(image, e); + for (var i = 0; i < e.Length; i++) e[i] = ColorSpace.EncodeFast(e[i]); + var threshold = double.Max(NoiseFactor * (double.IsFinite(noiseSigma) ? noiseSigma : 0), MinimumContrast); + + var count = 0; + for (var y = 2; y < h - 2; y++) + { + for (var x = 2; x < w - 2; x++) + { + var i = y * w + x; + var c = e[i]; + // Ties go to the later pixel in raster order so a plateau counts once. + if (!(c >= e[i - w - 1] && c >= e[i - w] && c >= e[i - w + 1] && c >= e[i - 1] + && c > e[i + 1] && c > e[i + w - 1] && c > e[i + w] && c > e[i + w + 1])) continue; + double ring = 0, ringMax = 0; + for (var d = -2; d <= 2; d++) + { + ring += e[i - 2 * w + d] + e[i + 2 * w + d]; + ringMax = double.Max(ringMax, double.Max(e[i - 2 * w + d], e[i + 2 * w + d])); + if (d is <= -2 or >= 2) continue; + ring += e[i + d * w - 2] + e[i + d * w + 2]; + ringMax = double.Max(ringMax, double.Max(e[i + d * w - 2], e[i + d * w + 2])); + } + var background = ring / 16; + var rise = c - background; + if (rise < threshold || ringMax - background >= 0.5 * rise) continue; + double inner = e[i - w - 1] + e[i - w] + e[i - w + 1] + e[i - 1] + e[i + 1] + e[i + w - 1] + e[i + w] + e[i + w + 1]; + if (inner / 8 - background <= 0.5 * rise) count++; + } + } + return count * 1e6 / ((double)w * h); + } +} diff --git a/tests/Titano.Core.Tests/Analysis/Deep/DeepAnalyzerTests.cs b/tests/Titano.Core.Tests/Analysis/Deep/DeepAnalyzerTests.cs new file mode 100644 index 00000000..b198d41c --- /dev/null +++ b/tests/Titano.Core.Tests/Analysis/Deep/DeepAnalyzerTests.cs @@ -0,0 +1,311 @@ +using Titano.Core.Analysis; +using Titano.Core.Analysis.Deep; +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Imaging; +using Titano.Testing; + +namespace Titano.Core.Tests.Analysis.Deep; + +public sealed class DeepAnalyzerTests(TestContext context) +{ + [Test] + public async Task NightSkyShowsStarsDarknessFlickerAndNoShake() + { + var report = await DeepReports.Night; + context.Log($"{report.Sky} {report.Flicker} {report.Shake}"); + Assert.True(report.Sky.StarField); + Assert.Equal(DayNight.Night, report.Sky.Light); + Assert.Equal(report.FrameCount, report.Sky.StarFrames); + // Uniform ±0.12 EV flicker has an RMS of 0.069 EV. + Assert.Approximately(0.069, report.Flicker.AmplitudeEv, 0.01); + Assert.Equal(15, report.Flicker.RecommendedWindow); + Assert.Less(report.Shake.RmsPx, 0.05); + Assert.Equal(1.0, report.Shake.MeasuredFraction); + Assert.Empty(report.Issues.SceneChanges); + Assert.Equal(0.0, report.Motion.P95Px!.Value); + Assert.Equal(RegionMethod.Horizon, report.Regions!.Method); + Assert.Contains(report.Findings, f => f.Key == DeepFindingKeys.StarField); + Assert.Contains(report.Findings, f => f.Key == DeepFindingKeys.Flicker); + Assert.True(report.Frames.FlagsOf(0).HasFlag(DeepFrameFlags.StarField | DeepFrameFlags.Dark)); + } + + [Test] + public async Task InjectedCameraJitterIsMeasuredAsShake() + { + var report = await DeepReports.NightJitter; + context.Log($"{report.Shake} {report.Motion}"); + // Integer offsets uniform in −2..2 per axis: 2 px RMS in 2D. + Assert.InRange(report.Shake.RmsPx, 1.6, 2.4); + Assert.InRange(report.Shake.CropCost, 0.02, 0.2); + Assert.False(report.Shake.PanDetected); + Assert.Greater(report.Motion.P95Px!.Value, 1); + Assert.Contains(report.Findings, f => f.Key == DeepFindingKeys.Shake); + } + + [Test] + public async Task HolyGrailStepsAreReceivedAndSpacedByHalfTheirDistance() + { + var report = await DeepReports.HolyGrail; + Assert.Equal(5, report.Exposure.MetadataSteps); + Assert.Equal(5, report.Exposure.ReceivedSteps); + Assert.Equal(40.0, report.Exposure.TypicalSpacing); + Assert.Equal(20, report.Exposure.TransitionLength); + Assert.True(report.Exposure.Steps.All(s => s.MeasuredEv is > 0.9 and < 1.1)); + Assert.InRange(report.Exposure.SceneRangeEv, 5, 6); + Assert.Equal(DayNight.DayToNight, report.Sky.Light); + // Brightness ramps and steps are levelled before the flow, so the static scene shows no motion. + Assert.Equal(0.0, report.Motion.P95Px!.Value); + Assert.Empty(report.Issues.SceneChanges); + } + + [Theory] + [Case(8.0, 33)] + [Case(12.5, 51)] + [Case(40.0, 151)] + public async Task PeriodicFlickerGivesAWindowOfFourPeriods(double period, int window) + { + var sequence = SyntheticSequence.Create(new SyntheticSequenceOptions + { + Width = 96, Height = 64, FrameCount = 240, FlickerEv = 0.15, FlickerPeriodFrames = period, + }); + var report = await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence), DeepScenes.Options); + Assert.Approximately(period, report.Flicker.PeriodFrames!.Value, 0.05 * period); + Assert.Equal(window, report.Flicker.RecommendedWindow); + Assert.Equal(1, window % 2); + } + + [Test] + public async Task BlackWhiteDuplicateAndUnreadableFramesAreFlagged() + { + var sequence = SyntheticSequence.Create(new SyntheticSequenceOptions + { + Width = 96, Height = 64, FrameCount = 60, FlickerEv = 0.02, BlackFrames = [10], WhiteFrames = [20], DuplicateFrames = [30], + }); + var report = await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence) { Undecodable = [40] }, DeepScenes.Options); + Assert.SequenceEqual([10], report.Issues.Black); + Assert.SequenceEqual([20], report.Issues.White); + Assert.SequenceEqual([30], report.Issues.Duplicate); + Assert.SequenceEqual([40], report.Issues.Undecodable); + Assert.SequenceEqual([10, 20, 30, 40], report.Issues.Excludable()); + Assert.Empty(report.Issues.SceneChanges); + Assert.Null(report.Frames.LogLuminance[40]); + Assert.True(report.Frames.FlagsOf(40).HasFlag(DeepFrameFlags.Undecodable)); + Assert.False(report.Issues.Excludable().Contains(report.HeroIndex!.Value)); + foreach (var key in new[] { DeepFindingKeys.BlackFrames, DeepFindingKeys.WhiteFrames, DeepFindingKeys.DuplicateFrames, DeepFindingKeys.UnreadableFrames }) + Assert.Equal(FindingSeverity.Warning, report.Findings.Single(f => f.Key == key).Severity); + Assert.Equal(4, AnalysisSummary.From(report).WarningCount); + } + + [Test] + public async Task ACutToAnotherViewIsASceneChange() + { + var first = new SyntheticFrameSource(SyntheticSequence.Create(new SyntheticSequenceOptions { Width = 96, Height = 64, FrameCount = 50, FlickerEv = 0.05 })); + var second = new SyntheticFrameSource(SyntheticSequence.Create(new SyntheticSequenceOptions + { + Width = 96, Height = 64, FrameCount = 50, FlickerEv = 0.05, Seed = 5, HorizonFraction = 0.75, SkyLevel = 0.6, LandLevel = 0.01, + })); + var report = await DeepAnalyzer.AnalyzeAsync(new SplicedSource(first, second, 30), DeepScenes.Options); + Assert.SequenceEqual([30], report.Issues.SceneChanges); + Assert.True(report.Frames.FlagsOf(30).HasFlag(DeepFrameFlags.SceneChange)); + } + + [Test] + public async Task ResumingFromACheckpointGivesTheSameReport() + { + var expected = (await DeepReports.HolyGrail).ToJson(); + var checkpoint = new MemoryCheckpoint(); + var crashing = new SyntheticFrameSource(DeepScenes.HolyGrail()) + { + OnLoad = i => + { + if (i == 101) throw new IOException("simulated crash"); + }, + }; + await Assert.ThrowsAsync(() => DeepAnalyzer.AnalyzeAsync(crashing, DeepScenes.Options, checkpoint)); + var persisted = checkpoint.Frames.Count; + Assert.InRange(persisted, 90, 101); + + var resumed = new SyntheticFrameSource(DeepScenes.HolyGrail()); + var report = await DeepAnalyzer.AnalyzeAsync(resumed, DeepScenes.Options, checkpoint); + Assert.Equal(expected, report.ToJson()); + // Region sample + one priming frame + the frames not yet persisted. + Assert.Equal(DeepScenes.Options.RegionSampleFrames + 1 + (240 - persisted), resumed.Loads); + Assert.Equal(240, checkpoint.Frames.Count); + } + + [Test] + public async Task FileCheckpointDropsATornLineAndResumes() + { + var path = Path.Combine(context.CreateTempDirectory(), ".titano", DeepAnalysisCheckpointFile.FileName); + var sequence = SyntheticSequence.Create(new SyntheticSequenceOptions { Width = 96, Height = 64, FrameCount = 40, FlickerEv = 0.05, DuplicateFrames = [7] }); + var expected = (await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence), DeepScenes.Options)).ToJson(); + var file = new DeepAnalysisCheckpointFile(path); + var crashing = new SyntheticFrameSource(sequence) { OnLoad = i => { if (i == 21) throw new IOException("crash"); } }; + await Assert.ThrowsAsync(() => DeepAnalyzer.AnalyzeAsync(crashing, DeepScenes.Options, file)); + await File.AppendAllTextAsync(path, "{\"index\":20,\"flags\":\"No", context.Cancellation); + + var report = await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence), DeepScenes.Options, file); + Assert.Equal(expected, report.ToJson()); + Assert.Equal(41, File.ReadAllLines(path).Length); + + // Other options: the stale checkpoint is discarded instead of mixed in. + var other = DeepScenes.Options with { FlowEvery = 3 }; + var fresh = await new DeepAnalysisCheckpointFile(path).LoadAsync(DeepAnalyzer.Fingerprint(40, other), context.Cancellation); + Assert.Empty(fresh); + file.Delete(); + Assert.FileMissing(path); + } + + [Test] + public async Task ReportRoundTripsThroughJsonWithinTheSizeBudget() + { + var report = await DeepReports.NightJitter; + var json = report.ToJson(); + var copy = DeepAnalysisReport.FromJson(json); + Assert.Equal(json, copy.ToJson()); + var perFrame = (double)System.Text.Encoding.UTF8.GetByteCount(json) / report.FrameCount; + context.Log($"{perFrame:0.0} bytes per frame"); + Assert.LessOrEqual(perFrame, 120); + + var path = Path.Combine(context.CreateTempDirectory(), DeepAnalysisReport.FileName); + report.Save(path); + Assert.Equal(json, DeepAnalysisReport.Load(path).ToJson()); + + var summary = AnalysisSummary.From(report); + Assert.Equal(summary, AnalysisSummary.FromJson(summary.ToJson())); + Assert.True(summary.StarField); + Assert.Equal(DayNight.Night, summary.Light); + Assert.Equal(report.HeroIndex, summary.HeroIndex); + Assert.Less(summary.ToJson().Length, 400); + Assert.Throws(() => DeepAnalysisReport.FromJson(json.Replace("\"version\":1", "\"version\":99", StringComparison.Ordinal))); + } + + [Test] + public async Task ResultsDoNotDependOnParallelism() + { + var sequence = DeepScenes.NightSky(frames: 40, jitter: 2); + var serial = await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence, 2), DeepScenes.Options with { MaxDegreeOfParallelism = 1 }); + var parallel = await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence, 2), DeepScenes.Options with { MaxDegreeOfParallelism = 3 }); + Assert.Equal(serial.ToJson(), parallel.ToJson()); + } + + [Test] + public async Task CancellationStopsWithinOneBatch() + { + using var cancellation = new CancellationTokenSource(); + var checkpoint = new MemoryCheckpoint(); + var source = new SyntheticFrameSource(DeepScenes.HolyGrail()) + { + OnLoad = i => + { + if (i == 50) cancellation.Cancel(); + }, + }; + await Assert.ThrowsAsync(() => DeepAnalyzer.AnalyzeAsync(source, DeepScenes.Options, checkpoint, null, cancellation.Token)); + Assert.LessOrEqual(source.Loads, DeepScenes.Options.RegionSampleFrames + 52); + Assert.Equal(50, checkpoint.Frames.Count); + } + + [Test] + public async Task ProgressCountsEveryFrameUpToTheTotal() + { + var sequence = SyntheticSequence.Create(new SyntheticSequenceOptions { Width = 64, Height = 48, FrameCount = 21 }); + var reports = new List(); + await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence), DeepScenes.Options, progress: new SyncProgress(reports.Add)); + Assert.Equal(11, reports.Count); + Assert.Monotonic(reports.Select(r => (double)r.Done), strict: true); + Assert.Equal(21, reports[^1].Done); + Assert.True(reports.All(r => r.Total == 21 && r.FramesPerSecond > 0)); + Assert.Equal(TimeSpan.Zero, reports[^1].Eta); + } + + [Test] + public void StarDetectorCountsPointsButNotBlobs() + { + using var image = new ImageBuffer(128, 96, 3); + image.Data.Fill(1f / 2048); + var random = new Random(3); + var stars = 0; + for (var y = 6; y < 90; y += 12) + { + for (var x = 6; x < 122; x += 12) + { + var v = (float)(0.2 + 0.8 * random.NextDouble()); + for (var c = 0; c < 3; c++) image[c, x, y] = v; + stars++; + } + } + // A lit window: 4×4 bright block (extended, not a star). + for (var y = 40; y < 44; y++) + { + for (var x = 60; x < 64; x++) + { + for (var c = 0; c < 3; c++) image[c, x, y] = 0.8f; + } + } + var density = StarFieldDetector.Density(image, noiseSigma: 0); + Assert.Approximately(stars * 1e6 / (128 * 96), density, 1e-6); + } + + [Test] + public void CloudCoverageCountsTexturedSkyBlocks() + { + using var image = new ImageBuffer(128, 64, 3); + var sky = new float[image.PlaneLength]; + var random = new Random(9); + for (var y = 0; y < 64; y++) + { + for (var x = 0; x < 128; x++) + { + var i = y * 128 + x; + sky[i] = y < 32 ? 1 : 0; + // Smooth gradient everywhere; cloud texture on the left half of the sky. + var v = 0.2 + 0.002 * y + (y < 32 && x < 64 ? 0.15 * random.NextDouble() : 0); + for (var c = 0; c < 3; c++) image[c, x, y] = (float)v; + } + } + Assert.Approximately(0.5, CloudCoverage.Estimate(image, sky), 1e-9); + Assert.True(double.IsNaN(CloudCoverage.Estimate(image, new float[image.PlaneLength]))); + } + + [Test] + public void HistogramDistanceIgnoresExposureButNotContent() + { + using var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { Width = 96, Height = 64 }).Render(0); + using var brighter = scene.Clone(); + foreach (ref var v in brighter.Data) v *= 1.7f; + using var other = SyntheticSequence.Create(new SyntheticSequenceOptions { Width = 96, Height = 64, HorizonFraction = 0.8, SkyLevel = 0.6 }).Render(0); + var a = FrameStatistics.Measure(scene); + Assert.Less(HistogramDistance.ExposureAligned(a, FrameStatistics.Measure(brighter)), 0.02); + Assert.Greater(HistogramDistance.ExposureAligned(a, FrameStatistics.Measure(other)), 0.3); + } + + /// Frames of before , then frames of . + private sealed class SplicedSource(IFrameSource first, IFrameSource second, int cut) : IFrameSource + { + public int Count => first.Count; + + public LightFrame Metadata(int index) => first.Metadata(index); + + public ValueTask LoadAsync(int index, int maxLongEdge, CancellationToken cancellationToken) => + (index < cut ? first : second).LoadAsync(index, maxLongEdge, cancellationToken); + } + + /// Reports synchronously (Progress<T> posts to the thread pool and may reorder). + private sealed class SyncProgress(Action report) : IProgress + { + public void Report(AnalysisProgress value) => report(value); + } +} + +/// Deep reports of the shared scenes, computed once per test run. +internal static class DeepReports +{ + private static readonly Lazy> NightReport = new(() => DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(DeepScenes.NightSky()), DeepScenes.Options)); + private static readonly Lazy> NightJitterReport = new(() => DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(DeepScenes.NightSky(jitter: 2), 2), DeepScenes.Options)); + private static readonly Lazy> HolyGrailReport = new(() => DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(DeepScenes.HolyGrail()), DeepScenes.Options)); + + public static Task Night => NightReport.Value; + public static Task NightJitter => NightJitterReport.Value; + public static Task HolyGrail => HolyGrailReport.Value; +} diff --git a/tests/Titano.Core.Tests/Analysis/Deep/DeepFixtures.cs b/tests/Titano.Core.Tests/Analysis/Deep/DeepFixtures.cs new file mode 100644 index 00000000..c7890215 --- /dev/null +++ b/tests/Titano.Core.Tests/Analysis/Deep/DeepFixtures.cs @@ -0,0 +1,125 @@ +using Titano.Core.Analysis; +using Titano.Core.Analysis.Deep; +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Imaging; + +namespace Titano.Core.Tests.Analysis.Deep; + +/// +/// over a . With > 0 the sequence is +/// rendered pixels larger on every side and each frame is cropped at a seeded integer offset +/// in ±jitter, which shakes the camera by a known amount. +/// +internal sealed class SyntheticFrameSource(SyntheticSequence sequence, int jitter = 0, int seed = 11) : IFrameSource +{ + private readonly (int X, int Y)[] _offsets = Offsets(sequence.Count, jitter, seed); + private int _loads; + + public SyntheticSequence Sequence { get; } = sequence; + + public int Width => Sequence.Options.Width - 2 * jitter; + public int Height => Sequence.Options.Height - 2 * jitter; + public int Count => Sequence.Count; + + /// Total calls of . + public int Loads => Volatile.Read(ref _loads); + + /// Frames that "fail to decode". + public HashSet Undecodable { get; init; } = []; + + /// Called on every load before decoding (may throw to simulate a crash, or cancel a token). + public Action? OnLoad { get; set; } + + /// Metadata without shutter time and aperture when true. + public bool StripExposure { get; init; } + + public LightFrame Metadata(int index) + { + var frame = LightFrame.From(Sequence.Metadata[index], index) with { Width = Width, Height = Height }; + return StripExposure ? frame with { ExposureTime = null, FNumber = null } : frame; + } + + public ValueTask LoadAsync(int index, int maxLongEdge, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _loads); + OnLoad?.Invoke(index); + if (Undecodable.Contains(index)) return ValueTask.FromResult(null); + using var full = Sequence.Render(index); + var (dx, dy) = _offsets[index]; + var image = new ImageBuffer(Width, Height, 3, clear: false); + for (var c = 0; c < 3; c++) + { + for (var y = 0; y < Height; y++) + full.Row(c, y + jitter + dy).Slice(jitter + dx, Width).CopyTo(image.Row(c, y)); + } + if (int.Max(Width, Height) <= maxLongEdge) return ValueTask.FromResult(image); + using (image) return ValueTask.FromResult(AnalysisImage.Downscale(image, maxLongEdge)); + } + + private static (int, int)[] Offsets(int count, int jitter, int seed) + { + var random = new Random(seed); + var offsets = new (int, int)[count]; + for (var i = 0; i < count; i++) offsets[i] = jitter > 0 ? (random.Next(-jitter, jitter + 1), random.Next(-jitter, jitter + 1)) : (0, 0); + return offsets; + } +} + +/// In-memory checkpoint that records what was persisted. +internal sealed class MemoryCheckpoint : IDeepAnalysisCheckpoint +{ + private string? _fingerprint; + + public List Frames { get; } = []; + + public ValueTask> LoadAsync(string fingerprint, CancellationToken cancellationToken) + { + if (_fingerprint != fingerprint) Frames.Clear(); + _fingerprint = fingerprint; + return ValueTask.FromResult>([.. Frames]); + } + + public ValueTask AppendAsync(IReadOnlyList frames, CancellationToken cancellationToken) + { + Frames.AddRange(frames); + return ValueTask.CompletedTask; + } +} + +/// The synthetic scenes the deep-analysis and advisor tests share (small frames, fast). +internal static class DeepScenes +{ + /// Options for the tests: small sources are analysed at native size, flow on every 5th pair, two workers. + public static DeepAnalysisOptions Options { get; } = new() { MinAnalysisLongEdge = 64, FlowEvery = 5, MaxDegreeOfParallelism = 2, RegionSampleFrames = 9 }; + + /// Night sky with stars over dark land and random flicker. + public static SyntheticSequence NightSky(int frames = 90, int jitter = 0) => SyntheticSequence.Create(new SyntheticSequenceOptions + { + Width = 160 + 2 * jitter, + Height = 112 + 2 * jitter, + FrameCount = frames, + IntervalSeconds = 20, + BaseExposureTime = 15, + BaseFNumber = 2.8, + BaseIso = 3200, + StarField = true, + StarCount = 160, + FlickerEv = 0.12, + Start = new DateTime(2024, 8, 12, 23, 0, 0), + }); + + /// Sunset: the scene darkens 6 EV while the camera opens up 1 EV every 40 frames. + public static SyntheticSequence HolyGrail(int frames = 240) => SyntheticSequence.Create(new SyntheticSequenceOptions + { + Width = 160, + Height = 112, + FrameCount = frames, + IntervalSeconds = 15, + BaseExposureTime = 1.0 / 250, + StartEv = 0.5, + EndEv = -5.5, + FlickerEv = 0.03, + ExposureSteps = [new(40, -1), new(80, -1), new(120, -1), new(160, -1), new(200, -1)], + }); +}