feat(analysis): add streaming deep analysis with resumable checkpoints
Per-frame statistics, nine-patch shift, Draft flow on sampled pairs, stars, clouds, sky/land luminance and exposure-aligned scene distance, then sequence-level flicker, received exposure steps, white balance, shake, day/night, hero frame and findings in a versioned analysis.json. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
14 files changed
+1980
No files matched your search
@@ -0,0 +1,84 @@
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Mathematics;
|
||||
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>
|
||||
/// Cloud coverage estimate (BRIEF §10.2) from sky texture: the sky (weight ≥ 0.9 of a <see cref="RegionMask"/>) is
|
||||
/// cut into square blocks and a block is cloudy when the 30th percentile of |Laplacian| of its sRGB-encoded luminance
|
||||
/// exceeds max(<see cref="NoiseFactor"/>·σ, <see cref="MinimumTexture"/>). σ is the 10th percentile of the blocks'
|
||||
/// Immerkær (1996) noise estimates, so the clouds' own texture does not raise the noise floor. Cloud texture lifts nearly every pixel of a block while stars touch a few, so the low percentile ignores
|
||||
/// star fields; the Laplacian ignores smooth sky gradients. A uniform overcast reads as clear sky.
|
||||
/// </summary>
|
||||
public static class CloudCoverage
|
||||
{
|
||||
/// <summary>The 30th percentile of |Laplacian| of pure Gaussian noise is ≈ 1.7σ, so this is 3.5 × the noise level.</summary>
|
||||
public const double NoiseFactor = 6;
|
||||
|
||||
public const double MinimumTexture = 0.004;
|
||||
|
||||
/// <summary>Fraction of sky blocks that are cloudy; NaN when no block is sky.</summary>
|
||||
public static double Estimate(ImageBuffer image, ReadOnlySpan<float> skyWeights, int block = 8)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(image);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(block, 3);
|
||||
if (skyWeights.Length != image.PlaneLength) throw new ArgumentException("One sky weight per pixel is required.", nameof(skyWeights));
|
||||
int w = image.Width, h = image.Height;
|
||||
if (w < 3 || h < 3) return double.NaN;
|
||||
var e = new float[image.PlaneLength];
|
||||
ColorSpace.Luminance(image, e);
|
||||
for (var i = 0; i < e.Length; i++) e[i] = ColorSpace.EncodeFast(e[i]);
|
||||
var threshold = double.Max(NoiseFactor * Noise(e, w, h, block), MinimumTexture);
|
||||
|
||||
var values = new double[block * block];
|
||||
int sky = 0, cloudy = 0;
|
||||
for (var by = 0; by + block <= h; by += block)
|
||||
{
|
||||
for (var bx = 0; bx + block <= w; bx += block)
|
||||
{
|
||||
double weight = 0;
|
||||
var count = 0;
|
||||
for (var y = by; y < by + block; y++)
|
||||
{
|
||||
for (var x = bx; x < bx + block; x++)
|
||||
{
|
||||
var i = y * w + x;
|
||||
weight += skyWeights[i];
|
||||
if (x == 0 || y == 0 || x == w - 1 || y == h - 1) continue;
|
||||
values[count++] = double.Abs(e[i - 1] + e[i + 1] + e[i - w] + e[i + w] - 4.0 * e[i]);
|
||||
}
|
||||
}
|
||||
if (weight < 0.9 * block * block || count == 0) continue;
|
||||
sky++;
|
||||
if (Statistics.Select(values.AsSpan(0, count), (int)(0.3 * (count - 1))) > threshold) cloudy++;
|
||||
}
|
||||
}
|
||||
return sky > 0 ? (double)cloudy / sky : double.NaN;
|
||||
}
|
||||
|
||||
/// <summary>10th percentile over whole blocks of the Immerkær fast noise σ (interior pixels only).</summary>
|
||||
private static double Noise(float[] e, int w, int h, int block)
|
||||
{
|
||||
var estimates = new List<double>();
|
||||
for (var by = 0; by + block <= h; by += block)
|
||||
{
|
||||
for (var bx = 0; bx + block <= w; bx += block)
|
||||
{
|
||||
double sum = 0;
|
||||
var count = 0;
|
||||
for (var y = int.Max(1, by); y < int.Min(h - 1, by + block); y++)
|
||||
{
|
||||
for (var x = int.Max(1, bx); x < int.Min(w - 1, bx + block); x++)
|
||||
{
|
||||
var i = y * w + x;
|
||||
sum += double.Abs(e[i - w - 1] + e[i - w + 1] + e[i + w - 1] + e[i + w + 1]
|
||||
- 2.0 * (e[i - w] + e[i + w] + e[i - 1] + e[i + 1]) + 4.0 * e[i]);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
if (count > 0) estimates.Add(double.Sqrt(double.Pi / 2) / 6 * sum / count);
|
||||
}
|
||||
}
|
||||
return estimates.Count > 0 ? Statistics.Percentile(estimates.ToArray(), 0.1) : 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>
|
||||
/// Where <see cref="DeepAnalyzer"/> persists per-frame results while it runs, so a crashed or cancelled analysis
|
||||
/// resumes after the last persisted frame.
|
||||
/// </summary>
|
||||
public interface IDeepAnalysisCheckpoint
|
||||
{
|
||||
/// <summary>
|
||||
/// Frames persisted by an earlier run with the same <paramref name="fingerprint"/>, in index order from 0. A store
|
||||
/// holding another fingerprint discards its content and starts over.
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<DeepFrame>> LoadAsync(string fingerprint, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>Persists the next frames (consecutive indices) durably before returning.</summary>
|
||||
ValueTask AppendAsync(IReadOnlyList<DeepFrame> frames, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checkpoint as a JSON-lines file (e.g. <c>.titano/analysis.partial</c>): a header line with the fingerprint, then
|
||||
/// one <see cref="DeepFrame"/> per line, flushed to disk after every batch. A torn last line from a crash is dropped
|
||||
/// on load. Delete the file once the final report is saved.
|
||||
/// </summary>
|
||||
public sealed class DeepAnalysisCheckpointFile(string path) : IDeepAnalysisCheckpoint
|
||||
{
|
||||
/// <summary>Conventional file name next to <see cref="DeepAnalysisReport.FileName"/>.</summary>
|
||||
public const string FileName = "analysis.partial";
|
||||
|
||||
private static readonly UTF8Encoding Utf8 = new(false);
|
||||
|
||||
public string Path { get; } = path;
|
||||
|
||||
public async ValueTask<IReadOnlyList<DeepFrame>> LoadAsync(string fingerprint, CancellationToken cancellationToken)
|
||||
{
|
||||
var frames = new List<DeepFrame>();
|
||||
if (File.Exists(Path))
|
||||
{
|
||||
var bytes = await File.ReadAllBytesAsync(Path, cancellationToken).ConfigureAwait(false);
|
||||
var valid = Parse(bytes, fingerprint, frames);
|
||||
if (valid > 0)
|
||||
{
|
||||
// Cut a torn or foreign tail so later appends continue a well-formed file.
|
||||
await using var stream = new FileStream(Path, FileMode.Open, FileAccess.Write, FileShare.None);
|
||||
stream.SetLength(valid);
|
||||
return frames;
|
||||
}
|
||||
}
|
||||
var header = JsonSerializer.Serialize(new CheckpointHeader(fingerprint), DeepCheckpointJson.Default.CheckpointHeader) + "\n";
|
||||
Directory.CreateDirectory(System.IO.Path.GetDirectoryName(System.IO.Path.GetFullPath(Path))!);
|
||||
await File.WriteAllBytesAsync(Path, Utf8.GetBytes(header), cancellationToken).ConfigureAwait(false);
|
||||
return frames;
|
||||
}
|
||||
|
||||
public async ValueTask AppendAsync(IReadOnlyList<DeepFrame> frames, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(frames);
|
||||
var text = new StringBuilder();
|
||||
foreach (var frame in frames) text.Append(JsonSerializer.Serialize(frame, DeepCheckpointJson.Default.DeepFrame)).Append('\n');
|
||||
await using var stream = new FileStream(Path, FileMode.Append, FileAccess.Write, FileShare.None);
|
||||
await stream.WriteAsync(Utf8.GetBytes(text.ToString()), cancellationToken).ConfigureAwait(false);
|
||||
stream.Flush(flushToDisk: true);
|
||||
}
|
||||
|
||||
public void Delete()
|
||||
{
|
||||
if (File.Exists(Path)) File.Delete(Path);
|
||||
}
|
||||
|
||||
/// <summary>Reads the header and the contiguous frames after it; returns the byte length of the valid prefix
|
||||
/// (0 when the header is missing or belongs to another fingerprint).</summary>
|
||||
private static int Parse(byte[] bytes, string fingerprint, List<DeepFrame> frames)
|
||||
{
|
||||
var offset = 0;
|
||||
var valid = 0;
|
||||
var line = 0;
|
||||
while (offset < bytes.Length)
|
||||
{
|
||||
var end = Array.IndexOf(bytes, (byte)'\n', offset);
|
||||
if (end < 0) break;
|
||||
var span = bytes.AsSpan(offset, end - offset);
|
||||
try
|
||||
{
|
||||
if (line == 0)
|
||||
{
|
||||
if (JsonSerializer.Deserialize(span, DeepCheckpointJson.Default.CheckpointHeader)?.Fingerprint != fingerprint) return 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
var frame = JsonSerializer.Deserialize(span, DeepCheckpointJson.Default.DeepFrame);
|
||||
if (frame is null || frame.Index != frames.Count) break;
|
||||
frames.Add(frame);
|
||||
}
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
if (line == 0) return 0;
|
||||
break;
|
||||
}
|
||||
line++;
|
||||
offset = end + 1;
|
||||
valid = offset;
|
||||
}
|
||||
return valid;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>Source-generated JSON for <c>analysis.json</c> and the database summary: compact, camelCase, enums as names.</summary>
|
||||
[JsonSourceGenerationOptions(
|
||||
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
UseStringEnumConverter = true,
|
||||
PropertyNameCaseInsensitive = true)]
|
||||
[JsonSerializable(typeof(DeepAnalysisReport))]
|
||||
[JsonSerializable(typeof(AnalysisSummary))]
|
||||
internal sealed partial class DeepAnalysisJson : JsonSerializerContext;
|
||||
|
||||
/// <summary>Checkpoint lines: exact doubles, NaN written as a literal so unmeasured values survive a resume.</summary>
|
||||
[JsonSourceGenerationOptions(
|
||||
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
|
||||
NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals,
|
||||
UseStringEnumConverter = true)]
|
||||
[JsonSerializable(typeof(DeepFrame))]
|
||||
[JsonSerializable(typeof(CheckpointHeader))]
|
||||
internal sealed partial class DeepCheckpointJson : JsonSerializerContext;
|
||||
|
||||
/// <summary>First line of a checkpoint file.</summary>
|
||||
internal sealed record CheckpointHeader(string Fingerprint);
|
||||
@@ -0,0 +1,69 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>Parameters of <see cref="DeepAnalyzer"/>; every value enters the checkpoint fingerprint.</summary>
|
||||
public sealed record DeepAnalysisOptions
|
||||
{
|
||||
public static DeepAnalysisOptions Default { get; } = new();
|
||||
|
||||
/// <summary>Upper bound of the analysis long edge (BRIEF §10.2: half native, capped at 2048).</summary>
|
||||
public int MaxAnalysisLongEdge { get; init; } = 2048;
|
||||
|
||||
/// <summary>Sources smaller than twice this are analysed at this size (or native when smaller).</summary>
|
||||
public int MinAnalysisLongEdge { get; init; } = 512;
|
||||
|
||||
/// <summary>Long edge of the plane the per-frame shift is measured on (nine phase-correlation patches).</summary>
|
||||
public int CorrelationLongEdge { get; init; } = 1024;
|
||||
|
||||
/// <summary>Long edge of the optical-flow images (Draft quality).</summary>
|
||||
public int FlowLongEdge { get; init; } = 512;
|
||||
|
||||
/// <summary>Optical flow runs on the pair (k − 1, k) for every k divisible by this.</summary>
|
||||
public int FlowEvery { get; init; } = 10;
|
||||
|
||||
/// <summary>Long edge of the thumbnails used for regions and cloud coverage.</summary>
|
||||
public int ThumbnailLongEdge { get; init; } = 256;
|
||||
|
||||
/// <summary>Frames sampled evenly for the sky/land mask.</summary>
|
||||
public int RegionSampleFrames { get; init; } = 15;
|
||||
|
||||
/// <summary>Frames whose trimmed mean log2 luminance is at or below this are dark (night) frames.</summary>
|
||||
public double DarkLogLuminance { get; init; } = -4;
|
||||
|
||||
/// <summary>Point sources per megapixel from which a dark frame shows a star field.</summary>
|
||||
public double StarFieldDensity { get; init; } = 30;
|
||||
|
||||
/// <summary>Smallest exposure-aligned histogram distance that can be a scene change.</summary>
|
||||
public double SceneChangeMinimum { get; init; } = 0.25;
|
||||
|
||||
/// <summary>σ (frames) of the camera-path smoothing used for the shake measure (as <c>StabilizerOptions</c>).</summary>
|
||||
public double ShakeSmoothingSigma { get; init; } = 12;
|
||||
|
||||
/// <summary>Frames measured concurrently; also the batch size between checkpoints.</summary>
|
||||
public int MaxDegreeOfParallelism { get; init; } = int.Clamp(Environment.ProcessorCount, 1, 4);
|
||||
|
||||
/// <summary>Analysis long edge for a source whose oriented long edge is <paramref name="sourceLongEdge"/> (0 = unknown).</summary>
|
||||
public int AnalysisLongEdge(int sourceLongEdge)
|
||||
{
|
||||
if (sourceLongEdge <= 0) return MaxAnalysisLongEdge;
|
||||
var half = (sourceLongEdge + 1) / 2;
|
||||
return int.Min(sourceLongEdge, int.Min(MaxAnalysisLongEdge, int.Max(MinAnalysisLongEdge, half)));
|
||||
}
|
||||
|
||||
/// <summary>Stable text of every value that changes per-frame results (parallelism excluded).</summary>
|
||||
public string Fingerprint() => string.Create(CultureInfo.InvariantCulture,
|
||||
$"edge={MinAnalysisLongEdge}-{MaxAnalysisLongEdge};corr={CorrelationLongEdge};flow={FlowLongEdge}/{FlowEvery};thumb={ThumbnailLongEdge};regions={RegionSampleFrames};dark={DarkLogLuminance}");
|
||||
|
||||
internal void Validate()
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(MinAnalysisLongEdge, 16);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(MaxAnalysisLongEdge, MinAnalysisLongEdge);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(CorrelationLongEdge, 16);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(FlowLongEdge, 16);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(FlowEvery, 1);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(ThumbnailLongEdge, 16);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(RegionSampleFrames, 1);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(MaxDegreeOfParallelism, 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.IO;
|
||||
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>Light condition of a whole sequence.</summary>
|
||||
public enum DayNight
|
||||
{
|
||||
Unknown,
|
||||
Day,
|
||||
Night,
|
||||
DayToNight,
|
||||
NightToDay,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-frame series of the report as compact parallel arrays (one entry per frame, null = not measured), rounded to
|
||||
/// what charts and render rules need (~100 bytes per frame in JSON). Shifts and motion are in source pixels.
|
||||
/// </summary>
|
||||
public sealed record DeepFrameColumns
|
||||
{
|
||||
/// <summary>Trimmed mean log2 luminance, 0.001 EV.</summary>
|
||||
public double?[] LogLuminance { get; init; } = [];
|
||||
|
||||
public double?[] ClipLow { get; init; } = [];
|
||||
public double?[] ClipHigh { get; init; } = [];
|
||||
|
||||
/// <summary>log2(R/G) and log2(B/G) of the mean colour, 0.001.</summary>
|
||||
public double?[] RedRatio { get; init; } = [];
|
||||
|
||||
public double?[] BlueRatio { get; init; } = [];
|
||||
|
||||
/// <summary>Correlated colour temperature in kelvin, null when not near-Planckian.</summary>
|
||||
public double?[] Kelvin { get; init; } = [];
|
||||
|
||||
/// <summary>Laplacian variance and noise σ of sRGB-encoded luminance, two significant digits.</summary>
|
||||
public double?[] Sharpness { get; init; } = [];
|
||||
|
||||
public double?[] Noise { get; init; } = [];
|
||||
|
||||
/// <summary>Content motion from the previous usable frame, 0.01 px.</summary>
|
||||
public double?[] ShiftX { get; init; } = [];
|
||||
|
||||
public double?[] ShiftY { get; init; } = [];
|
||||
|
||||
/// <summary>Stars per megapixel of the analysis image (dark frames only).</summary>
|
||||
public double?[] StarDensity { get; init; } = [];
|
||||
|
||||
public double?[] CloudCoverage { get; init; } = [];
|
||||
public double?[] SkyLogLuminance { get; init; } = [];
|
||||
public double?[] LandLogLuminance { get; init; } = [];
|
||||
public double?[] SceneDistance { get; init; } = [];
|
||||
|
||||
/// <summary><see cref="DeepFrameFlags"/> per frame.</summary>
|
||||
public int[] Flags { get; init; } = [];
|
||||
|
||||
/// <summary>Frames k whose optical flow from k − 1 was sampled, with median and 95th-percentile magnitude (px per frame).</summary>
|
||||
public int[] MotionFrames { get; init; } = [];
|
||||
|
||||
public double[] MotionMedian { get; init; } = [];
|
||||
public double[] MotionP95 { get; init; } = [];
|
||||
|
||||
public DeepFrameFlags FlagsOf(int index) => (DeepFrameFlags)Flags[index];
|
||||
}
|
||||
|
||||
/// <param name="AmplitudeEv">RMS deviation of the step-free luminance from its smooth curve.</param>
|
||||
/// <param name="FrameToFrameEv">RMS frame-to-frame flicker relative to the curve (<see cref="DeflickerDiagnostics.FlickerBefore"/>).</param>
|
||||
/// <param name="PeriodFrames">Period of periodic flicker (<see cref="FlickerSpectrumResult.DominantPeriod"/>).</param>
|
||||
/// <param name="RecommendedWindow">Deflicker window: 4 × period or correlation length, odd, bounded.</param>
|
||||
public sealed record FlickerSummary(double AmplitudeEv, double FrameToFrameEv, double? PeriodFrames, double PeakCorrelation, double CorrelationLength, int RecommendedWindow);
|
||||
|
||||
/// <param name="Horizon">Horizon row / mask height at up to 64 evenly spaced columns (horizon method only).</param>
|
||||
/// <param name="DivergenceEv">RMS of the difference between the sky and land residuals: how independently they flicker.</param>
|
||||
public sealed record RegionSummary(RegionMethod Method, double Confidence, double SkyFraction, int MaskWidth, int MaskHeight, double[]? Horizon, double DivergenceEv);
|
||||
|
||||
/// <summary>One metadata exposure step as the pixels received it (<see cref="ExposureTransition"/>).</summary>
|
||||
public sealed record ExposureStepSummary(int Index, double ExpectedEv, double? MeasuredEv, bool Received, double Length);
|
||||
|
||||
/// <param name="TypicalSpacing">Median frames between received steps (two or more received).</param>
|
||||
/// <param name="TransitionLength">Half the typical spacing, 12 for a single step, null without received steps.</param>
|
||||
/// <param name="SceneRangeEv">5–95 % range of the step-free luminance: how much the light itself changed.</param>
|
||||
public sealed record ExposureSummary(int MetadataSteps, int ReceivedSteps, double? TypicalSpacing, int? TransitionLength, double SceneRangeEv, IReadOnlyList<ExposureStepSummary> Steps);
|
||||
|
||||
/// <param name="JitterLog2">RMS frame-to-frame colour-ratio deviation from the smoothed ratios (log2 units).</param>
|
||||
/// <param name="DriftLog2">Largest range of the smoothed colour ratios (slow, deliberate change).</param>
|
||||
public sealed record WhiteBalanceSummary(double JitterLog2, double DriftLog2, double? KelvinLow, double? KelvinHigh);
|
||||
|
||||
/// <param name="RmsPx">RMS distance of the camera path from its smoothed path, source pixels.</param>
|
||||
/// <param name="CropCost">Area fraction a centred crop must give up to hide every correction.</param>
|
||||
/// <param name="PanPxPerFrame">Average drift of the smoothed path.</param>
|
||||
/// <param name="MeasuredFraction">Share of frame pairs whose shift could be measured (textured patches).</param>
|
||||
public sealed record ShakeSummary(double RmsPx, double MaxPx, double CropCost, double PanPxPerFrame, bool PanDetected, double MeasuredFraction);
|
||||
|
||||
/// <param name="MedianPx">Median of the sampled pairs' median flow magnitude, source px per frame.</param>
|
||||
/// <param name="P95Px">Median of the sampled pairs' 95th-percentile magnitude: the typical speed of moving content.</param>
|
||||
public sealed record MotionSummary(int Pairs, double? MedianPx, double? P95Px, double? MaxP95Px);
|
||||
|
||||
/// <param name="StarDensity">Median stars per megapixel over dark frames.</param>
|
||||
/// <param name="CloudCoverage">Mean cloudy fraction of the sky.</param>
|
||||
public sealed record SkySummary(DayNight Light, double DarkFraction, int DarkFrames, int StarFrames, double? StarDensity, bool StarField, double? CloudCoverage);
|
||||
|
||||
public sealed record FrameIssues(int[] Undecodable, int[] Black, int[] White, int[] Duplicate, int[] SceneChanges)
|
||||
{
|
||||
/// <summary>Every frame that would flash or stutter in a render, ascending.</summary>
|
||||
public int[] Excludable() => [.. Undecodable.Concat(Black).Concat(White).Concat(Duplicate).Distinct().Order()];
|
||||
}
|
||||
|
||||
/// <summary>A <see cref="Finding"/> with numeric arguments, so the report round-trips through JSON.</summary>
|
||||
public sealed record DeepFinding(string Key, FindingSeverity Severity, double[] Arguments, int[] Frames)
|
||||
{
|
||||
public Finding ToFinding() => new(Key, Severity, [.. Arguments.Cast<object>()], Frames);
|
||||
}
|
||||
|
||||
/// <summary>Resource keys of the findings produced by the deep analysis; arguments are numbers.</summary>
|
||||
public static class DeepFindingKeys
|
||||
{
|
||||
public const string UnreadableFrames = "Advisor.Finding.UnreadableFrames";
|
||||
public const string BlackFrames = "Advisor.Finding.BlackFrames";
|
||||
public const string WhiteFrames = "Advisor.Finding.WhiteFrames";
|
||||
public const string DuplicateFrames = "Advisor.Finding.DuplicateFrames";
|
||||
public const string SceneChanges = "Advisor.Finding.SceneChanges";
|
||||
public const string Flicker = "Advisor.Finding.Flicker";
|
||||
public const string Shake = "Advisor.Finding.Shake";
|
||||
public const string Panning = "Advisor.Finding.Panning";
|
||||
public const string StarField = "Advisor.Finding.StarField";
|
||||
public const string Clouds = "Advisor.Finding.Clouds";
|
||||
public const string WhiteBalanceJitter = "Advisor.Finding.WhiteBalanceJitter";
|
||||
public const string StepsNotReceived = "Advisor.Finding.StepsNotReceived";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of the deep analysis (BRIEF §10.2), stored as <c>.titano/analysis.json</c>. Versioned; per-frame data is
|
||||
/// columnar (<see cref="DeepFrameColumns"/>), sequence-level results are small records.
|
||||
/// </summary>
|
||||
public sealed record DeepAnalysisReport
|
||||
{
|
||||
public const int CurrentVersion = 1;
|
||||
|
||||
/// <summary>File name inside a sequence's <c>.titano</c> folder.</summary>
|
||||
public const string FileName = "analysis.json";
|
||||
|
||||
public int Version { get; init; } = CurrentVersion;
|
||||
public int FrameCount { get; init; }
|
||||
|
||||
/// <summary>Dominant oriented source size from metadata (0 when unknown): the pixel unit of shifts and motion.</summary>
|
||||
public int SourceWidth { get; init; }
|
||||
|
||||
public int SourceHeight { get; init; }
|
||||
public int AnalysisLongEdge { get; init; }
|
||||
public DeepFrameColumns Frames { get; init; } = new();
|
||||
public required FlickerSummary Flicker { get; init; }
|
||||
public RegionSummary? Regions { get; init; }
|
||||
public required ExposureSummary Exposure { get; init; }
|
||||
public required WhiteBalanceSummary WhiteBalance { get; init; }
|
||||
public required ShakeSummary Shake { get; init; }
|
||||
public required MotionSummary Motion { get; init; }
|
||||
public required SkySummary Sky { get; init; }
|
||||
public required FrameIssues Issues { get; init; }
|
||||
|
||||
/// <summary>Suggested hero frame: sharp, well exposed, near golden hour or mid-sequence, not excluded.</summary>
|
||||
public int? HeroIndex { get; init; }
|
||||
|
||||
public IReadOnlyList<DeepFinding> Findings { get; init; } = [];
|
||||
|
||||
public string ToJson() => JsonSerializer.Serialize(this, DeepAnalysisJson.Default.DeepAnalysisReport);
|
||||
|
||||
public static DeepAnalysisReport FromJson(string json)
|
||||
{
|
||||
var report = JsonSerializer.Deserialize(json, DeepAnalysisJson.Default.DeepAnalysisReport)
|
||||
?? throw new InvalidDataException("analysis.json is empty.");
|
||||
if (report.Version > CurrentVersion)
|
||||
throw new InvalidDataException($"analysis.json version {report.Version} is newer than this Titano ({CurrentVersion}).");
|
||||
return report;
|
||||
}
|
||||
|
||||
/// <summary>Writes the report atomically (temp file, flush, rename).</summary>
|
||||
public void Save(string path) => AtomicFile.WriteAllText(path, ToJson());
|
||||
|
||||
public static DeepAnalysisReport Load(string path) => FromJson(File.ReadAllText(path));
|
||||
}
|
||||
|
||||
/// <summary>Compact digest of a <see cref="DeepAnalysisReport"/> for the sequence row of the database (BRIEF §5).</summary>
|
||||
public sealed record AnalysisSummary
|
||||
{
|
||||
public int Version { get; init; } = DeepAnalysisReport.CurrentVersion;
|
||||
public int FrameCount { get; init; }
|
||||
public DayNight Light { get; init; }
|
||||
public double FlickerEv { get; init; }
|
||||
public double? FlickerPeriod { get; init; }
|
||||
|
||||
/// <summary>Shake RMS in source pixels.</summary>
|
||||
public double ShakePx { get; init; }
|
||||
|
||||
/// <summary>Typical speed of moving content (<see cref="MotionSummary.P95Px"/>).</summary>
|
||||
public double? MotionPx { get; init; }
|
||||
|
||||
public bool StarField { get; init; }
|
||||
public double? StarDensity { get; init; }
|
||||
public double? CloudCoverage { get; init; }
|
||||
public int WarningCount { get; init; }
|
||||
public int? HeroIndex { get; init; }
|
||||
|
||||
public static AnalysisSummary From(DeepAnalysisReport report)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(report);
|
||||
return new AnalysisSummary
|
||||
{
|
||||
FrameCount = report.FrameCount,
|
||||
Light = report.Sky.Light,
|
||||
FlickerEv = report.Flicker.AmplitudeEv,
|
||||
FlickerPeriod = report.Flicker.PeriodFrames,
|
||||
ShakePx = report.Shake.RmsPx,
|
||||
MotionPx = report.Motion.P95Px,
|
||||
StarField = report.Sky.StarField,
|
||||
StarDensity = report.Sky.StarDensity,
|
||||
CloudCoverage = report.Sky.CloudCoverage,
|
||||
WarningCount = report.Findings.Count(f => f.Severity >= FindingSeverity.Warning),
|
||||
HeroIndex = report.HeroIndex,
|
||||
};
|
||||
}
|
||||
|
||||
public string ToJson() => JsonSerializer.Serialize(this, DeepAnalysisJson.Default.AnalysisSummary);
|
||||
|
||||
public static AnalysisSummary FromJson(string json) =>
|
||||
JsonSerializer.Deserialize(json, DeepAnalysisJson.Default.AnalysisSummary) ?? throw new InvalidDataException("Analysis summary is empty.");
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
using System.Diagnostics;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Motion;
|
||||
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>
|
||||
/// Deep analysis (BRIEF §10.2): one streaming pass over every frame. A pre-pass segments sky and land on a sample of
|
||||
/// thumbnails; then frames are probed in parallel batches (<see cref="FrameProbe"/>: statistics, stars, region
|
||||
/// luminance, clouds) and compared in order with the previous usable frame (nine-patch phase-correlation shift, duplicate
|
||||
/// test, exposure-aligned histogram distance, Draft optical flow on every <see cref="DeepAnalysisOptions.FlowEvery"/>-th
|
||||
/// pair). Memory is one batch of analysis images plus the previous frame's reduced planes. After every batch the new
|
||||
/// <see cref="DeepFrame"/>s go to the checkpoint, so an interrupted run resumes after the last persisted frame with
|
||||
/// identical results. Sequence-level results are computed by <see cref="DeepSequenceAnalysis"/>.
|
||||
/// </summary>
|
||||
public static class DeepAnalyzer
|
||||
{
|
||||
public static async Task<DeepAnalysisReport> AnalyzeAsync(
|
||||
IFrameSource source,
|
||||
DeepAnalysisOptions? options = null,
|
||||
IDeepAnalysisCheckpoint? checkpoint = null,
|
||||
IProgress<AnalysisProgress>? progress = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(source);
|
||||
options ??= DeepAnalysisOptions.Default;
|
||||
options.Validate();
|
||||
var n = source.Count;
|
||||
var metadata = new LightFrame[n];
|
||||
for (var i = 0; i < n; i++) metadata[i] = source.Metadata(i);
|
||||
|
||||
var frames = new DeepFrame[n];
|
||||
var start = 0;
|
||||
if (checkpoint is not null)
|
||||
{
|
||||
var saved = await checkpoint.LoadAsync(Fingerprint(n, options), cancellationToken).ConfigureAwait(false);
|
||||
while (start < saved.Count && start < n && saved[start].Index == start) frames[start] = saved[start++];
|
||||
}
|
||||
|
||||
var mask = n > 0 ? await FrameProbe.SegmentRegionsAsync(source, options, cancellationToken).ConfigureAwait(false) : null;
|
||||
using var state = new SequentialState(options);
|
||||
await PrimeAsync(source, frames, start, options, mask, state, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var clock = Stopwatch.StartNew();
|
||||
var batch = options.MaxDegreeOfParallelism;
|
||||
for (var first = start; first < n; first += batch)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var count = int.Min(batch, n - first);
|
||||
var tasks = new Task<FrameProbe>[count];
|
||||
for (var k = 0; k < count; k++)
|
||||
{
|
||||
var index = first + k;
|
||||
tasks[k] = Task.Run(() => FrameProbe.RunAsync(source, index, options, mask, cancellationToken), cancellationToken);
|
||||
}
|
||||
var produced = new DeepFrame[count];
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(tasks).ConfigureAwait(false);
|
||||
for (var k = 0; k < count; k++) produced[k] = frames[first + k] = state.Next(tasks[k].Result, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var task in tasks)
|
||||
{
|
||||
if (task.IsCompletedSuccessfully) task.Result.Dispose();
|
||||
}
|
||||
}
|
||||
if (checkpoint is not null) await checkpoint.AppendAsync(produced, cancellationToken).ConfigureAwait(false);
|
||||
if (progress is not null)
|
||||
{
|
||||
var done = first + count;
|
||||
var rate = (done - start) / double.Max(clock.Elapsed.TotalSeconds, 1e-3);
|
||||
progress.Report(new AnalysisProgress(done, n, rate, TimeSpan.FromSeconds((n - done) / rate)));
|
||||
}
|
||||
}
|
||||
|
||||
return DeepSequenceAnalysis.Build(frames, metadata, mask, options);
|
||||
}
|
||||
|
||||
/// <summary>Identifies checkpoints written by this version for this frame count and these options.</summary>
|
||||
public static string Fingerprint(int frameCount, DeepAnalysisOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
return $"titano-deep/{DeepAnalysisReport.CurrentVersion};frames={frameCount};{options.Fingerprint()}";
|
||||
}
|
||||
|
||||
/// <summary>Restores the comparison state of a resumed run from the last usable frame before <paramref name="start"/>.</summary>
|
||||
private static async Task PrimeAsync(IFrameSource source, DeepFrame[] frames, int start, DeepAnalysisOptions options, RegionMask? mask, SequentialState state, CancellationToken cancellationToken)
|
||||
{
|
||||
var last = start - 1;
|
||||
while (last >= 0 && !frames[last].IsUsable) last--;
|
||||
if (last < 0) return;
|
||||
using var probe = await FrameProbe.RunAsync(source, last, options, mask, cancellationToken).ConfigureAwait(false);
|
||||
state.Next(probe, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The order-dependent comparisons: each usable frame is compared with the previous usable frame and becomes the
|
||||
/// new reference; undecodable, black and white frames leave the state untouched.
|
||||
/// </summary>
|
||||
private sealed class SequentialState(DeepAnalysisOptions options) : IDisposable
|
||||
{
|
||||
private const int MinimumStabilizerSize = 16;
|
||||
private readonly OpticalFlowEngine _flow = new(FlowQuality.Draft);
|
||||
private Stabilizer? _stabilizer;
|
||||
private FrameStatistics? _previous;
|
||||
private ImageBuffer? _previousFlow;
|
||||
private int _previousIndex = -1;
|
||||
|
||||
public DeepFrame Next(FrameProbe probe, CancellationToken cancellationToken)
|
||||
{
|
||||
if (probe.Statistics is not { } stats) return new DeepFrame { Index = probe.Index, Flags = DeepFrameFlags.Undecodable };
|
||||
var frame = Measured(probe.Index, stats);
|
||||
if (!probe.IsUsable) return frame;
|
||||
|
||||
var (shiftX, shiftY) = Shift(probe);
|
||||
var duplicate = _previous is not null && FrameStatistics.AreLikelyDuplicates(_previous, stats);
|
||||
var scene = _previous is null ? double.NaN : HistogramDistance.ExposureAligned(_previous, stats);
|
||||
var (motionMedian, motionP95) = Motion(probe, stats, cancellationToken);
|
||||
frame = frame with
|
||||
{
|
||||
Flags = frame.Flags | (duplicate ? DeepFrameFlags.Duplicate : DeepFrameFlags.None),
|
||||
ShiftX = shiftX,
|
||||
ShiftY = shiftY,
|
||||
MotionMedian = motionMedian,
|
||||
MotionP95 = motionP95,
|
||||
StarDensity = probe.StarDensity,
|
||||
CloudCoverage = probe.CloudCoverage,
|
||||
SkyLogLuminance = probe.SkyLogLuminance,
|
||||
LandLogLuminance = probe.LandLogLuminance,
|
||||
SceneDistance = scene,
|
||||
};
|
||||
|
||||
_previous = stats;
|
||||
_previousIndex = probe.Index;
|
||||
_previousFlow?.Dispose();
|
||||
_previousFlow = null;
|
||||
_previousFlow = probe.TakeFlow();
|
||||
return frame;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_flow.Dispose();
|
||||
_stabilizer?.Dispose();
|
||||
_previousFlow?.Dispose();
|
||||
}
|
||||
|
||||
private DeepFrame Measured(int index, FrameStatistics stats)
|
||||
{
|
||||
var flags = DeepFrameFlags.None;
|
||||
if (stats.IsBlack()) flags |= DeepFrameFlags.Black;
|
||||
if (stats.IsWhite()) flags |= DeepFrameFlags.White;
|
||||
if (stats.MeanLogLuminance <= options.DarkLogLuminance) flags |= DeepFrameFlags.Dark;
|
||||
var green = stats.MeanGreen;
|
||||
return new DeepFrame
|
||||
{
|
||||
Index = index,
|
||||
Flags = flags,
|
||||
LogLuminance = stats.MeanLogLuminance,
|
||||
ClipLow = stats.ClipLowFraction,
|
||||
ClipHigh = stats.ClipHighFraction,
|
||||
RedRatio = stats.MeanRed > 0 && green > 0 ? double.Log2(stats.MeanRed / green) : double.NaN,
|
||||
BlueRatio = stats.MeanBlue > 0 && green > 0 ? double.Log2(stats.MeanBlue / green) : double.NaN,
|
||||
Kelvin = stats.ColorTemperature?.Kelvin ?? double.NaN,
|
||||
Sharpness = stats.Sharpness,
|
||||
Noise = stats.Noise,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Content motion at the centre of the correlation plane, in source pixels; NaN for the first frame of a
|
||||
/// size, too small planes and frames without textured patches.</summary>
|
||||
private (double X, double Y) Shift(FrameProbe probe)
|
||||
{
|
||||
int w = probe.CorrelationWidth, h = probe.CorrelationHeight;
|
||||
if (w < MinimumStabilizerSize || h < MinimumStabilizerSize || probe.Correlation is null) return (double.NaN, double.NaN);
|
||||
if (_stabilizer is null || _stabilizer.Width != w || _stabilizer.Height != h)
|
||||
{
|
||||
_stabilizer?.Dispose();
|
||||
_stabilizer = new Stabilizer(w, h);
|
||||
_stabilizer.AddFrame(probe.Correlation);
|
||||
return (double.NaN, double.NaN);
|
||||
}
|
||||
var motion = _stabilizer.AddFrame(probe.Correlation);
|
||||
if (!motion.Estimated) return (double.NaN, double.NaN);
|
||||
double cx = (w - 1) / 2.0, cy = (h - 1) / 2.0;
|
||||
var (x, y) = motion.Transform.Apply(cx, cy);
|
||||
return ((x - cx) * probe.CorrelationScale, (y - cy) * probe.CorrelationScale);
|
||||
}
|
||||
|
||||
/// <summary>Draft flow from the previous frame with the current frame brought to the previous mean level, so
|
||||
/// flicker and exposure steps do not read as motion; magnitudes in source pixels per frame.</summary>
|
||||
private (double Median, double P95) Motion(FrameProbe probe, FrameStatistics stats, CancellationToken cancellationToken)
|
||||
{
|
||||
if (probe.Index % options.FlowEvery != 0 || _previousIndex != probe.Index - 1) return (double.NaN, double.NaN);
|
||||
if (probe.Flow is not { } current || _previousFlow is not { } previous || previous.Width != current.Width || previous.Height != current.Height)
|
||||
return (double.NaN, double.NaN);
|
||||
using var levelled = current.Clone();
|
||||
var gain = (float)double.Exp2(_previous!.MeanLogLuminance - stats.MeanLogLuminance);
|
||||
if (float.IsFinite(gain))
|
||||
{
|
||||
foreach (ref var v in levelled.Data) v *= gain;
|
||||
}
|
||||
using var field = _flow.Compute(previous, levelled, cancellationToken);
|
||||
var statistics = field.Statistics();
|
||||
return (statistics.MedianMagnitude * probe.FlowScale, statistics.P95Magnitude * probe.FlowScale);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>Per-frame conditions found by the deep analysis.</summary>
|
||||
[Flags]
|
||||
public enum DeepFrameFlags
|
||||
{
|
||||
None = 0,
|
||||
|
||||
/// <summary>The source could not decode the frame; every measurement is NaN.</summary>
|
||||
Undecodable = 1,
|
||||
|
||||
Black = 2,
|
||||
White = 4,
|
||||
|
||||
/// <summary>Statistically identical to the previous decoded frame.</summary>
|
||||
Duplicate = 8,
|
||||
|
||||
/// <summary>Exposure-aligned histogram jump from the previous frame (camera moved or bumped); set at sequence level.</summary>
|
||||
SceneChange = 16,
|
||||
|
||||
/// <summary>Trimmed mean log2 luminance at or below <see cref="DeepAnalysisOptions.DarkLogLuminance"/>.</summary>
|
||||
Dark = 32,
|
||||
|
||||
/// <summary>Dark frame with at least <see cref="DeepAnalysisOptions.StarFieldDensity"/> stars per megapixel; set at sequence level.</summary>
|
||||
StarField = 64,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exact per-frame measurements of <see cref="DeepAnalyzer"/>: what a checkpoint persists and what the
|
||||
/// sequence-level results are computed from. NaN marks a value that was not measured. Shifts and motion are in
|
||||
/// source pixels (oriented full resolution).
|
||||
/// </summary>
|
||||
public sealed record DeepFrame
|
||||
{
|
||||
public int Index { get; init; }
|
||||
public DeepFrameFlags Flags { get; init; }
|
||||
|
||||
/// <summary>Trimmed mean log2 luminance (<see cref="FrameStatistics.MeanLogLuminance"/>).</summary>
|
||||
public double LogLuminance { get; init; } = double.NaN;
|
||||
|
||||
public double ClipLow { get; init; } = double.NaN;
|
||||
public double ClipHigh { get; init; } = double.NaN;
|
||||
|
||||
/// <summary>log2(R/G) of the mean colour (white-balance drift signal).</summary>
|
||||
public double RedRatio { get; init; } = double.NaN;
|
||||
|
||||
/// <summary>log2(B/G) of the mean colour.</summary>
|
||||
public double BlueRatio { get; init; } = double.NaN;
|
||||
|
||||
/// <summary>Correlated colour temperature of the mean colour; NaN when not near-Planckian.</summary>
|
||||
public double Kelvin { get; init; } = double.NaN;
|
||||
|
||||
public double Sharpness { get; init; } = double.NaN;
|
||||
public double Noise { get; init; } = double.NaN;
|
||||
|
||||
/// <summary>Content motion from the previous decoded frame at the image centre (nine-patch similarity fit).</summary>
|
||||
public double ShiftX { get; init; } = double.NaN;
|
||||
|
||||
public double ShiftY { get; init; } = double.NaN;
|
||||
|
||||
/// <summary>Median optical-flow magnitude from the previous frame, px per frame (sampled pairs only).</summary>
|
||||
public double MotionMedian { get; init; } = double.NaN;
|
||||
|
||||
/// <summary>95th-percentile optical-flow magnitude from the previous frame (sampled pairs only).</summary>
|
||||
public double MotionP95 { get; init; } = double.NaN;
|
||||
|
||||
/// <summary>Point sources per megapixel of the analysis image (dark frames only).</summary>
|
||||
public double StarDensity { get; init; } = double.NaN;
|
||||
|
||||
/// <summary>Fraction of sky blocks with cloud texture (frames with a sky region only).</summary>
|
||||
public double CloudCoverage { get; init; } = double.NaN;
|
||||
|
||||
/// <summary>Trimmed mean log2 luminance of the sky region (region deflicker signal).</summary>
|
||||
public double SkyLogLuminance { get; init; } = double.NaN;
|
||||
|
||||
public double LandLogLuminance { get; init; } = double.NaN;
|
||||
|
||||
/// <summary>Exposure-aligned chi-square histogram distance from the previous decoded frame.</summary>
|
||||
public double SceneDistance { get; init; } = double.NaN;
|
||||
|
||||
public bool IsUsable => (Flags & (DeepFrameFlags.Undecodable | DeepFrameFlags.Black | DeepFrameFlags.White)) == 0;
|
||||
}
|
||||
@@ -0,0 +1,450 @@
|
||||
using Titano.Core.Mathematics;
|
||||
using Titano.Core.Motion;
|
||||
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>
|
||||
/// Sequence-level results of the deep analysis, computed from the exact <see cref="DeepFrame"/>s and the frame
|
||||
/// metadata: exposure steps as received (<see cref="HolyGrailEngine"/>), flicker spectrum and amplitude on the
|
||||
/// step-free luminance (<see cref="FlickerSpectrum"/>, <see cref="DeflickerEngine"/>), region divergence, white-balance
|
||||
/// jitter (<see cref="WhiteBalanceSmoother"/>), shake from the integrated shifts (camera path minus its
|
||||
/// <see cref="PathSmoother.LocalLinear"/> smoothing, as <see cref="Stabilizer"/>), motion, stars, clouds, day/night,
|
||||
/// scene changes, the hero frame and the findings.
|
||||
/// </summary>
|
||||
public static class DeepSequenceAnalysis
|
||||
{
|
||||
/// <summary>Flicker below this RMS is not visible (≈ 1.4 % brightness).</summary>
|
||||
public const double VisibleFlickerEv = 0.02;
|
||||
|
||||
/// <summary>Colour-ratio jitter (log2 RMS) above which white-balance smoothing helps (≈ 0.7 %).</summary>
|
||||
public const double VisibleWhiteBalanceJitter = 0.01;
|
||||
|
||||
/// <summary>Mid-grey (18 %) in log2 luminance, the hero frame's exposure target.</summary>
|
||||
private const double MidGreyEv = -2.47;
|
||||
|
||||
private const int HorizonSamples = 64;
|
||||
|
||||
public static DeepAnalysisReport Build(IReadOnlyList<DeepFrame> frames, IReadOnlyList<LightFrame> metadata, RegionMask? mask, DeepAnalysisOptions? options = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(frames);
|
||||
ArgumentNullException.ThrowIfNull(metadata);
|
||||
options ??= DeepAnalysisOptions.Default;
|
||||
var n = frames.Count;
|
||||
if (metadata.Count != n) throw new ArgumentException("One metadata entry per frame is required.", nameof(metadata));
|
||||
var light = LightAnalyzer.Analyze(metadata);
|
||||
var (sourceWidth, sourceHeight) = light.DominantSize ?? (0, 0);
|
||||
var luminance = new double[n];
|
||||
for (var i = 0; i < n; i++) luminance[i] = frames[i].IsUsable ? frames[i].LogLuminance : double.NaN;
|
||||
|
||||
var (exposure, offsets) = Exposure(luminance, light);
|
||||
var flat = new double[n];
|
||||
for (var i = 0; i < n; i++) flat[i] = luminance[i] - offsets[i];
|
||||
exposure = exposure with { SceneRangeEv = Range(flat, 0.05, 0.95) };
|
||||
var flicker = Flicker(flat);
|
||||
var sceneChanges = SceneChanges(frames, options);
|
||||
var starFrames = new HashSet<int>();
|
||||
var sky = Sky(frames, flat, light, options, starFrames);
|
||||
var shake = Shake(frames, sourceWidth, sourceHeight, options);
|
||||
var issues = new FrameIssues(
|
||||
Where(frames, f => (f.Flags & DeepFrameFlags.Undecodable) != 0),
|
||||
Where(frames, f => (f.Flags & DeepFrameFlags.Black) != 0),
|
||||
Where(frames, f => (f.Flags & DeepFrameFlags.White) != 0),
|
||||
Where(frames, f => (f.Flags & DeepFrameFlags.Duplicate) != 0),
|
||||
[.. sceneChanges.Order()]);
|
||||
var report = new DeepAnalysisReport
|
||||
{
|
||||
FrameCount = n,
|
||||
SourceWidth = sourceWidth,
|
||||
SourceHeight = sourceHeight,
|
||||
AnalysisLongEdge = options.AnalysisLongEdge(int.Max(sourceWidth, sourceHeight)),
|
||||
Frames = Columns(frames, sceneChanges, starFrames),
|
||||
Flicker = flicker,
|
||||
Regions = Regions(frames, mask, flicker.RecommendedWindow),
|
||||
Exposure = exposure,
|
||||
WhiteBalance = WhiteBalance(frames),
|
||||
Shake = shake,
|
||||
Motion = Motion(frames),
|
||||
Sky = sky,
|
||||
Issues = issues,
|
||||
HeroIndex = Hero(frames, sceneChanges, light.DayPhases),
|
||||
};
|
||||
return report with { Findings = Findings(report) };
|
||||
}
|
||||
|
||||
private static (ExposureSummary Summary, double[] Offsets) Exposure(double[] luminance, LightAnalysisReport light)
|
||||
{
|
||||
var n = luminance.Length;
|
||||
if (light.ExposureSteps.Count == 0 || n == 0) return (new ExposureSummary(0, 0, null, null, 0, []), new double[n]);
|
||||
var result = HolyGrailEngine.Compute(luminance, light.ExposureValues);
|
||||
var received = result.Transitions.Where(t => t.Received).Select(t => t.Index).ToArray();
|
||||
double? spacing = null;
|
||||
int? length = null;
|
||||
if (received.Length >= 2)
|
||||
{
|
||||
var gaps = new double[received.Length - 1];
|
||||
for (var i = 1; i < received.Length; i++) gaps[i - 1] = received[i] - received[i - 1];
|
||||
spacing = Statistics.Median(gaps);
|
||||
length = int.Max(2, (int)double.Round(spacing.Value / 2));
|
||||
}
|
||||
else if (received.Length == 1)
|
||||
{
|
||||
length = HolyGrailOptions.Default.SingleStepTransitionFrames;
|
||||
}
|
||||
var steps = result.Transitions
|
||||
.Select(t => new ExposureStepSummary(t.Index, Round(t.ExpectedEv, 3), double.IsFinite(t.MeasuredEv) ? Round(t.MeasuredEv, 3) : null, t.Received, Round(t.Length, 2)))
|
||||
.ToList();
|
||||
return (new ExposureSummary(result.Transitions.Count, received.Length, spacing, length, 0, steps), result.StepOffsets);
|
||||
}
|
||||
|
||||
private static FlickerSummary Flicker(double[] flat)
|
||||
{
|
||||
if (Count(flat) < 3) return new FlickerSummary(0, 0, null, 0, 0, FlickerSpectrumOptions.Default.MinimumWindow);
|
||||
var spectrum = FlickerSpectrum.Analyze(flat);
|
||||
var diagnostics = DeflickerEngine.Compute(flat, new DeflickerOptions { Window = spectrum.RecommendedWindow }).Diagnostics;
|
||||
return new FlickerSummary(
|
||||
Round(diagnostics.ResidualRms, 4),
|
||||
Round(diagnostics.FlickerBefore, 4),
|
||||
spectrum.DominantPeriod is { } p ? Round(p, 2) : null,
|
||||
Round(spectrum.PeakCorrelation, 3),
|
||||
Round(spectrum.CorrelationLength, 2),
|
||||
spectrum.RecommendedWindow);
|
||||
}
|
||||
|
||||
private static RegionSummary? Regions(IReadOnlyList<DeepFrame> frames, RegionMask? mask, int window)
|
||||
{
|
||||
if (mask is null) return null;
|
||||
double[]? horizon = null;
|
||||
if (mask.Horizon is { } h)
|
||||
{
|
||||
var count = int.Min(HorizonSamples, mask.Width);
|
||||
horizon = new double[count];
|
||||
for (var k = 0; k < count; k++)
|
||||
{
|
||||
var x = count > 1 ? (int)((long)k * (mask.Width - 1) / (count - 1)) : 0;
|
||||
horizon[k] = Round(h[x] / mask.Height, 4);
|
||||
}
|
||||
}
|
||||
double skyFraction = 0;
|
||||
foreach (var w in mask.Sky) skyFraction += w;
|
||||
skyFraction /= mask.Sky.Length;
|
||||
|
||||
var n = frames.Count;
|
||||
var sky = new double[n];
|
||||
var land = new double[n];
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
sky[i] = frames[i].IsUsable ? frames[i].SkyLogLuminance : double.NaN;
|
||||
land[i] = frames[i].IsUsable ? frames[i].LandLogLuminance : double.NaN;
|
||||
}
|
||||
var divergence = 0.0;
|
||||
if (Count(sky) >= 3 && Count(land) >= 3)
|
||||
{
|
||||
var skyFit = LocalRegression.Smooth(sky, window, 1, 3).Fitted;
|
||||
var landFit = LocalRegression.Smooth(land, window, 1, 3).Fitted;
|
||||
var difference = new double[n];
|
||||
for (var i = 0; i < n; i++) difference[i] = sky[i] - skyFit[i] - (land[i] - landFit[i]);
|
||||
divergence = Statistics.Rms(difference);
|
||||
}
|
||||
return new RegionSummary(mask.Method, Round(mask.Confidence, 3), Round(skyFraction, 3), mask.Width, mask.Height, horizon,
|
||||
Round(double.IsFinite(divergence) ? divergence : 0, 4));
|
||||
}
|
||||
|
||||
private static WhiteBalanceSummary WhiteBalance(IReadOnlyList<DeepFrame> frames)
|
||||
{
|
||||
var n = frames.Count;
|
||||
var red = new double[n];
|
||||
var green = new double[n];
|
||||
var blue = new double[n];
|
||||
var kelvin = new double[n];
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
var f = frames[i];
|
||||
var valid = f.IsUsable && double.IsFinite(f.RedRatio) && double.IsFinite(f.BlueRatio);
|
||||
red[i] = valid ? double.Exp2(f.RedRatio) : double.NaN;
|
||||
green[i] = valid ? 1 : double.NaN;
|
||||
blue[i] = valid ? double.Exp2(f.BlueRatio) : double.NaN;
|
||||
kelvin[i] = f.IsUsable ? f.Kelvin : double.NaN;
|
||||
}
|
||||
if (Count(red) < 3) return new WhiteBalanceSummary(0, 0, null, null);
|
||||
var result = WhiteBalanceSmoother.Smooth(red, green, blue);
|
||||
var deviation = new double[n];
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
var dr = result.RedRatio[i] - result.SmoothedRedRatio[i];
|
||||
var db = result.BlueRatio[i] - result.SmoothedBlueRatio[i];
|
||||
deviation[i] = double.Sqrt(dr * dr + db * db);
|
||||
}
|
||||
var drift = double.Max(Range(result.SmoothedRedRatio, 0, 1), Range(result.SmoothedBlueRatio, 0, 1));
|
||||
var hasKelvin = Count(kelvin) > 0;
|
||||
return new WhiteBalanceSummary(
|
||||
Round(Statistics.Rms(deviation), 4),
|
||||
Round(drift, 4),
|
||||
hasKelvin ? double.Round(Statistics.Percentile(kelvin, 0.05)) : null,
|
||||
hasKelvin ? double.Round(Statistics.Percentile(kelvin, 0.95)) : null);
|
||||
}
|
||||
|
||||
private static ShakeSummary Shake(IReadOnlyList<DeepFrame> frames, int sourceWidth, int sourceHeight, DeepAnalysisOptions options)
|
||||
{
|
||||
var n = frames.Count;
|
||||
if (n < 2) return new ShakeSummary(0, 0, 0, 0, false, 0);
|
||||
var x = new double[n];
|
||||
var y = new double[n];
|
||||
int measured = 0, pairs = 0;
|
||||
for (var i = 1; i < n; i++)
|
||||
{
|
||||
var f = frames[i];
|
||||
var known = double.IsFinite(f.ShiftX) && double.IsFinite(f.ShiftY);
|
||||
if (f.IsUsable) pairs++;
|
||||
if (known) measured++;
|
||||
x[i] = x[i - 1] + (known ? f.ShiftX : 0);
|
||||
y[i] = y[i - 1] + (known ? f.ShiftY : 0);
|
||||
}
|
||||
var sx = PathSmoother.LocalLinear(x, options.ShakeSmoothingSigma);
|
||||
var sy = PathSmoother.LocalLinear(y, options.ShakeSmoothingSigma);
|
||||
double sum = 0, max = 0, maxX = 0, maxY = 0;
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
double rx = x[i] - sx[i], ry = y[i] - sy[i];
|
||||
var r2 = rx * rx + ry * ry;
|
||||
sum += r2;
|
||||
max = double.Max(max, double.Sqrt(r2));
|
||||
maxX = double.Max(maxX, double.Abs(rx));
|
||||
maxY = double.Max(maxY, double.Abs(ry));
|
||||
}
|
||||
var width = sourceWidth > 0 ? sourceWidth : options.AnalysisLongEdge(0);
|
||||
var height = sourceHeight > 0 ? sourceHeight : width;
|
||||
var scale = double.Clamp(double.Min(1 - 2 * maxX / width, 1 - 2 * maxY / height), 0, 1);
|
||||
var drift = double.Hypot(sx[n - 1] - sx[0], sy[n - 1] - sy[0]);
|
||||
return new ShakeSummary(
|
||||
Round(double.Sqrt(sum / n), 3),
|
||||
Round(max, 3),
|
||||
Round(1 - scale * scale, 4),
|
||||
Round(drift / (n - 1), 3),
|
||||
drift >= 0.02 * width,
|
||||
Round(pairs > 1 ? double.Min(1, measured / (double)(pairs - 1)) : 0, 3));
|
||||
}
|
||||
|
||||
private static MotionSummary Motion(IReadOnlyList<DeepFrame> frames)
|
||||
{
|
||||
var medians = frames.Where(f => double.IsFinite(f.MotionP95)).Select(f => f.MotionMedian).ToArray();
|
||||
var p95 = frames.Where(f => double.IsFinite(f.MotionP95)).Select(f => f.MotionP95).ToArray();
|
||||
if (p95.Length == 0) return new MotionSummary(0, null, null, null);
|
||||
return new MotionSummary(p95.Length, Round(Statistics.Median(medians), 3), Round(Statistics.Median(p95), 3), Round(p95.Max(), 3));
|
||||
}
|
||||
|
||||
private static SkySummary Sky(IReadOnlyList<DeepFrame> frames, double[] flat, LightAnalysisReport light, DeepAnalysisOptions options, HashSet<int> starFrames)
|
||||
{
|
||||
int usable = 0, dark = 0;
|
||||
var densities = new List<double>();
|
||||
var clouds = new List<double>();
|
||||
foreach (var f in frames)
|
||||
{
|
||||
if (!f.IsUsable) continue;
|
||||
usable++;
|
||||
if (double.IsFinite(f.CloudCoverage)) clouds.Add(f.CloudCoverage);
|
||||
if ((f.Flags & DeepFrameFlags.Dark) == 0) continue;
|
||||
dark++;
|
||||
if (!double.IsFinite(f.StarDensity)) continue;
|
||||
densities.Add(f.StarDensity);
|
||||
if (f.StarDensity >= options.StarFieldDensity) starFrames.Add(f.Index);
|
||||
}
|
||||
var darkFraction = usable > 0 ? (double)dark / usable : 0;
|
||||
return new SkySummary(
|
||||
DayNightOf(flat, darkFraction, usable, light.DayPhases),
|
||||
Round(darkFraction, 3),
|
||||
dark,
|
||||
starFrames.Count,
|
||||
densities.Count > 0 ? double.Round(Statistics.Median(densities.ToArray())) : null,
|
||||
starFrames.Count > 0 && starFrames.Count >= 0.25 * usable,
|
||||
clouds.Count > 0 ? Round(clouds.Average(), 3) : null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Solar phases when GPS gave them for most frames; otherwise the step-free luminance: a 10–90 % range of at
|
||||
/// least 3 EV is a transition (direction from the first and last tenth), else the dark-frame share decides.
|
||||
/// </summary>
|
||||
private static DayNight DayNightOf(double[] flat, double darkFraction, int usable, DayPhase?[] phases)
|
||||
{
|
||||
int day = 0, night = 0, known = 0;
|
||||
DayPhase? first = null, last = null;
|
||||
foreach (var phase in phases)
|
||||
{
|
||||
if (phase is not { } p) continue;
|
||||
known++;
|
||||
first ??= p;
|
||||
last = p;
|
||||
if (p is DayPhase.Day or DayPhase.GoldenHour) day++;
|
||||
else if (p is DayPhase.Night or DayPhase.AstronomicalTwilight or DayPhase.NauticalTwilight) night++;
|
||||
}
|
||||
if (known * 2 >= phases.Length && known > 0)
|
||||
{
|
||||
if (day > 0 && night > 0) return first >= DayPhase.CivilTwilight && last < DayPhase.CivilTwilight ? DayNight.DayToNight : DayNight.NightToDay;
|
||||
return night > day ? DayNight.Night : DayNight.Day;
|
||||
}
|
||||
if (usable == 0) return DayNight.Unknown;
|
||||
if (Range(flat, 0.1, 0.9) >= 3)
|
||||
{
|
||||
var tenth = int.Max(1, flat.Length / 10);
|
||||
var start = Statistics.Mean(flat.AsSpan(0, tenth));
|
||||
var end = Statistics.Mean(flat.AsSpan(flat.Length - tenth));
|
||||
if (double.IsFinite(start) && double.IsFinite(end)) return start > end ? DayNight.DayToNight : DayNight.NightToDay;
|
||||
}
|
||||
return darkFraction >= 0.5 ? DayNight.Night : DayNight.Day;
|
||||
}
|
||||
|
||||
/// <summary>Spikes of the exposure-aligned histogram distance: above max(minimum, median + 10 robust σ).</summary>
|
||||
private static HashSet<int> SceneChanges(IReadOnlyList<DeepFrame> frames, DeepAnalysisOptions options)
|
||||
{
|
||||
var candidates = frames.Where(f => f.IsUsable && (f.Flags & DeepFrameFlags.Duplicate) == 0 && double.IsFinite(f.SceneDistance)).ToList();
|
||||
var result = new HashSet<int>();
|
||||
if (candidates.Count == 0) return result;
|
||||
var distances = candidates.Select(f => f.SceneDistance).ToArray();
|
||||
var median = Statistics.Median(distances);
|
||||
var sigma = Statistics.MadToSigma * Statistics.MedianAbsoluteDeviation(distances, median);
|
||||
var threshold = double.Max(options.SceneChangeMinimum, median + 10 * sigma);
|
||||
foreach (var f in candidates)
|
||||
{
|
||||
if (f.SceneDistance > threshold) result.Add(f.Index);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Highest √(relative sharpness) × exposure × time score over usable, unique, non-scene-change frames. Exposure
|
||||
/// favours mid-grey and penalises clipping; time favours golden hour (when GPS gave phases) or the middle of the
|
||||
/// sequence. Ties go to the earliest frame.
|
||||
/// </summary>
|
||||
private static int? Hero(IReadOnlyList<DeepFrame> frames, HashSet<int> sceneChanges, DayPhase?[] phases)
|
||||
{
|
||||
var candidates = frames.Where(f => f.IsUsable && (f.Flags & DeepFrameFlags.Duplicate) == 0 && !sceneChanges.Contains(f.Index)
|
||||
&& double.IsFinite(f.Sharpness) && double.IsFinite(f.LogLuminance)).ToList();
|
||||
if (candidates.Count == 0) return null;
|
||||
var medianSharpness = Statistics.Median(candidates.Select(f => f.Sharpness).ToArray());
|
||||
var n = frames.Count;
|
||||
int? best = null;
|
||||
var bestScore = double.NegativeInfinity;
|
||||
foreach (var f in candidates)
|
||||
{
|
||||
var sharp = medianSharpness > 0 ? double.Min(2, f.Sharpness / medianSharpness) : 1;
|
||||
var level = (f.LogLuminance - MidGreyEv) / 1.5;
|
||||
var exposure = double.Exp(-level * level) * double.Max(0, 1 - 5 * f.ClipHigh - 2 * f.ClipLow);
|
||||
var phase = f.Index < phases.Length ? phases[f.Index] : null;
|
||||
var time = phase switch
|
||||
{
|
||||
DayPhase.GoldenHour => 1.0,
|
||||
DayPhase.CivilTwilight => 0.8,
|
||||
DayPhase.Day => 0.6,
|
||||
null => n > 1 ? 1 - 2 * double.Abs((double)f.Index / (n - 1) - 0.5) : 1,
|
||||
_ => 0.4,
|
||||
};
|
||||
var score = double.Sqrt(sharp) * exposure * (0.5 + 0.5 * time);
|
||||
if (score > bestScore)
|
||||
{
|
||||
bestScore = score;
|
||||
best = f.Index;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
private static List<DeepFinding> Findings(DeepAnalysisReport report)
|
||||
{
|
||||
var findings = new List<DeepFinding>();
|
||||
void Add(string key, FindingSeverity severity, int[] frames, params double[] args) =>
|
||||
findings.Add(new DeepFinding(key, severity, args, frames.Length > Finding.MaxFrames ? frames[..Finding.MaxFrames] : frames));
|
||||
|
||||
var issues = report.Issues;
|
||||
if (issues.Undecodable.Length > 0) Add(DeepFindingKeys.UnreadableFrames, FindingSeverity.Warning, issues.Undecodable, issues.Undecodable.Length);
|
||||
if (issues.Black.Length > 0) Add(DeepFindingKeys.BlackFrames, FindingSeverity.Warning, issues.Black, issues.Black.Length);
|
||||
if (issues.White.Length > 0) Add(DeepFindingKeys.WhiteFrames, FindingSeverity.Warning, issues.White, issues.White.Length);
|
||||
if (issues.Duplicate.Length > 0) Add(DeepFindingKeys.DuplicateFrames, FindingSeverity.Warning, issues.Duplicate, issues.Duplicate.Length);
|
||||
if (issues.SceneChanges.Length > 0) Add(DeepFindingKeys.SceneChanges, FindingSeverity.Warning, issues.SceneChanges, issues.SceneChanges.Length);
|
||||
var ignored = report.Exposure.Steps.Where(s => !s.Received).Select(s => s.Index).ToArray();
|
||||
if (ignored.Length > 0) Add(DeepFindingKeys.StepsNotReceived, FindingSeverity.Warning, ignored, ignored.Length);
|
||||
if (report.Flicker.AmplitudeEv >= VisibleFlickerEv) Add(DeepFindingKeys.Flicker, FindingSeverity.Info, [], Round(report.Flicker.AmplitudeEv, 3));
|
||||
if (report.Shake.RmsPx >= 0.3) Add(DeepFindingKeys.Shake, FindingSeverity.Info, [], Round(report.Shake.RmsPx, 2), Round(100 * report.Shake.CropCost, 1));
|
||||
if (report.Shake.PanDetected) Add(DeepFindingKeys.Panning, FindingSeverity.Info, [], Round(report.Shake.PanPxPerFrame, 2));
|
||||
if (report.Sky.StarField) Add(DeepFindingKeys.StarField, FindingSeverity.Info, [], report.Sky.StarFrames);
|
||||
if (report.Sky.CloudCoverage is { } cover && cover >= 0.1) Add(DeepFindingKeys.Clouds, FindingSeverity.Info, [], double.Round(100 * cover));
|
||||
if (report.WhiteBalance.JitterLog2 >= VisibleWhiteBalanceJitter)
|
||||
Add(DeepFindingKeys.WhiteBalanceJitter, FindingSeverity.Info, [], Round(100 * (double.Exp2(report.WhiteBalance.JitterLog2) - 1), 1));
|
||||
return findings;
|
||||
}
|
||||
|
||||
private static DeepFrameColumns Columns(IReadOnlyList<DeepFrame> frames, HashSet<int> sceneChanges, HashSet<int> starFrames)
|
||||
{
|
||||
var n = frames.Count;
|
||||
double?[] Column(Func<DeepFrame, double> value, Func<double, double> quantise)
|
||||
{
|
||||
var column = new double?[n];
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
var v = value(frames[i]);
|
||||
column[i] = double.IsFinite(v) ? quantise(v) : null;
|
||||
}
|
||||
return column;
|
||||
}
|
||||
|
||||
var flags = new int[n];
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
var f = frames[i].Flags;
|
||||
if (sceneChanges.Contains(i)) f |= DeepFrameFlags.SceneChange;
|
||||
if (starFrames.Contains(i)) f |= DeepFrameFlags.StarField;
|
||||
flags[i] = (int)f;
|
||||
}
|
||||
var sampled = frames.Where(f => double.IsFinite(f.MotionP95)).ToArray();
|
||||
return new DeepFrameColumns
|
||||
{
|
||||
LogLuminance = Column(f => f.LogLuminance, v => Round(v, 3)),
|
||||
ClipLow = Column(f => f.ClipLow, v => Round(v, 3)),
|
||||
ClipHigh = Column(f => f.ClipHigh, v => Round(v, 3)),
|
||||
RedRatio = Column(f => f.RedRatio, v => Round(v, 3)),
|
||||
BlueRatio = Column(f => f.BlueRatio, v => Round(v, 3)),
|
||||
Kelvin = Column(f => f.Kelvin, double.Round),
|
||||
Sharpness = Column(f => f.Sharpness, v => Significant(v, 2)),
|
||||
Noise = Column(f => f.Noise, v => Significant(v, 2)),
|
||||
ShiftX = Column(f => f.ShiftX, v => Round(v, 2)),
|
||||
ShiftY = Column(f => f.ShiftY, v => Round(v, 2)),
|
||||
StarDensity = Column(f => f.StarDensity, double.Round),
|
||||
CloudCoverage = Column(f => f.CloudCoverage, v => Round(v, 2)),
|
||||
SkyLogLuminance = Column(f => f.SkyLogLuminance, v => Round(v, 3)),
|
||||
LandLogLuminance = Column(f => f.LandLogLuminance, v => Round(v, 3)),
|
||||
SceneDistance = Column(f => f.SceneDistance, v => Round(v, 3)),
|
||||
Flags = flags,
|
||||
MotionFrames = [.. sampled.Select(f => f.Index)],
|
||||
MotionMedian = [.. sampled.Select(f => Round(f.MotionMedian, 2))],
|
||||
MotionP95 = [.. sampled.Select(f => Round(f.MotionP95, 2))],
|
||||
};
|
||||
}
|
||||
|
||||
private static int[] Where(IReadOnlyList<DeepFrame> frames, Func<DeepFrame, bool> predicate) => [.. frames.Where(predicate).Select(f => f.Index)];
|
||||
|
||||
private static int Count(double[] values)
|
||||
{
|
||||
var count = 0;
|
||||
foreach (var v in values)
|
||||
{
|
||||
if (double.IsFinite(v)) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/// <summary>Spread between two percentiles of the finite values (0 when fewer than two).</summary>
|
||||
private static double Range(double[] values, double low, double high)
|
||||
{
|
||||
if (Count(values) < 2) return 0;
|
||||
return Statistics.Percentile(values, high) - Statistics.Percentile(values, low);
|
||||
}
|
||||
|
||||
/// <summary>Rounds half away from zero; adding +0 turns −0 into 0 so JSON never spends a byte on the sign.</summary>
|
||||
private static double Round(double value, int decimals) => double.Round(value, decimals, MidpointRounding.AwayFromZero) + 0.0;
|
||||
|
||||
private static double Significant(double value, int digits)
|
||||
{
|
||||
if (value == 0) return 0;
|
||||
var magnitude = (int)double.Floor(double.Log10(double.Abs(value)));
|
||||
var decimals = digits - 1 - magnitude;
|
||||
return decimals is >= 0 and <= 15 ? Round(value, decimals) : value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Motion;
|
||||
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>
|
||||
/// The measurements of one frame that need no other frame (statistics, stars, sky/land luminance, clouds) plus the
|
||||
/// reduced planes the sequential stage of <see cref="DeepAnalyzer"/> compares with the previous frame. Frames are
|
||||
/// probed in parallel; the full analysis image is released before the probe is returned.
|
||||
/// </summary>
|
||||
internal sealed class FrameProbe : IDisposable
|
||||
{
|
||||
private FrameProbe(int index) => Index = index;
|
||||
|
||||
public int Index { get; }
|
||||
|
||||
/// <summary>Null when the frame could not be decoded.</summary>
|
||||
public FrameStatistics? Statistics { get; private set; }
|
||||
|
||||
/// <summary>Perceptual luminance at the correlation size, row-major.</summary>
|
||||
public float[]? Correlation { get; private set; }
|
||||
|
||||
public int CorrelationWidth { get; private set; }
|
||||
public int CorrelationHeight { get; private set; }
|
||||
|
||||
/// <summary>Source pixels per correlation-plane pixel.</summary>
|
||||
public double CorrelationScale { get; private set; } = 1;
|
||||
|
||||
/// <summary>Flow-sized image; only for frames that take part in a sampled flow pair.</summary>
|
||||
public ImageBuffer? Flow { get; private set; }
|
||||
|
||||
/// <summary>Source pixels per flow-image pixel.</summary>
|
||||
public double FlowScale { get; private set; } = 1;
|
||||
|
||||
public double StarDensity { get; private set; } = double.NaN;
|
||||
public double CloudCoverage { get; private set; } = double.NaN;
|
||||
public double SkyLogLuminance { get; private set; } = double.NaN;
|
||||
public double LandLogLuminance { get; private set; } = double.NaN;
|
||||
|
||||
/// <summary>True when the frame is decoded and neither black nor white (it may update the sequential state).</summary>
|
||||
public bool IsUsable => Statistics is { } s && !s.IsBlack() && !s.IsWhite();
|
||||
|
||||
/// <summary>Whether frame <paramref name="index"/> takes part in a sampled optical-flow pair (k − 1, k), k divisible by <paramref name="every"/>.</summary>
|
||||
public static bool NeedsFlow(int index, int every) => index % every == 0 || (index + 1) % every == 0;
|
||||
|
||||
public static async Task<FrameProbe> RunAsync(IFrameSource source, int index, DeepAnalysisOptions options, RegionMask? mask, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var (sourceWidth, sourceHeight) = source.Metadata(index).OrientedSize;
|
||||
var edge = options.AnalysisLongEdge(int.Max(sourceWidth, sourceHeight));
|
||||
var loaded = await source.LoadAsync(index, edge, cancellationToken).ConfigureAwait(false);
|
||||
var probe = new FrameProbe(index);
|
||||
if (loaded is null) return probe;
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
using var analysis = int.Max(loaded.Width, loaded.Height) > edge ? Reduce(loaded, edge) : loaded;
|
||||
var sourceLong = sourceWidth > 0 && sourceHeight > 0 ? int.Max(sourceWidth, sourceHeight) : int.Max(analysis.Width, analysis.Height);
|
||||
probe.Measure(analysis, sourceLong, options, mask);
|
||||
return probe;
|
||||
}
|
||||
|
||||
/// <summary>Hands the flow image over to the caller, who then disposes it.</summary>
|
||||
public ImageBuffer? TakeFlow()
|
||||
{
|
||||
var flow = Flow;
|
||||
Flow = null;
|
||||
return flow;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Flow?.Dispose();
|
||||
Flow = null;
|
||||
}
|
||||
|
||||
private static ImageBuffer Reduce(ImageBuffer image, int edge)
|
||||
{
|
||||
using (image) return AnalysisImage.Downscale(image, edge);
|
||||
}
|
||||
|
||||
private void Measure(ImageBuffer analysis, int sourceLong, DeepAnalysisOptions options, RegionMask? mask)
|
||||
{
|
||||
var stats = FrameStatistics.Measure(analysis);
|
||||
Statistics = stats;
|
||||
if (!IsUsable) return;
|
||||
|
||||
if (stats.MeanLogLuminance <= options.DarkLogLuminance) StarDensity = StarFieldDetector.Density(analysis, stats.Noise);
|
||||
|
||||
if (mask is not null)
|
||||
{
|
||||
using var thumbnail = AnalysisImage.Downscale(analysis, options.ThumbnailLongEdge);
|
||||
var fitted = mask.Resample(thumbnail.Width, thumbnail.Height);
|
||||
(SkyLogLuminance, LandLogLuminance) = RegionDeflicker.Measure(thumbnail, fitted);
|
||||
CloudCoverage = Deep.CloudCoverage.Estimate(thumbnail, fitted.Sky);
|
||||
}
|
||||
|
||||
using (var correlation = AnalysisImage.Downscale(analysis, options.CorrelationLongEdge))
|
||||
{
|
||||
CorrelationWidth = correlation.Width;
|
||||
CorrelationHeight = correlation.Height;
|
||||
CorrelationScale = (double)sourceLong / int.Max(correlation.Width, correlation.Height);
|
||||
var plane = new float[correlation.PlaneLength];
|
||||
PlaneOps.PerceptualLuminance(correlation, plane);
|
||||
Correlation = plane;
|
||||
}
|
||||
|
||||
if (NeedsFlow(Index, options.FlowEvery))
|
||||
{
|
||||
Flow = AnalysisImage.Downscale(analysis, options.FlowLongEdge);
|
||||
FlowScale = (double)sourceLong / int.Max(Flow.Width, Flow.Height);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Loads and measures the region-mask sample (thumbnails of usable frames that share the first one's size).</summary>
|
||||
public static async Task<RegionMask?> SegmentRegionsAsync(IFrameSource source, DeepAnalysisOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
var n = source.Count;
|
||||
var count = int.Min(options.RegionSampleFrames, n);
|
||||
var thumbnails = new List<ImageBuffer>(count);
|
||||
try
|
||||
{
|
||||
for (var k = 0; k < count; k++)
|
||||
{
|
||||
var index = count > 1 ? (int)((long)k * (n - 1) / (count - 1)) : 0;
|
||||
var (w, h) = source.Metadata(index).OrientedSize;
|
||||
var edge = options.AnalysisLongEdge(int.Max(w, h));
|
||||
using var loaded = await source.LoadAsync(index, edge, cancellationToken).ConfigureAwait(false);
|
||||
if (loaded is null) continue;
|
||||
using var analysis = AnalysisImage.Downscale(loaded, edge);
|
||||
var stats = FrameStatistics.Measure(analysis);
|
||||
if (stats.IsBlack() || stats.IsWhite()) continue;
|
||||
var thumbnail = AnalysisImage.Downscale(analysis, options.ThumbnailLongEdge);
|
||||
if (thumbnails.Count > 0 && (thumbnail.Width != thumbnails[0].Width || thumbnail.Height != thumbnails[0].Height) || thumbnail.Height < 8)
|
||||
{
|
||||
thumbnail.Dispose();
|
||||
continue;
|
||||
}
|
||||
thumbnails.Add(thumbnail);
|
||||
}
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
return thumbnails.Count > 0 ? RegionSegmenter.Segment(thumbnails) : null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var t in thumbnails) t.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>Scene-change measure between two <see cref="FrameStatistics.Histogram"/>s.</summary>
|
||||
internal static class HistogramDistance
|
||||
{
|
||||
private const double SmoothingBins = 2;
|
||||
private const double BinsPerEv = FrameStatistics.HistogramBins / (FrameStatistics.HistogramMaxEv - FrameStatistics.HistogramMinEv);
|
||||
|
||||
/// <summary>
|
||||
/// Symmetric chi-square distance (0 = same, 1 = disjoint) after both histograms are smoothed with a Gaussian of
|
||||
/// σ = 2 bins (⅛ EV) and the current one is shifted by the change of mean log-luminance (linear interpolation),
|
||||
/// so flicker, exposure steps and brightness ramps score near zero while a new view scores high. Mass shifted out
|
||||
/// of range counts as disjoint.
|
||||
/// </summary>
|
||||
public static double ExposureAligned(FrameStatistics previous, FrameStatistics current)
|
||||
{
|
||||
var a = Smooth(previous.Histogram);
|
||||
var b = Smooth(current.Histogram);
|
||||
var delta = current.MeanLogLuminance - previous.MeanLogLuminance;
|
||||
var shift = double.IsFinite(delta) ? delta * BinsPerEv : 0;
|
||||
double sum = 0, matched = 0, total = 0;
|
||||
foreach (var v in b) total += v;
|
||||
for (var i = 0; i < a.Length; i++)
|
||||
{
|
||||
var x = i + shift;
|
||||
var j = (int)double.Floor(x);
|
||||
var t = x - j;
|
||||
var q = (1 - t) * At(b, j) + t * At(b, j + 1);
|
||||
matched += q;
|
||||
var both = a[i] + q;
|
||||
if (both > 0) sum += (a[i] - q) * (a[i] - q) / both;
|
||||
}
|
||||
sum += double.Max(0, total - matched);
|
||||
return 0.5 * sum;
|
||||
}
|
||||
|
||||
private static double At(double[] values, int index) => index >= 0 && index < values.Length ? values[index] : 0;
|
||||
|
||||
/// <summary>Gaussian smoothing that keeps the total mass (kernel renormalised at the edges).</summary>
|
||||
private static double[] Smooth(double[] histogram)
|
||||
{
|
||||
var radius = (int)double.Ceiling(3 * SmoothingBins);
|
||||
Span<double> kernel = stackalloc double[2 * radius + 1];
|
||||
for (var d = -radius; d <= radius; d++) kernel[d + radius] = double.Exp(-0.5 * d * d / (SmoothingBins * SmoothingBins));
|
||||
var result = new double[histogram.Length];
|
||||
for (var i = 0; i < histogram.Length; i++)
|
||||
{
|
||||
if (histogram[i] == 0) continue;
|
||||
double weight = 0;
|
||||
for (var d = -radius; d <= radius; d++)
|
||||
{
|
||||
if (i + d >= 0 && i + d < histogram.Length) weight += kernel[d + radius];
|
||||
}
|
||||
for (var d = -radius; d <= radius; d++)
|
||||
{
|
||||
if (i + d >= 0 && i + d < histogram.Length) result[i + d] += histogram[i] * kernel[d + radius] / weight;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using Titano.Core.Imaging;
|
||||
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>
|
||||
/// The frames of one sequence as <see cref="DeepAnalyzer"/> reads them. Decoding is the implementer's job (archive
|
||||
/// decoder on the client, proxies on the server, <c>SyntheticSequence</c> in tests).
|
||||
/// </summary>
|
||||
public interface IFrameSource
|
||||
{
|
||||
int Count { get; }
|
||||
|
||||
/// <summary>Metadata of frame <paramref name="index"/> (capture time, exposure, size, GPS); cheap, no decoding.</summary>
|
||||
LightFrame Metadata(int index);
|
||||
|
||||
/// <summary>
|
||||
/// Decodes frame <paramref name="index"/> in linear light, oriented, with its long edge at most
|
||||
/// <paramref name="maxLongEdge"/> (a larger result is downscaled by the caller). The caller owns and disposes the
|
||||
/// buffer. Null when the frame cannot be decoded.
|
||||
/// </summary>
|
||||
ValueTask<ImageBuffer?> LoadAsync(int index, int maxLongEdge, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Progress of a deep analysis: frames done (including resumed ones), throughput of this run and its ETA.</summary>
|
||||
public readonly record struct AnalysisProgress(int Done, int Total, double FramesPerSecond, TimeSpan? Eta);
|
||||
@@ -0,0 +1,57 @@
|
||||
using Titano.Core.Imaging;
|
||||
|
||||
namespace Titano.Core.Analysis.Deep;
|
||||
|
||||
/// <summary>
|
||||
/// Star-field detection (BRIEF §10.2): counts small bright points on sRGB-encoded luminance. A point is a strict
|
||||
/// 3×3 local maximum that rises above the mean of the 5×5 ring around it by at least
|
||||
/// max(<see cref="NoiseFactor"/>·σ, <see cref="MinimumContrast"/>), whose 8 neighbours average less than half that
|
||||
/// rise and whose ring stays below half of it everywhere (point-like: lit windows, the moon and textured land fail;
|
||||
/// stars up to 2×2 pixels pass).
|
||||
/// </summary>
|
||||
public static class StarFieldDetector
|
||||
{
|
||||
public const double NoiseFactor = 6;
|
||||
public const double MinimumContrast = 0.05;
|
||||
|
||||
/// <summary>Point sources per megapixel of <paramref name="image"/>; <paramref name="noiseSigma"/> is the encoded-luminance
|
||||
/// noise σ (e.g. <see cref="FrameStatistics.Noise"/>).</summary>
|
||||
public static double Density(ImageBuffer image, double noiseSigma)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(image);
|
||||
int w = image.Width, h = image.Height;
|
||||
if (w < 5 || h < 5) return 0;
|
||||
var e = new float[image.PlaneLength];
|
||||
ColorSpace.Luminance(image, e);
|
||||
for (var i = 0; i < e.Length; i++) e[i] = ColorSpace.EncodeFast(e[i]);
|
||||
var threshold = double.Max(NoiseFactor * (double.IsFinite(noiseSigma) ? noiseSigma : 0), MinimumContrast);
|
||||
|
||||
var count = 0;
|
||||
for (var y = 2; y < h - 2; y++)
|
||||
{
|
||||
for (var x = 2; x < w - 2; x++)
|
||||
{
|
||||
var i = y * w + x;
|
||||
var c = e[i];
|
||||
// Ties go to the later pixel in raster order so a plateau counts once.
|
||||
if (!(c >= e[i - w - 1] && c >= e[i - w] && c >= e[i - w + 1] && c >= e[i - 1]
|
||||
&& c > e[i + 1] && c > e[i + w - 1] && c > e[i + w] && c > e[i + w + 1])) continue;
|
||||
double ring = 0, ringMax = 0;
|
||||
for (var d = -2; d <= 2; d++)
|
||||
{
|
||||
ring += e[i - 2 * w + d] + e[i + 2 * w + d];
|
||||
ringMax = double.Max(ringMax, double.Max(e[i - 2 * w + d], e[i + 2 * w + d]));
|
||||
if (d is <= -2 or >= 2) continue;
|
||||
ring += e[i + d * w - 2] + e[i + d * w + 2];
|
||||
ringMax = double.Max(ringMax, double.Max(e[i + d * w - 2], e[i + d * w + 2]));
|
||||
}
|
||||
var background = ring / 16;
|
||||
var rise = c - background;
|
||||
if (rise < threshold || ringMax - background >= 0.5 * rise) continue;
|
||||
double inner = e[i - w - 1] + e[i - w] + e[i - w + 1] + e[i - 1] + e[i + 1] + e[i + w - 1] + e[i + w] + e[i + w + 1];
|
||||
if (inner / 8 - background <= 0.5 * rise) count++;
|
||||
}
|
||||
}
|
||||
return count * 1e6 / ((double)w * h);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
using Titano.Core.Analysis;
|
||||
using Titano.Core.Analysis.Deep;
|
||||
using Titano.Core.Diagnostics.Synthetic;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Core.Tests.Analysis.Deep;
|
||||
|
||||
public sealed class DeepAnalyzerTests(TestContext context)
|
||||
{
|
||||
[Test]
|
||||
public async Task NightSkyShowsStarsDarknessFlickerAndNoShake()
|
||||
{
|
||||
var report = await DeepReports.Night;
|
||||
context.Log($"{report.Sky} {report.Flicker} {report.Shake}");
|
||||
Assert.True(report.Sky.StarField);
|
||||
Assert.Equal(DayNight.Night, report.Sky.Light);
|
||||
Assert.Equal(report.FrameCount, report.Sky.StarFrames);
|
||||
// Uniform ±0.12 EV flicker has an RMS of 0.069 EV.
|
||||
Assert.Approximately(0.069, report.Flicker.AmplitudeEv, 0.01);
|
||||
Assert.Equal(15, report.Flicker.RecommendedWindow);
|
||||
Assert.Less(report.Shake.RmsPx, 0.05);
|
||||
Assert.Equal(1.0, report.Shake.MeasuredFraction);
|
||||
Assert.Empty(report.Issues.SceneChanges);
|
||||
Assert.Equal(0.0, report.Motion.P95Px!.Value);
|
||||
Assert.Equal(RegionMethod.Horizon, report.Regions!.Method);
|
||||
Assert.Contains(report.Findings, f => f.Key == DeepFindingKeys.StarField);
|
||||
Assert.Contains(report.Findings, f => f.Key == DeepFindingKeys.Flicker);
|
||||
Assert.True(report.Frames.FlagsOf(0).HasFlag(DeepFrameFlags.StarField | DeepFrameFlags.Dark));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task InjectedCameraJitterIsMeasuredAsShake()
|
||||
{
|
||||
var report = await DeepReports.NightJitter;
|
||||
context.Log($"{report.Shake} {report.Motion}");
|
||||
// Integer offsets uniform in −2..2 per axis: 2 px RMS in 2D.
|
||||
Assert.InRange(report.Shake.RmsPx, 1.6, 2.4);
|
||||
Assert.InRange(report.Shake.CropCost, 0.02, 0.2);
|
||||
Assert.False(report.Shake.PanDetected);
|
||||
Assert.Greater(report.Motion.P95Px!.Value, 1);
|
||||
Assert.Contains(report.Findings, f => f.Key == DeepFindingKeys.Shake);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task HolyGrailStepsAreReceivedAndSpacedByHalfTheirDistance()
|
||||
{
|
||||
var report = await DeepReports.HolyGrail;
|
||||
Assert.Equal(5, report.Exposure.MetadataSteps);
|
||||
Assert.Equal(5, report.Exposure.ReceivedSteps);
|
||||
Assert.Equal(40.0, report.Exposure.TypicalSpacing);
|
||||
Assert.Equal(20, report.Exposure.TransitionLength);
|
||||
Assert.True(report.Exposure.Steps.All(s => s.MeasuredEv is > 0.9 and < 1.1));
|
||||
Assert.InRange(report.Exposure.SceneRangeEv, 5, 6);
|
||||
Assert.Equal(DayNight.DayToNight, report.Sky.Light);
|
||||
// Brightness ramps and steps are levelled before the flow, so the static scene shows no motion.
|
||||
Assert.Equal(0.0, report.Motion.P95Px!.Value);
|
||||
Assert.Empty(report.Issues.SceneChanges);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[Case(8.0, 33)]
|
||||
[Case(12.5, 51)]
|
||||
[Case(40.0, 151)]
|
||||
public async Task PeriodicFlickerGivesAWindowOfFourPeriods(double period, int window)
|
||||
{
|
||||
var sequence = SyntheticSequence.Create(new SyntheticSequenceOptions
|
||||
{
|
||||
Width = 96, Height = 64, FrameCount = 240, FlickerEv = 0.15, FlickerPeriodFrames = period,
|
||||
});
|
||||
var report = await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence), DeepScenes.Options);
|
||||
Assert.Approximately(period, report.Flicker.PeriodFrames!.Value, 0.05 * period);
|
||||
Assert.Equal(window, report.Flicker.RecommendedWindow);
|
||||
Assert.Equal(1, window % 2);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task BlackWhiteDuplicateAndUnreadableFramesAreFlagged()
|
||||
{
|
||||
var sequence = SyntheticSequence.Create(new SyntheticSequenceOptions
|
||||
{
|
||||
Width = 96, Height = 64, FrameCount = 60, FlickerEv = 0.02, BlackFrames = [10], WhiteFrames = [20], DuplicateFrames = [30],
|
||||
});
|
||||
var report = await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence) { Undecodable = [40] }, DeepScenes.Options);
|
||||
Assert.SequenceEqual([10], report.Issues.Black);
|
||||
Assert.SequenceEqual([20], report.Issues.White);
|
||||
Assert.SequenceEqual([30], report.Issues.Duplicate);
|
||||
Assert.SequenceEqual([40], report.Issues.Undecodable);
|
||||
Assert.SequenceEqual([10, 20, 30, 40], report.Issues.Excludable());
|
||||
Assert.Empty(report.Issues.SceneChanges);
|
||||
Assert.Null(report.Frames.LogLuminance[40]);
|
||||
Assert.True(report.Frames.FlagsOf(40).HasFlag(DeepFrameFlags.Undecodable));
|
||||
Assert.False(report.Issues.Excludable().Contains(report.HeroIndex!.Value));
|
||||
foreach (var key in new[] { DeepFindingKeys.BlackFrames, DeepFindingKeys.WhiteFrames, DeepFindingKeys.DuplicateFrames, DeepFindingKeys.UnreadableFrames })
|
||||
Assert.Equal(FindingSeverity.Warning, report.Findings.Single(f => f.Key == key).Severity);
|
||||
Assert.Equal(4, AnalysisSummary.From(report).WarningCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACutToAnotherViewIsASceneChange()
|
||||
{
|
||||
var first = new SyntheticFrameSource(SyntheticSequence.Create(new SyntheticSequenceOptions { Width = 96, Height = 64, FrameCount = 50, FlickerEv = 0.05 }));
|
||||
var second = new SyntheticFrameSource(SyntheticSequence.Create(new SyntheticSequenceOptions
|
||||
{
|
||||
Width = 96, Height = 64, FrameCount = 50, FlickerEv = 0.05, Seed = 5, HorizonFraction = 0.75, SkyLevel = 0.6, LandLevel = 0.01,
|
||||
}));
|
||||
var report = await DeepAnalyzer.AnalyzeAsync(new SplicedSource(first, second, 30), DeepScenes.Options);
|
||||
Assert.SequenceEqual([30], report.Issues.SceneChanges);
|
||||
Assert.True(report.Frames.FlagsOf(30).HasFlag(DeepFrameFlags.SceneChange));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ResumingFromACheckpointGivesTheSameReport()
|
||||
{
|
||||
var expected = (await DeepReports.HolyGrail).ToJson();
|
||||
var checkpoint = new MemoryCheckpoint();
|
||||
var crashing = new SyntheticFrameSource(DeepScenes.HolyGrail())
|
||||
{
|
||||
OnLoad = i =>
|
||||
{
|
||||
if (i == 101) throw new IOException("simulated crash");
|
||||
},
|
||||
};
|
||||
await Assert.ThrowsAsync<IOException>(() => DeepAnalyzer.AnalyzeAsync(crashing, DeepScenes.Options, checkpoint));
|
||||
var persisted = checkpoint.Frames.Count;
|
||||
Assert.InRange(persisted, 90, 101);
|
||||
|
||||
var resumed = new SyntheticFrameSource(DeepScenes.HolyGrail());
|
||||
var report = await DeepAnalyzer.AnalyzeAsync(resumed, DeepScenes.Options, checkpoint);
|
||||
Assert.Equal(expected, report.ToJson());
|
||||
// Region sample + one priming frame + the frames not yet persisted.
|
||||
Assert.Equal(DeepScenes.Options.RegionSampleFrames + 1 + (240 - persisted), resumed.Loads);
|
||||
Assert.Equal(240, checkpoint.Frames.Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FileCheckpointDropsATornLineAndResumes()
|
||||
{
|
||||
var path = Path.Combine(context.CreateTempDirectory(), ".titano", DeepAnalysisCheckpointFile.FileName);
|
||||
var sequence = SyntheticSequence.Create(new SyntheticSequenceOptions { Width = 96, Height = 64, FrameCount = 40, FlickerEv = 0.05, DuplicateFrames = [7] });
|
||||
var expected = (await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence), DeepScenes.Options)).ToJson();
|
||||
var file = new DeepAnalysisCheckpointFile(path);
|
||||
var crashing = new SyntheticFrameSource(sequence) { OnLoad = i => { if (i == 21) throw new IOException("crash"); } };
|
||||
await Assert.ThrowsAsync<IOException>(() => DeepAnalyzer.AnalyzeAsync(crashing, DeepScenes.Options, file));
|
||||
await File.AppendAllTextAsync(path, "{\"index\":20,\"flags\":\"No", context.Cancellation);
|
||||
|
||||
var report = await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence), DeepScenes.Options, file);
|
||||
Assert.Equal(expected, report.ToJson());
|
||||
Assert.Equal(41, File.ReadAllLines(path).Length);
|
||||
|
||||
// Other options: the stale checkpoint is discarded instead of mixed in.
|
||||
var other = DeepScenes.Options with { FlowEvery = 3 };
|
||||
var fresh = await new DeepAnalysisCheckpointFile(path).LoadAsync(DeepAnalyzer.Fingerprint(40, other), context.Cancellation);
|
||||
Assert.Empty(fresh);
|
||||
file.Delete();
|
||||
Assert.FileMissing(path);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ReportRoundTripsThroughJsonWithinTheSizeBudget()
|
||||
{
|
||||
var report = await DeepReports.NightJitter;
|
||||
var json = report.ToJson();
|
||||
var copy = DeepAnalysisReport.FromJson(json);
|
||||
Assert.Equal(json, copy.ToJson());
|
||||
var perFrame = (double)System.Text.Encoding.UTF8.GetByteCount(json) / report.FrameCount;
|
||||
context.Log($"{perFrame:0.0} bytes per frame");
|
||||
Assert.LessOrEqual(perFrame, 120);
|
||||
|
||||
var path = Path.Combine(context.CreateTempDirectory(), DeepAnalysisReport.FileName);
|
||||
report.Save(path);
|
||||
Assert.Equal(json, DeepAnalysisReport.Load(path).ToJson());
|
||||
|
||||
var summary = AnalysisSummary.From(report);
|
||||
Assert.Equal(summary, AnalysisSummary.FromJson(summary.ToJson()));
|
||||
Assert.True(summary.StarField);
|
||||
Assert.Equal(DayNight.Night, summary.Light);
|
||||
Assert.Equal(report.HeroIndex, summary.HeroIndex);
|
||||
Assert.Less(summary.ToJson().Length, 400);
|
||||
Assert.Throws<InvalidDataException>(() => DeepAnalysisReport.FromJson(json.Replace("\"version\":1", "\"version\":99", StringComparison.Ordinal)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ResultsDoNotDependOnParallelism()
|
||||
{
|
||||
var sequence = DeepScenes.NightSky(frames: 40, jitter: 2);
|
||||
var serial = await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence, 2), DeepScenes.Options with { MaxDegreeOfParallelism = 1 });
|
||||
var parallel = await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence, 2), DeepScenes.Options with { MaxDegreeOfParallelism = 3 });
|
||||
Assert.Equal(serial.ToJson(), parallel.ToJson());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task CancellationStopsWithinOneBatch()
|
||||
{
|
||||
using var cancellation = new CancellationTokenSource();
|
||||
var checkpoint = new MemoryCheckpoint();
|
||||
var source = new SyntheticFrameSource(DeepScenes.HolyGrail())
|
||||
{
|
||||
OnLoad = i =>
|
||||
{
|
||||
if (i == 50) cancellation.Cancel();
|
||||
},
|
||||
};
|
||||
await Assert.ThrowsAsync<OperationCanceledException>(() => DeepAnalyzer.AnalyzeAsync(source, DeepScenes.Options, checkpoint, null, cancellation.Token));
|
||||
Assert.LessOrEqual(source.Loads, DeepScenes.Options.RegionSampleFrames + 52);
|
||||
Assert.Equal(50, checkpoint.Frames.Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ProgressCountsEveryFrameUpToTheTotal()
|
||||
{
|
||||
var sequence = SyntheticSequence.Create(new SyntheticSequenceOptions { Width = 64, Height = 48, FrameCount = 21 });
|
||||
var reports = new List<AnalysisProgress>();
|
||||
await DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(sequence), DeepScenes.Options, progress: new SyncProgress(reports.Add));
|
||||
Assert.Equal(11, reports.Count);
|
||||
Assert.Monotonic(reports.Select(r => (double)r.Done), strict: true);
|
||||
Assert.Equal(21, reports[^1].Done);
|
||||
Assert.True(reports.All(r => r.Total == 21 && r.FramesPerSecond > 0));
|
||||
Assert.Equal(TimeSpan.Zero, reports[^1].Eta);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void StarDetectorCountsPointsButNotBlobs()
|
||||
{
|
||||
using var image = new ImageBuffer(128, 96, 3);
|
||||
image.Data.Fill(1f / 2048);
|
||||
var random = new Random(3);
|
||||
var stars = 0;
|
||||
for (var y = 6; y < 90; y += 12)
|
||||
{
|
||||
for (var x = 6; x < 122; x += 12)
|
||||
{
|
||||
var v = (float)(0.2 + 0.8 * random.NextDouble());
|
||||
for (var c = 0; c < 3; c++) image[c, x, y] = v;
|
||||
stars++;
|
||||
}
|
||||
}
|
||||
// A lit window: 4×4 bright block (extended, not a star).
|
||||
for (var y = 40; y < 44; y++)
|
||||
{
|
||||
for (var x = 60; x < 64; x++)
|
||||
{
|
||||
for (var c = 0; c < 3; c++) image[c, x, y] = 0.8f;
|
||||
}
|
||||
}
|
||||
var density = StarFieldDetector.Density(image, noiseSigma: 0);
|
||||
Assert.Approximately(stars * 1e6 / (128 * 96), density, 1e-6);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CloudCoverageCountsTexturedSkyBlocks()
|
||||
{
|
||||
using var image = new ImageBuffer(128, 64, 3);
|
||||
var sky = new float[image.PlaneLength];
|
||||
var random = new Random(9);
|
||||
for (var y = 0; y < 64; y++)
|
||||
{
|
||||
for (var x = 0; x < 128; x++)
|
||||
{
|
||||
var i = y * 128 + x;
|
||||
sky[i] = y < 32 ? 1 : 0;
|
||||
// Smooth gradient everywhere; cloud texture on the left half of the sky.
|
||||
var v = 0.2 + 0.002 * y + (y < 32 && x < 64 ? 0.15 * random.NextDouble() : 0);
|
||||
for (var c = 0; c < 3; c++) image[c, x, y] = (float)v;
|
||||
}
|
||||
}
|
||||
Assert.Approximately(0.5, CloudCoverage.Estimate(image, sky), 1e-9);
|
||||
Assert.True(double.IsNaN(CloudCoverage.Estimate(image, new float[image.PlaneLength])));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void HistogramDistanceIgnoresExposureButNotContent()
|
||||
{
|
||||
using var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { Width = 96, Height = 64 }).Render(0);
|
||||
using var brighter = scene.Clone();
|
||||
foreach (ref var v in brighter.Data) v *= 1.7f;
|
||||
using var other = SyntheticSequence.Create(new SyntheticSequenceOptions { Width = 96, Height = 64, HorizonFraction = 0.8, SkyLevel = 0.6 }).Render(0);
|
||||
var a = FrameStatistics.Measure(scene);
|
||||
Assert.Less(HistogramDistance.ExposureAligned(a, FrameStatistics.Measure(brighter)), 0.02);
|
||||
Assert.Greater(HistogramDistance.ExposureAligned(a, FrameStatistics.Measure(other)), 0.3);
|
||||
}
|
||||
|
||||
/// <summary>Frames of <paramref name="first"/> before <paramref name="cut"/>, then frames of <paramref name="second"/>.</summary>
|
||||
private sealed class SplicedSource(IFrameSource first, IFrameSource second, int cut) : IFrameSource
|
||||
{
|
||||
public int Count => first.Count;
|
||||
|
||||
public LightFrame Metadata(int index) => first.Metadata(index);
|
||||
|
||||
public ValueTask<ImageBuffer?> LoadAsync(int index, int maxLongEdge, CancellationToken cancellationToken) =>
|
||||
(index < cut ? first : second).LoadAsync(index, maxLongEdge, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Reports synchronously (Progress<T> posts to the thread pool and may reorder).</summary>
|
||||
private sealed class SyncProgress(Action<AnalysisProgress> report) : IProgress<AnalysisProgress>
|
||||
{
|
||||
public void Report(AnalysisProgress value) => report(value);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Deep reports of the shared scenes, computed once per test run.</summary>
|
||||
internal static class DeepReports
|
||||
{
|
||||
private static readonly Lazy<Task<DeepAnalysisReport>> NightReport = new(() => DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(DeepScenes.NightSky()), DeepScenes.Options));
|
||||
private static readonly Lazy<Task<DeepAnalysisReport>> NightJitterReport = new(() => DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(DeepScenes.NightSky(jitter: 2), 2), DeepScenes.Options));
|
||||
private static readonly Lazy<Task<DeepAnalysisReport>> HolyGrailReport = new(() => DeepAnalyzer.AnalyzeAsync(new SyntheticFrameSource(DeepScenes.HolyGrail()), DeepScenes.Options));
|
||||
|
||||
public static Task<DeepAnalysisReport> Night => NightReport.Value;
|
||||
public static Task<DeepAnalysisReport> NightJitter => NightJitterReport.Value;
|
||||
public static Task<DeepAnalysisReport> HolyGrail => HolyGrailReport.Value;
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
using Titano.Core.Analysis;
|
||||
using Titano.Core.Analysis.Deep;
|
||||
using Titano.Core.Diagnostics.Synthetic;
|
||||
using Titano.Core.Imaging;
|
||||
|
||||
namespace Titano.Core.Tests.Analysis.Deep;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="IFrameSource"/> over a <see cref="SyntheticSequence"/>. With <paramref name="jitter"/> > 0 the sequence is
|
||||
/// rendered <paramref name="jitter"/> pixels larger on every side and each frame is cropped at a seeded integer offset
|
||||
/// in ±jitter, which shakes the camera by a known amount.
|
||||
/// </summary>
|
||||
internal sealed class SyntheticFrameSource(SyntheticSequence sequence, int jitter = 0, int seed = 11) : IFrameSource
|
||||
{
|
||||
private readonly (int X, int Y)[] _offsets = Offsets(sequence.Count, jitter, seed);
|
||||
private int _loads;
|
||||
|
||||
public SyntheticSequence Sequence { get; } = sequence;
|
||||
|
||||
public int Width => Sequence.Options.Width - 2 * jitter;
|
||||
public int Height => Sequence.Options.Height - 2 * jitter;
|
||||
public int Count => Sequence.Count;
|
||||
|
||||
/// <summary>Total calls of <see cref="LoadAsync"/>.</summary>
|
||||
public int Loads => Volatile.Read(ref _loads);
|
||||
|
||||
/// <summary>Frames that "fail to decode".</summary>
|
||||
public HashSet<int> Undecodable { get; init; } = [];
|
||||
|
||||
/// <summary>Called on every load before decoding (may throw to simulate a crash, or cancel a token).</summary>
|
||||
public Action<int>? OnLoad { get; set; }
|
||||
|
||||
/// <summary>Metadata without shutter time and aperture when true.</summary>
|
||||
public bool StripExposure { get; init; }
|
||||
|
||||
public LightFrame Metadata(int index)
|
||||
{
|
||||
var frame = LightFrame.From(Sequence.Metadata[index], index) with { Width = Width, Height = Height };
|
||||
return StripExposure ? frame with { ExposureTime = null, FNumber = null } : frame;
|
||||
}
|
||||
|
||||
public ValueTask<ImageBuffer?> LoadAsync(int index, int maxLongEdge, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
Interlocked.Increment(ref _loads);
|
||||
OnLoad?.Invoke(index);
|
||||
if (Undecodable.Contains(index)) return ValueTask.FromResult<ImageBuffer?>(null);
|
||||
using var full = Sequence.Render(index);
|
||||
var (dx, dy) = _offsets[index];
|
||||
var image = new ImageBuffer(Width, Height, 3, clear: false);
|
||||
for (var c = 0; c < 3; c++)
|
||||
{
|
||||
for (var y = 0; y < Height; y++)
|
||||
full.Row(c, y + jitter + dy).Slice(jitter + dx, Width).CopyTo(image.Row(c, y));
|
||||
}
|
||||
if (int.Max(Width, Height) <= maxLongEdge) return ValueTask.FromResult<ImageBuffer?>(image);
|
||||
using (image) return ValueTask.FromResult<ImageBuffer?>(AnalysisImage.Downscale(image, maxLongEdge));
|
||||
}
|
||||
|
||||
private static (int, int)[] Offsets(int count, int jitter, int seed)
|
||||
{
|
||||
var random = new Random(seed);
|
||||
var offsets = new (int, int)[count];
|
||||
for (var i = 0; i < count; i++) offsets[i] = jitter > 0 ? (random.Next(-jitter, jitter + 1), random.Next(-jitter, jitter + 1)) : (0, 0);
|
||||
return offsets;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>In-memory checkpoint that records what was persisted.</summary>
|
||||
internal sealed class MemoryCheckpoint : IDeepAnalysisCheckpoint
|
||||
{
|
||||
private string? _fingerprint;
|
||||
|
||||
public List<DeepFrame> Frames { get; } = [];
|
||||
|
||||
public ValueTask<IReadOnlyList<DeepFrame>> LoadAsync(string fingerprint, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_fingerprint != fingerprint) Frames.Clear();
|
||||
_fingerprint = fingerprint;
|
||||
return ValueTask.FromResult<IReadOnlyList<DeepFrame>>([.. Frames]);
|
||||
}
|
||||
|
||||
public ValueTask AppendAsync(IReadOnlyList<DeepFrame> frames, CancellationToken cancellationToken)
|
||||
{
|
||||
Frames.AddRange(frames);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The synthetic scenes the deep-analysis and advisor tests share (small frames, fast).</summary>
|
||||
internal static class DeepScenes
|
||||
{
|
||||
/// <summary>Options for the tests: small sources are analysed at native size, flow on every 5th pair, two workers.</summary>
|
||||
public static DeepAnalysisOptions Options { get; } = new() { MinAnalysisLongEdge = 64, FlowEvery = 5, MaxDegreeOfParallelism = 2, RegionSampleFrames = 9 };
|
||||
|
||||
/// <summary>Night sky with stars over dark land and random flicker.</summary>
|
||||
public static SyntheticSequence NightSky(int frames = 90, int jitter = 0) => SyntheticSequence.Create(new SyntheticSequenceOptions
|
||||
{
|
||||
Width = 160 + 2 * jitter,
|
||||
Height = 112 + 2 * jitter,
|
||||
FrameCount = frames,
|
||||
IntervalSeconds = 20,
|
||||
BaseExposureTime = 15,
|
||||
BaseFNumber = 2.8,
|
||||
BaseIso = 3200,
|
||||
StarField = true,
|
||||
StarCount = 160,
|
||||
FlickerEv = 0.12,
|
||||
Start = new DateTime(2024, 8, 12, 23, 0, 0),
|
||||
});
|
||||
|
||||
/// <summary>Sunset: the scene darkens 6 EV while the camera opens up 1 EV every 40 frames.</summary>
|
||||
public static SyntheticSequence HolyGrail(int frames = 240) => SyntheticSequence.Create(new SyntheticSequenceOptions
|
||||
{
|
||||
Width = 160,
|
||||
Height = 112,
|
||||
FrameCount = frames,
|
||||
IntervalSeconds = 15,
|
||||
BaseExposureTime = 1.0 / 250,
|
||||
StartEv = 0.5,
|
||||
EndEv = -5.5,
|
||||
FlickerEv = 0.03,
|
||||
ExposureSteps = [new(40, -1), new(80, -1), new(120, -1), new(160, -1), new(200, -1)],
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user