Median-based Immerkaer noise ignores stars and edges, so star detection and the noise column no longer read star fields as noise; motion statistics keep only vectors that pass the forward-backward check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
233 lines
11 KiB
C#
233 lines
11 KiB
C#
using System.Diagnostics;
|
||
using Titano.Core.Imaging;
|
||
using Titano.Core.Motion;
|
||
|
||
namespace Titano.Core.Analysis.Deep;
|
||
|
||
/// <summary>
|
||
/// Deep analysis (BRIEF §10.2): one streaming pass over every frame. A pre-pass segments sky and land on a sample of
|
||
/// thumbnails; then frames are probed in parallel batches (<see cref="FrameProbe"/>: statistics, stars, region
|
||
/// luminance, clouds) and compared in order with the previous usable frame (nine-patch phase-correlation shift, duplicate
|
||
/// test, exposure-aligned histogram distance, Draft optical flow on every <see cref="DeepAnalysisOptions.FlowEvery"/>-th
|
||
/// pair). Memory is one batch of analysis images plus the previous frame's reduced planes. After every batch the new
|
||
/// <see cref="DeepFrame"/>s go to the checkpoint, so an interrupted run resumes after the last persisted frame with
|
||
/// identical results. Sequence-level results are computed by <see cref="DeepSequenceAnalysis"/>.
|
||
/// </summary>
|
||
public static class DeepAnalyzer
|
||
{
|
||
public static async Task<DeepAnalysisReport> AnalyzeAsync(
|
||
IFrameSource source,
|
||
DeepAnalysisOptions? options = null,
|
||
IDeepAnalysisCheckpoint? checkpoint = null,
|
||
IProgress<AnalysisProgress>? progress = null,
|
||
CancellationToken cancellationToken = default)
|
||
{
|
||
ArgumentNullException.ThrowIfNull(source);
|
||
options ??= DeepAnalysisOptions.Default;
|
||
options.Validate();
|
||
var n = source.Count;
|
||
var metadata = new LightFrame[n];
|
||
for (var i = 0; i < n; i++) metadata[i] = source.Metadata(i);
|
||
|
||
var frames = new DeepFrame[n];
|
||
var start = 0;
|
||
if (checkpoint is not null)
|
||
{
|
||
var saved = await checkpoint.LoadAsync(Fingerprint(n, options), cancellationToken).ConfigureAwait(false);
|
||
while (start < saved.Count && start < n && saved[start].Index == start) frames[start] = saved[start++];
|
||
}
|
||
|
||
var mask = n > 0 ? await FrameProbe.SegmentRegionsAsync(source, options, cancellationToken).ConfigureAwait(false) : null;
|
||
using var state = new SequentialState(options);
|
||
await PrimeAsync(source, frames, start, options, mask, state, cancellationToken).ConfigureAwait(false);
|
||
|
||
var clock = Stopwatch.StartNew();
|
||
var batch = options.MaxDegreeOfParallelism;
|
||
for (var first = start; first < n; first += batch)
|
||
{
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
var count = int.Min(batch, n - first);
|
||
var tasks = new Task<FrameProbe>[count];
|
||
for (var k = 0; k < count; k++)
|
||
{
|
||
var index = first + k;
|
||
tasks[k] = Task.Run(() => FrameProbe.RunAsync(source, index, options, mask, cancellationToken), cancellationToken);
|
||
}
|
||
var produced = new DeepFrame[count];
|
||
try
|
||
{
|
||
await Task.WhenAll(tasks).ConfigureAwait(false);
|
||
for (var k = 0; k < count; k++) produced[k] = frames[first + k] = state.Next(tasks[k].Result, cancellationToken);
|
||
}
|
||
finally
|
||
{
|
||
foreach (var task in tasks)
|
||
{
|
||
if (task.IsCompletedSuccessfully) task.Result.Dispose();
|
||
}
|
||
}
|
||
if (checkpoint is not null) await checkpoint.AppendAsync(produced, cancellationToken).ConfigureAwait(false);
|
||
if (progress is not null)
|
||
{
|
||
var done = first + count;
|
||
var rate = (done - start) / double.Max(clock.Elapsed.TotalSeconds, 1e-3);
|
||
progress.Report(new AnalysisProgress(done, n, rate, TimeSpan.FromSeconds((n - done) / rate)));
|
||
}
|
||
}
|
||
|
||
return DeepSequenceAnalysis.Build(frames, metadata, mask, options);
|
||
}
|
||
|
||
/// <summary>Identifies checkpoints written by this version for this frame count and these options.</summary>
|
||
public static string Fingerprint(int frameCount, DeepAnalysisOptions options)
|
||
{
|
||
ArgumentNullException.ThrowIfNull(options);
|
||
return $"titano-deep/{DeepAnalysisReport.CurrentVersion};frames={frameCount};{options.Fingerprint()}";
|
||
}
|
||
|
||
/// <summary>Restores the comparison state of a resumed run from the last usable frame before <paramref name="start"/>.</summary>
|
||
private static async Task PrimeAsync(IFrameSource source, DeepFrame[] frames, int start, DeepAnalysisOptions options, RegionMask? mask, SequentialState state, CancellationToken cancellationToken)
|
||
{
|
||
var last = start - 1;
|
||
while (last >= 0 && !frames[last].IsUsable) last--;
|
||
if (last < 0) return;
|
||
using var probe = await FrameProbe.RunAsync(source, last, options, mask, cancellationToken).ConfigureAwait(false);
|
||
state.Next(probe, cancellationToken);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The order-dependent comparisons: each usable frame is compared with the previous usable frame and becomes the
|
||
/// new reference; undecodable, black and white frames leave the state untouched.
|
||
/// </summary>
|
||
private sealed class SequentialState(DeepAnalysisOptions options) : IDisposable
|
||
{
|
||
private const int MinimumStabilizerSize = 16;
|
||
private const float ReliableError = 0.5f;
|
||
private const double MinimumReliable = 0.05;
|
||
private readonly OpticalFlowEngine _flow = new(FlowQuality.Draft);
|
||
private Stabilizer? _stabilizer;
|
||
private FrameStatistics? _previous;
|
||
private ImageBuffer? _previousFlow;
|
||
private int _previousIndex = -1;
|
||
|
||
public DeepFrame Next(FrameProbe probe, CancellationToken cancellationToken)
|
||
{
|
||
if (probe.Statistics is not { } stats) return new DeepFrame { Index = probe.Index, Flags = DeepFrameFlags.Undecodable };
|
||
var frame = Measured(probe.Index, stats);
|
||
if (!probe.IsUsable) return frame;
|
||
|
||
var (shiftX, shiftY) = Shift(probe);
|
||
var duplicate = _previous is not null && FrameStatistics.AreLikelyDuplicates(_previous, stats);
|
||
var scene = _previous is null ? double.NaN : HistogramDistance.ExposureAligned(_previous, stats);
|
||
var (motionMedian, motionP95) = Motion(probe, stats, cancellationToken);
|
||
frame = frame with
|
||
{
|
||
Flags = frame.Flags | (duplicate ? DeepFrameFlags.Duplicate : DeepFrameFlags.None),
|
||
Noise = probe.Noise,
|
||
ShiftX = shiftX,
|
||
ShiftY = shiftY,
|
||
MotionMedian = motionMedian,
|
||
MotionP95 = motionP95,
|
||
StarDensity = probe.StarDensity,
|
||
CloudCoverage = probe.CloudCoverage,
|
||
SkyLogLuminance = probe.SkyLogLuminance,
|
||
LandLogLuminance = probe.LandLogLuminance,
|
||
SceneDistance = scene,
|
||
};
|
||
|
||
_previous = stats;
|
||
_previousIndex = probe.Index;
|
||
_previousFlow?.Dispose();
|
||
_previousFlow = null;
|
||
_previousFlow = probe.TakeFlow();
|
||
return frame;
|
||
}
|
||
|
||
public void Dispose()
|
||
{
|
||
_flow.Dispose();
|
||
_stabilizer?.Dispose();
|
||
_previousFlow?.Dispose();
|
||
}
|
||
|
||
private DeepFrame Measured(int index, FrameStatistics stats)
|
||
{
|
||
var flags = DeepFrameFlags.None;
|
||
if (stats.IsBlack()) flags |= DeepFrameFlags.Black;
|
||
if (stats.IsWhite()) flags |= DeepFrameFlags.White;
|
||
if (stats.MeanLogLuminance <= options.DarkLogLuminance) flags |= DeepFrameFlags.Dark;
|
||
var green = stats.MeanGreen;
|
||
return new DeepFrame
|
||
{
|
||
Index = index,
|
||
Flags = flags,
|
||
LogLuminance = stats.MeanLogLuminance,
|
||
ClipLow = stats.ClipLowFraction,
|
||
ClipHigh = stats.ClipHighFraction,
|
||
RedRatio = stats.MeanRed > 0 && green > 0 ? double.Log2(stats.MeanRed / green) : double.NaN,
|
||
BlueRatio = stats.MeanBlue > 0 && green > 0 ? double.Log2(stats.MeanBlue / green) : double.NaN,
|
||
Kelvin = stats.ColorTemperature?.Kelvin ?? double.NaN,
|
||
Sharpness = stats.Sharpness,
|
||
};
|
||
}
|
||
|
||
/// <summary>Content motion at the centre of the correlation plane, in source pixels; NaN for the first frame of a
|
||
/// size, too small planes and frames without textured patches.</summary>
|
||
private (double X, double Y) Shift(FrameProbe probe)
|
||
{
|
||
int w = probe.CorrelationWidth, h = probe.CorrelationHeight;
|
||
if (w < MinimumStabilizerSize || h < MinimumStabilizerSize || probe.Correlation is null) return (double.NaN, double.NaN);
|
||
if (_stabilizer is null || _stabilizer.Width != w || _stabilizer.Height != h)
|
||
{
|
||
_stabilizer?.Dispose();
|
||
_stabilizer = new Stabilizer(w, h);
|
||
_stabilizer.AddFrame(probe.Correlation);
|
||
return (double.NaN, double.NaN);
|
||
}
|
||
var motion = _stabilizer.AddFrame(probe.Correlation);
|
||
if (!motion.Estimated) return (double.NaN, double.NaN);
|
||
double cx = (w - 1) / 2.0, cy = (h - 1) / 2.0;
|
||
var (x, y) = motion.Transform.Apply(cx, cy);
|
||
return ((x - cx) * probe.CorrelationScale, (y - cy) * probe.CorrelationScale);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Draft flow from the previous frame, with the current frame brought to the previous mean level so flicker and
|
||
/// exposure steps do not read as motion. Only vectors that pass the forward–backward check (error ≤
|
||
/// <see cref="ReliableError"/> flow pixels) count, so flat noisy areas do not either; NaN when under
|
||
/// <see cref="MinimumReliable"/> of the vectors are reliable. Magnitudes in source pixels per frame.
|
||
/// </summary>
|
||
private (double Median, double P95) Motion(FrameProbe probe, FrameStatistics stats, CancellationToken cancellationToken)
|
||
{
|
||
if (probe.Index % options.FlowEvery != 0 || _previousIndex != probe.Index - 1) return (double.NaN, double.NaN);
|
||
if (probe.Flow is not { } current || _previousFlow is not { } previous || previous.Width != current.Width || previous.Height != current.Height)
|
||
return (double.NaN, double.NaN);
|
||
using var levelled = current.Clone();
|
||
var gain = (float)double.Exp2(_previous!.MeanLogLuminance - stats.MeanLogLuminance);
|
||
if (float.IsFinite(gain))
|
||
{
|
||
foreach (ref var v in levelled.Data) v *= gain;
|
||
}
|
||
var (forward, backward) = _flow.ComputeBidirectional(previous, levelled, cancellationToken);
|
||
using (forward)
|
||
using (backward)
|
||
{
|
||
var u = forward.U;
|
||
var w = forward.V;
|
||
var error = forward.Error;
|
||
var magnitudes = new float[forward.Count];
|
||
var count = 0;
|
||
for (var i = 0; i < magnitudes.Length; i++)
|
||
{
|
||
if (error[i] <= ReliableError) magnitudes[count++] = float.Hypot(u[i], w[i]);
|
||
}
|
||
if (count < MinimumReliable * magnitudes.Length) return (double.NaN, double.NaN);
|
||
var reliable = magnitudes.AsSpan(0, count);
|
||
var median = PlaneOps.Median(reliable);
|
||
var p95 = PlaneOps.Select(reliable, int.Max(0, (int)double.Ceiling(0.95 * count) - 1));
|
||
return (median * probe.FlowScale, p95 * probe.FlowScale);
|
||
}
|
||
}
|
||
}
|
||
}
|