Files
Chronos/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs
T
Claude 4014b0e669 fix(proxies): judge proxies by one rule and record tiers and reports as they stand
- ArchiveImages.IsProxyOf compares capture times to the microsecond the index
  keeps (sidecars and Exif keep 100 ns ticks), and for frames read through
  proxies also applies the policy's size rule, so the proxy pass, uploads,
  thumbnails, renders, analyses, tiers, manifests and downloads agree.
- Proxy headers are cached per file while length and write time are unchanged
  (and the file has settled), so manifests cost a stat per proxy.
- FrameTiers.RecordAsync checks proxy presence when it writes the sidecar and
  returns the sequence's frames as they are, so a deep analysis or render that
  ran while proxies arrived no longer flags the frames preview-only again, and
  the analysis advises from the current flags.
- Server manifests ignore ProxiesBuilt for frames the server cannot decode, so
  frames added since the last proxy pass are asked for their proxies.
- GET /sequences/{id}/analysis/report answers 404 for a report that found
  frames unreadable before their proxies arrived (DeepReports.ReadCurrentAsync).
- The deep checkpoint key marks only frames read from proxies, so checkpoints
  written by earlier versions still resume.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 16:52:30 +00:00

593 lines
36 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Security.Cryptography;
using System.Text;
using Titano.Core.Advice;
using Titano.Core.Analysis;
using Titano.Core.Analysis.Deep;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Dng;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Diagnostics.Synthetic;
using Titano.Core.Export;
using Titano.Core.Imaging;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Import;
using Titano.Core.Localization;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Processing;
using Titano.Data.Processing.Jobs;
using Titano.Data.Repositories;
using Titano.Data.Tests.Archiving;
using Titano.Testing;
namespace Titano.Data.Tests.Processing;
[Category("processing")]
public sealed class AnalysisJobTests(TestContext context)
{
private static SyntheticSequence Flickering(int frames = 40) => SyntheticSequence.Create(new SyntheticSequenceOptions
{
FrameCount = frames,
FlickerEv = 0.1,
StartEv = 0.5,
EndEv = -1.5,
Latitude = 45.4,
Longitude = 12.3,
});
[Test]
public async Task LightAnalysisFillsAutoValuesAndKeepsManualOnes()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering());
var manual = new ProjectSettings();
// The user's frame rate, and the highlight knee turned off (hard clip).
manual = manual with
{
Timing = manual.Timing with { FrameRate = manual.Timing.FrameRate.WithManual("25") },
Exposure = manual.Exposure with { HighlightKnee = null },
};
await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = manual.ToJson() }, null, fx.Ct);
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id], sequence.DisplayTitle));
var done = await fx.WaitForAsync(job.Id, JobState.Done);
var folder = fx.Folder(sequence);
var project = ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct));
Assert.Equal((false, "25"), (project.Timing.FrameRate.IsAuto, project.Timing.FrameRate.Resolve("30")));
Assert.Null(project.Exposure.HighlightKnee);
Assert.True(project.Timing.Mode.HasAdvice, "timing mode advised");
Assert.True(project.Timing.FrameRate.HasAdvice, "the advice is kept next to the manual frame rate");
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(project.ToJson(), ProjectSettings.FromJson(row.SettingsJson!).ToJson());
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), "no deep status without a report");
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
Assert.Contains(Loc.L("Advisor.Log.AutoUpdated", sequence.DisplayTitle), done.LogTail);
// Idempotent: a second pass changes nothing.
var version = row.SettingsVersion;
var again = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
await fx.WaitForAsync(again.Id, JobState.Done);
Assert.Equal(version, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.SettingsVersion);
}
[Test]
public async Task LightAnalysisRecordsFormatTiersAndTheAdvisorCapsTheOutputAtThePreviews()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering(12));
// This host reads the first ten files only through 48×32 embedded previews and cannot decode the last one.
for (var i = 0; i < 10; i++)
fx.Decoder.Tiers[ProcessingFixture.FileName(i)] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", "test", 48, 32);
fx.Decoder.Tiers[ProcessingFixture.FileName(11)] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.Unavailable, null, "test", 0, 0);
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
await fx.WaitForAsync(job.Id, JobState.Done);
var folder = fx.Folder(sequence);
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
Assert.Equal((DecodeTier.EmbeddedPreview, FrameFlags.PreviewOnly, (int?)48, (int?)32), (sidecar[0].Tier, sidecar[0].Flags, sidecar[0].PreviewW, sidecar[0].PreviewH));
Assert.Equal((DecodeTier.Native, FrameFlags.None), (sidecar[10].Tier, sidecar[10].Flags));
Assert.Equal((DecodeTier.Unavailable, FrameFlags.Undecodable), (sidecar[11].Tier, sidecar[11].Flags));
var rows = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
Assert.Equal((FrameFlags.PreviewOnly, 48, 32), (rows[3].Flags, rows[3].PreviewWidth, rows[3].PreviewHeight));
Assert.Equal(FrameFlags.Undecodable, rows[11].Flags);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(10, row.PreviewOnlyCount);
Assert.GreaterOrEqual(row.WarningCount, 1);
Assert.True(row.Status.HasFlag(SequenceStatus.HasWarnings), "the undecodable frame is a sequence warning");
// The advisor reads the preview size from the index and caps the output there (BRIEF §8.2 tier 2).
var tiers = SequenceContext.FromFrames(rows);
Assert.Equal((10, 48, 32), (tiers.PreviewOnlyCount, tiers.PreviewWidth, tiers.PreviewHeight));
var light = LightAnalyzer.Analyze(rows);
var advice = Advisor.Recommend(light, null, tiers);
Assert.Equal(AdvisorKeys.ResolutionPreview, advice.Find(AdvisorParameters.Resolution)!.ReasonKey);
Assert.Contains(advice.Findings, f => f.Key == FindingKeys.PreviewOnly && Equals(f.Arguments[0], 10));
Assert.Contains(advice.Findings, f => f.Key == DeepFindingKeys.UnreadableFrames && f.Frames.SequenceEqual([11]));
// Probed once: a second pass opens no file for its tier and leaves the sidecar alone.
fx.Decoder.Tiers.Clear();
var written = File.GetLastWriteTimeUtc(folder.ManifestPath);
var again = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
await fx.WaitForAsync(again.Id, JobState.Done);
Assert.Equal(FrameFlags.PreviewOnly, ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].Flags);
Assert.Equal(written, File.GetLastWriteTimeUtc(folder.ManifestPath));
}
[Test]
public async Task AHeifFrameTheServerReadsFromItsProxyCapsNothing()
{
// The server's decoders (BRIEF §4.2, §8.1): a HEIF gives them only the 160 × 120 JPEG thumbnail of its Exif.
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Archive = o.Archive with { Decoder = DecoderFactory.CreateServer() } });
var directory = Path.Combine(fx.Archive.Paths.Root, "Heif");
Directory.CreateDirectory(directory);
var start = new DateTime(2026, 9, 28, 18, 0, 0);
for (var i = 0; i < 4; i++)
await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024), fx.Ct);
await fx.Archive.ScanAsync(ScanMode.Full);
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif");
var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
var folder = fx.Folder(sequence);
await fx.StartEngineAsync();
// Without proxies the server has nothing better than those thumbnails: preview-only, and the advisor caps the output.
await LightAnalysisAsync(fx, sequence);
var probed = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
Assert.True(probed.All(f => f is { Tier: DecodeTier.EmbeddedPreview, PreviewW: 160, PreviewH: 120 } && f.Flags.HasFlag(FrameFlags.PreviewOnly)),
"preview-only at the thumbnail's size");
Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
var rows = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
var capped = Advisor.Recommend(LightAnalyzer.Analyze(rows), null, SequenceContext.FromFrames(rows));
Assert.Equal((AdvisorKeys.ResolutionPreview, "720p-preview"), (capped.Find(AdvisorParameters.Resolution)!.ReasonKey, capped.Find(AdvisorParameters.ExportPreset)!.Value as string));
Assert.True(capped.Findings.Any(f => f.Key == FindingKeys.PreviewOnly), "preview-only finding");
// A client sends their proxies (BRIEF §2.3) at the policy's 1920 px: the server renders the frames from them, so none
// is preview-only and the advice follows. The tier and preview size stay the originals'.
Directory.CreateDirectory(folder.Proxies);
foreach (var frame in rows) await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct);
await LightAnalysisAsync(fx, sequence);
var proxied = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
Assert.True(proxied.All(f => f is { Tier: DecodeTier.EmbeddedPreview, PreviewW: 160 } && !f.Flags.HasFlag(FrameFlags.PreviewOnly)), "not preview-only");
Assert.Equal(0, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
rows = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
Assert.True(rows.All(f => !f.Flags.HasFlag(FrameFlags.PreviewOnly)), "index flags");
var advice = Advisor.Recommend(LightAnalyzer.Analyze(rows), null, SequenceContext.FromFrames(rows));
Assert.Equal((AdvisorKeys.ResolutionSource, "4k-hevc"), (advice.Find(AdvisorParameters.Resolution)!.ReasonKey, advice.Find(AdvisorParameters.ExportPreset)!.Value as string));
Assert.False(advice.Findings.Any(f => f.Key == FindingKeys.PreviewOnly), "no preview-only finding");
}
[Test]
public async Task ProxiesThatArriveDuringADeepAnalysisStillCount()
{
// A server, whose advisor rules no codec out: it reads these HEIF files only through their 160 × 120 Exif
// thumbnails until proxies come (BRIEF §2.3).
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server });
var directory = Path.Combine(fx.Archive.Paths.Root, "Heif");
Directory.CreateDirectory(directory);
var start = new DateTime(2026, 9, 28, 18, 0, 0);
for (var i = 0; i < 4; i++)
await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024), fx.Ct);
await fx.Archive.ScanAsync(ScanMode.Full);
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif");
var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
var folder = fx.Folder(sequence);
await fx.StartEngineAsync(workers: 2);
async Task<string?> PresetAsync() => ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).Export.Preset.AutoValue;
await LightAnalysisAsync(fx, sequence);
Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
Assert.Equal("720p-preview", await PresetAsync());
// A deep analysis starts on the originals; while it runs, the proxies arrive and a light analysis on the other
// worker counts them.
fx.Decoder.Hold = "IMG_0001.HEIC";
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
Directory.CreateDirectory(folder.Proxies);
foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))
await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct);
await LightAnalysisAsync(fx, sequence);
Assert.Equal(0, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
Assert.Equal("4k-hevc", await PresetAsync());
fx.Decoder.Release();
await fx.WaitForAsync(deep.Id, JobState.Done);
// The deep pass decoded the originals from their thumbnails, yet the frames are read from their proxies by now:
// it flags none preview-only again, and its advice caps nothing.
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Frames.All(f => f.Tier == DecodeTier.EmbeddedPreview && !f.Flags.HasFlag(FrameFlags.PreviewOnly)),
"not preview-only");
Assert.Equal(0, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
Assert.Equal("4k-hevc", await PresetAsync());
}
[Test]
public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering());
var folder = fx.Folder(sequence);
// Frame 4 is a HEIF this host reads only from its proxy (BRIEF §2.3); the proxy a client sends shows the photo.
var original = Path.Combine(folder.Folder, ProcessingFixture.FileName(3));
var photo = await File.ReadAllBytesAsync(original, fx.Ct);
await File.WriteAllBytesAsync(original, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8], fx.Ct);
var frame = (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[3];
await using (var session = await fx.Db.Database.OpenAsync(fx.Ct))
await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE id = $2", [(int)ImageFileFormat.Heif, frame.Id], fx.Ct);
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
ManifestStore.WriteSequence(folder.ManifestPath, sidecar with
{
Frames = [.. sidecar.Frames.Select(f => f.Id == frame.Id ? f with { Format = ImageFileFormat.Heif } : f)],
});
fx.Decoder.HoldFrame(25);
await fx.StartEngineAsync();
// A deep analysis reads the original, finds no pixels, and is interrupted once its checkpoint holds that verdict.
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
await fx.Engine.CancelAsync(job.Id, fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Cancelled);
// The proxy arrives meanwhile: the frame is no longer read as the checkpoint read it, so the resumed run starts over.
Directory.CreateDirectory(folder.Proxies);
await File.WriteAllBytesAsync(folder.Proxy(3), photo, fx.Ct);
fx.Decoder.Hold = null;
await fx.Engine.ResumeAsync(job.Id, fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var report = SequenceAnalyser.ReadReport(folder, sequence.FrameCount)!;
Assert.Empty(report.Issues.Undecodable);
Assert.SequenceEqual([3], report.ProxyFrames!);
// Without its proxy, a deep analysis lists the frame as unreadable, a sequence warning...
File.Delete(folder.Proxy(3));
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.WaitForAsync(deep.Id, JobState.Done);
Assert.SequenceEqual([3], SequenceAnalyser.ReadReport(folder, sequence.FrameCount)!.Issues.Undecodable);
var unreadable = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.True(unreadable.Status.HasFlag(SequenceStatus.AnalyzedDeep | SequenceStatus.HasWarnings), unreadable.Status.ToString());
// ...a verdict that no longer holds once the proxy is there: the next light analysis drops the report and its warning.
await File.WriteAllBytesAsync(folder.Proxy(3), photo, fx.Ct);
await LightAnalysisAsync(fx, sequence);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), "the report is stale");
Assert.Less(row.WarningCount, unreadable.WarningCount);
}
[Test]
public async Task ACheckpointOfASequenceReadFromItsOriginalsKeepsTheKeyEarlierVersionsWrote()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering(4));
var frames = await Titano.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive.Archive, sequence, useProxies: false, new ImageDecoder([fx.Decoder]), fx.Ct);
Assert.SequenceEqual([false, false, false, false], frames.Proxied!);
// A deep analysis interrupted before an upgrade resumes after it (a server restarted mid-run does not start over).
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
foreach (var f in frames.Frames)
hash.AppendData(Encoding.UTF8.GetBytes($"{f.Id:N}/{f.Sha256}/{f.Size}/{f.Flags & Titano.Data.Processing.SequenceFrames.NotAnalysed};"));
var earlier = Convert.ToHexStringLower(hash.GetHashAndReset())[..16];
Assert.Equal(earlier, SequenceAnalyser.ContentKey(frames));
// A frame read from its proxy since makes the checkpoint stale.
Assert.NotEqual(earlier, SequenceAnalyser.ContentKey(frames with { Proxied = [false, true, false, false] }));
}
[Test]
public async Task DeepAnalysisFlagsFramesItCannotDecodeAndListsThemOnce()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering(12));
// Damaged after indexing: the header probe still says JPEG, only decoding finds out.
await File.WriteAllBytesAsync(Path.Combine(fx.Folder(sequence).Folder, ProcessingFixture.FileName(4)), new byte[3000], fx.Ct);
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
var done = await fx.WaitForAsync(job.Id, JobState.Done);
var sidecar = ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Frames;
Assert.Equal((DecodeTier.Unavailable, FrameFlags.Undecodable), (sidecar[4].Tier, sidecar[4].Flags));
Assert.Equal(DecodeTier.Native, sidecar[5].Tier);
Assert.Equal(FrameFlags.Undecodable, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[4].Flags);
var report = SequenceAnalyser.ReadReport(fx.Folder(sequence), sequence.FrameCount)!;
var rows = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
var findings = Advisor.Recommend(LightAnalyzer.Analyze(rows), report, SequenceContext.FromFrames(rows)).Findings;
Assert.Count(1, findings.Where(f => f.Key == DeepFindingKeys.UnreadableFrames));
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.WarningCount >= 1, "listed in the sequence warnings");
Assert.Equal(JobState.Done, done.State);
}
[Test]
public async Task AnImportedGoProRawDecodesThroughTheJpegItsImportKeptInAnotherSequence()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
// A GoPro card: each GPR (VC-5, without a usable embedded image here) has the camera's full-size JPEG beside it.
// An upload may send the GPRs first: they arrive as two cards here.
var raws = Path.Combine(context.CreateTempDirectory("card"), "DCIM", "100GOPRO");
var jpegs = Path.Combine(context.CreateTempDirectory("card"), "DCIM", "100GOPRO");
Directory.CreateDirectory(raws);
Directory.CreateDirectory(jpegs);
var start = new DateTime(2026, 9, 28, 18, 0, 0);
for (var i = 0; i < 4; i++)
{
var metadata = new FrameMetadata { Make = "GoPro", Model = "HERO12 Black", BodySerial = "C3501", CapturedAt = start.AddSeconds(5 * i) };
using var sensor = new ImageBuffer(64, 48);
File.WriteAllBytes(Path.Combine(raws, $"GOPR{i + 1:0000}.GPR"),
LinearDngWriter.Write(sensor, new LinearDngOptions { Exif = ExifWriter.BuildTiff(metadata), TileSize = 64 }));
using var picture = new ImageBuffer(160, 120);
picture.Plane(1).Fill(0.25f * (i + 1));
File.WriteAllBytes(Path.Combine(jpegs, $"GOPR{i + 1:0000}.JPG"),
JpegEncoder.Encode(picture, new JpegEncodeOptions { App1Segments = [ExifWriter.BuildJpegApp1(metadata)[4..]] }));
}
var service = fx.Archive.Service;
async Task<ImportPlan> ImportAsync(string card)
{
var plan = await service.PlanImportAsync(fx.Archive.Archive.Id, Path.GetDirectoryName(Path.GetDirectoryName(card))!, new ImportOptions(),
cancellationToken: fx.Ct);
await service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Archive.Id, plan, null, true, null), Guid.NewGuid(), 0,
_ => ValueTask.CompletedTask, null, fx.Ct);
return plan;
}
var gprSession = (await ImportAsync(raws)).Sessions.Single();
Assert.Equal(ImageFileFormat.Gpr, gprSession.Format);
var id = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Archive.Id, gprSession.SessionKey, fx.Ct);
var sequence = (await fx.Db.Sequences.GetAsync(id!.Value, fx.Ct))!;
await fx.StartEngineAsync();
// Alone, a GPR without an embedded preview gives no pixels here, which proves nothing about the file.
var first = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
await fx.WaitForAsync(first.Id, JobState.Done);
var alone = ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Frames;
Assert.True(alone.All(f => f is { Tier: DecodeTier.Unavailable, Flags: FrameFlags.None, PreviewW: null }), "unavailable, not flagged");
// The JPEGs arrive as a sequence of their own (the import splits per file format, DECISIONS 2026-10-02); the next
// probe finds each GPR's JPEG there, so the output cap is the JPEG's size.
Assert.Equal(ImageFileFormat.Jpeg, (await ImportAsync(jpegs)).Sessions.Single().Format);
var jpegFrame = (await fx.Db.Frames.ListByCaptureRangeAsync(fx.Archive.Archive.Id, ImageFileFormat.Jpeg, start, start.AddMinutes(1), fx.Ct))[0];
Assert.NotEqual(Path.GetDirectoryName(fx.Folder(sequence).File(alone[0].File)), Path.GetDirectoryName(fx.Archive.Paths.FromRelative(jpegFrame.RelativePath)),
"not beside the GPR");
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
await fx.WaitForAsync(light.Id, JobState.Done);
var sidecar = ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Frames;
Assert.True(sidecar.All(f => f is { Tier: DecodeTier.EmbeddedPreview, Flags: FrameFlags.PreviewOnly, PreviewW: 160, PreviewH: 120 }),
"preview-only at the JPEG's size");
// The deep pass decodes the GPRs through those JPEGs (decoder "managed-sidecar"), never failing a frame.
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.WaitForAsync(deep.Id, JobState.Done);
var gprs = fx.Decoder.Decoded.Where(d => d.File.EndsWith(".GPR", StringComparison.OrdinalIgnoreCase)).ToList();
Assert.NotEmpty(gprs);
Assert.True(gprs.All(d => d.Decoder == "managed-sidecar"), string.Join(", ", gprs.Select(d => d.Decoder).Distinct()));
Assert.Empty(SequenceAnalyser.ReadReport(fx.Folder(sequence), sequence.FrameCount)!.Issues.Undecodable);
}
[Test]
public async Task DeepAnalysisNeverFlagsFramesThatVanishedOrCouldNotBeRead()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering(12));
// Deleted after indexing (a verify flags it missing), and held by another program during the pass.
File.Delete(Path.Combine(fx.Folder(sequence).Folder, ProcessingFixture.FileName(3)));
fx.Decoder.Unreadable[ProcessingFixture.FileName(6)] = 0;
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.WaitForAsync(job.Id, JobState.Done);
// The pass could not read them, so it lists them as unreadable, but nothing proves the files undecodable.
var report = SequenceAnalyser.ReadReport(fx.Folder(sequence), sequence.FrameCount)!;
Assert.SequenceEqual([3, 6], report.Issues.Undecodable);
var sidecar = ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Frames;
foreach (var index in new[] { 3, 6 })
{
Assert.Equal(FrameFlags.None, sidecar[index].Flags & FrameFlags.Undecodable);
Assert.NotEqual(DecodeTier.Unavailable, sidecar[index].Tier);
}
var rows = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
Assert.True(rows.All(f => (f.Flags & FrameFlags.Undecodable) == 0), "no frame flagged in the index");
var unreadable = Advisor.Recommend(LightAnalyzer.Analyze(rows), report, SequenceContext.FromFrames(rows)).Findings
.Single(f => f.Key == DeepFindingKeys.UnreadableFrames);
Assert.SequenceEqual([3, 6], unreadable.Frames);
// Back in place and readable, the next pass reads both frames.
fx.Decoder.Unreadable.Clear();
await File.WriteAllBytesAsync(Path.Combine(fx.Folder(sequence).Folder, ProcessingFixture.FileName(3)),
await File.ReadAllBytesAsync(Path.Combine(fx.Folder(sequence).Folder, ProcessingFixture.FileName(2)), fx.Ct), fx.Ct);
var again = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.WaitForAsync(again.Id, JobState.Done);
Assert.Empty(SequenceAnalyser.ReadReport(fx.Folder(sequence), sequence.FrameCount)!.Issues.Undecodable);
}
[Test]
public async Task AResumedDeepAnalysisDecodesAgainWhatTheEarlierRunCouldNotRead()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering());
// The first run cannot read frame 12 (held by another program) and is cancelled later, at frame 26.
fx.Decoder.Unreadable[ProcessingFixture.FileName(11)] = 0;
fx.Decoder.HoldFrame(25);
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
await fx.Engine.CancelAsync(job.Id, fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Cancelled);
// Readable again; the resumed run takes frame 12 from its checkpoint (it is not in the region sample).
fx.Decoder.Unreadable.Clear();
fx.Decoder.Hold = null;
while (fx.Decoder.Requests.TryDequeue(out _))
{
}
await fx.Engine.ResumeAsync(job.Id, fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Contains(11, SequenceAnalyser.ReadReport(fx.Folder(sequence), sequence.FrameCount)!.Issues.Undecodable);
// That verdict was a read error: decoding the frame again shows it is fine, so nothing flags it.
Assert.Count(1, fx.Decoder.Requests.Where(r => Path.GetFileName(r.Path) == ProcessingFixture.FileName(11)));
var frame = ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Frames[11];
Assert.Equal((DecodeTier.Native, FrameFlags.None), (frame.Tier, frame.Flags & FrameFlags.Undecodable));
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[11].Flags);
}
[Test]
public async Task DeepAnalysisResumesAfterCancellationWithTheSameReport()
{
await using var reference = await ProcessingFixture.CreateAsync(context);
var expectedSequence = await reference.AddAsync(Flickering());
await reference.StartEngineAsync();
var full = await reference.EnqueueAsync(ProcessingJobs.DeepAnalysis([expectedSequence.Id]));
await reference.WaitForAsync(full.Id, JobState.Done);
var expected = await File.ReadAllTextAsync(reference.Folder(expectedSequence).AnalysisPath, reference.Ct);
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering());
var folder = fx.Folder(sequence);
var partial = Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName);
// Frame 25 is not in the region sample, so the hold stops the pass inside the batch of frames 24–25.
fx.Decoder.HoldFrame(25);
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
await fx.Engine.CancelAsync(job.Id, fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Cancelled);
var saved = File.ReadAllLines(partial).Length - 1;
Assert.Equal(24, saved);
Assert.FileMissing(folder.AnalysisPath);
fx.Decoder.Hold = null;
while (fx.Decoder.Requests.TryDequeue(out _))
{
}
await fx.Engine.ResumeAsync(job.Id, fx.Ct);
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(expected, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct));
Assert.FileMissing(partial);
// After the region sample, only the last checkpointed frame is decoded again (to restore the comparison state).
var analysed = fx.Decoder.Requests.Skip(ProcessingFixture.DeepOptions.RegionSampleFrames)
.Select(r => int.Parse(Path.GetFileNameWithoutExtension(r.Path)[4..]) - 1).ToList();
Assert.Equal(saved - 1, analysed.Min());
Assert.Equal(sequence.FrameCount, (int)done.ProgressDone);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep | SequenceStatus.AnalyzedLight), "index status");
var summary = AnalysisSummary.FromJson(row.AnalysisJson!);
Assert.Equal(sequence.FrameCount, summary.FrameCount);
Assert.NotNull((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[5].MeanLogLuminance);
var project = ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct));
Assert.True(project.Exposure.Deflicker.HasAdvice, "deflicker advised from the pixels");
Assert.Contains(Loc.L("Advisor.Log.Presets", string.Empty), done.LogTail);
}
[Test]
public async Task EachDeepAnalysisTakesTheHostsTuningOfThatMoment()
{
// The desktop's Settings › Analysis: the width cap the user sets now applies to the next analysis.
var cap = 40;
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with
{
Tune = basis => basis with { DeepAnalysis = basis.DeepAnalysis with { MinAnalysisLongEdge = 16, MaxAnalysisLongEdge = cap } },
});
var first = await fx.AddAsync(Flickering(10), "A");
var second = await fx.AddAsync(Flickering(10), "B");
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([first.Id]));
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(40, DeepAnalysisReport.Load(fx.Folder(first).AnalysisPath).AnalysisLongEdge);
cap = 32;
job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([second.Id]));
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(32, DeepAnalysisReport.Load(fx.Folder(second).AnalysisPath).AnalysisLongEdge);
}
[Test]
public async Task ASequenceThatFailsDoesNotStopTheOthersAndARetryRedoesOnlyIt()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var a = await fx.AddAsync(Flickering(10), "A");
var b = await fx.AddAsync(Flickering(10), "B");
var c = await fx.AddAsync(Flickering(10), "C");
// B's sidecar is gone for a while: its status cannot be recorded.
var sidecar = fx.Folder(b).ManifestPath;
var saved = await File.ReadAllBytesAsync(sidecar, fx.Ct);
File.Delete(sidecar);
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([a.Id, b.Id, c.Id]));
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.True(failed.Error!.StartsWith(Loc.L("Advisor.Log.Failed", b.DisplayTitle, string.Empty), StringComparison.Ordinal), failed.Error);
Assert.Equal("3;1", failed.Checkpoint);
Assert.SequenceEqual([true, false, true], await AnalysedAsync(fx, a, b, c));
await File.WriteAllBytesAsync(sidecar, saved, fx.Ct);
await fx.Engine.ResumeAsync(job.Id, fx.Ct);
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.SequenceEqual([true, true, true], await AnalysedAsync(fx, a, b, c));
Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal));
// The retry analysed B alone.
var lightOfA = Loc.L("Advisor.Log.LightDone", a.DisplayTitle, 10, 0).Split(',')[0];
Assert.Equal(1, done.LogTail!.Split('\n').Count(line => line.Contains(lightOfA, StringComparison.Ordinal)));
}
[Test]
public void AnalysisCheckpointsKeepTheFailedTargets()
{
Assert.Equal("3", AnalyseJobHandler.Format(3, []));
Assert.Equal("4;0,2", AnalyseJobHandler.Format(4, [2, 0]));
var (next, retry) = AnalyseJobHandler.Parse("4;0,2", 5);
Assert.Equal(4, next);
Assert.SequenceEqual([0, 2], retry);
Assert.Equal(2, AnalyseJobHandler.Parse("2", 5).Next);
// Anything else starts over: no checkpoint, beyond the targets, a failed target not yet reached, garbage.
foreach (var text in new[] { null, "6", "2;3", "2;x", "a" }) Assert.Equal(0, AnalyseJobHandler.Parse(text, 5).Next);
}
private static async Task LightAnalysisAsync(ProcessingFixture fx, Sequence sequence)
{
var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
await fx.WaitForAsync(job.Id, JobState.Done);
}
/// <summary>A JPEG (320 × 240 unless sized) with <paramref name="captured"/> in its Exif, as a client makes a proxy.</summary>
private static byte[] Proxy(DateTime captured, int width = 320, int height = 240)
{
using var image = new ImageBuffer(width, height);
for (var c = 0; c < 3; c++) image.Plane(c).Fill(0.3f + c * 0.1f);
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Titano" });
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
}
private static async Task<List<bool>> AnalysedAsync(ProcessingFixture fx, params Sequence[] sequences)
{
var analysed = new List<bool>();
foreach (var s in sequences) analysed.Add((await fx.Db.Sequences.GetAsync(s.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedLight));
return analysed;
}
[Test]
public async Task LightAnalysisAfterNewFramesDropsTheStaleDeepStatus()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering(12));
await fx.StartEngineAsync();
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.WaitForAsync(deep.Id, JobState.Done);
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedDeep), "deep status");
// A thirteenth frame arrives: the report describes 12 frames, so it no longer counts.
await fx.AddAsync(Flickering(13), first: 12);
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
await fx.WaitForAsync(light.Id, JobState.Done);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(13, row.FrameCount);
Assert.Equal((true, false), (row.Status.HasFlag(SequenceStatus.AnalyzedLight), row.Status.HasFlag(SequenceStatus.AnalyzedDeep)));
}
}