Second review of owner review 2, phase 4a (Reorganize), cross-cutting findings. - A second index of the archive (a desktop sharing a server's archive, or a folder moved by hand) moved the sequence row at its next scan but left renders.output_path at the old folder: the export answered 404 there. The scan now rebases the render rows in the same transaction when it finds a sequence in another folder. - The rebase covers every render of the archive under the old folder: the sequence's own exports always, and another sequence's export saved inside it (through the Export folder picker) when its file moved with the folder. That check keeps the exports of a sequence that took the old folder since (a later Reorganize on the other index) where they are. A change of case only counts as a move. - The desktop now tells an open editor after each Reorganize step (ArchiveServiceOptions.SequenceChanged -> AppServices.SequenceReorganized), and the editor reads its sequence again where it is now, so frames no longer decode blank until the next save. LocalSequenceSource.LoadAsync also requires the same folder on both row reads, so a move that commits during the frame listing is never taken as current. - Writers that wrote without the lease, or trusted a row another index had moved, could create the old folder again. A proxy sent by a client is now filed under the sequence's lease into the folder the index has then, and a gone sequence or folder answers 404 (sequence_busy 503 after the lease wait). The server files a render a client or render node sends under the lease (ArchiveService.UnderLeaseAsync). "Save both" keeps its copy under the lease. The proxy pass holds the thumbnail lease, as the thumbnail pass does. The deep analysis opens its checkpoint under the sequence lock only while the folder is still there, and a pass whose folder another process moved records nothing (the job logs Advisor.Log.FramesChangedElsewhere). Editor saves and sync applies refuse when the folder is gone. - ARCHITECTURE.md no longer claims more than this: a render of another process whose folder moves while it runs is still not stopped. - ArchiveVerifier.FlushAsync: corrected the comment. A move keeps each file's stamp, so only a check that found the file missing no longer applies. Tests cover the scan path with a later swap into the vacated folder, another sequence's export inside a moving folder, a photo deletion refused when the folder moved while it waited for the lease, proxies sent and made after a move, Save both during a move, a deep analysis whose folder another index moved, a render filed after a move on the server, and an open editor reloaded by a Move job. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit c857b98f795b22956c060f1c5acf4d94b6d1c994)
1023 lines
64 KiB
C#
1023 lines
64 KiB
C#
using System.Security.Cryptography;
|
||
using System.Text;
|
||
using Chronos.Core.Advice;
|
||
using Chronos.Core.Analysis;
|
||
using Chronos.Core.Analysis.Deep;
|
||
using Chronos.Core.Archiving;
|
||
using Chronos.Core.Codecs.Dng;
|
||
using Chronos.Core.Codecs.Jpeg;
|
||
using Chronos.Core.Diagnostics.Synthetic;
|
||
using Chronos.Core.Export;
|
||
using Chronos.Core.Imaging;
|
||
using Chronos.Core.Imaging.Decoders;
|
||
using Chronos.Core.Import;
|
||
using Chronos.Core.Localization;
|
||
using Chronos.Core.Metadata;
|
||
using Chronos.Core.Model;
|
||
using Chronos.Core.Project;
|
||
using Chronos.Data.Archiving;
|
||
using Chronos.Data.Archiving.Jobs;
|
||
using Chronos.Data.Jobs;
|
||
using Chronos.Data.Processing;
|
||
using Chronos.Data.Processing.Jobs;
|
||
using Chronos.Data.Repositories;
|
||
using Chronos.Data.Tests.Archiving;
|
||
using Chronos.Testing;
|
||
|
||
namespace Chronos.Data.Tests.Processing;
|
||
|
||
[Category("processing")]
|
||
public sealed class AnalysisJobTests(TestContext context)
|
||
{
|
||
private static SyntheticSequence Flickering(int frames = 40) => SyntheticSequence.Create(new SyntheticSequenceOptions
|
||
{
|
||
FrameCount = frames,
|
||
FlickerEv = 0.1,
|
||
StartEv = 0.5,
|
||
EndEv = -1.5,
|
||
Latitude = 45.4,
|
||
Longitude = 12.3,
|
||
});
|
||
|
||
[Test]
|
||
public async Task LightAnalysisFillsAutoValuesAndKeepsManualOnes()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering());
|
||
var manual = new ProjectSettings();
|
||
// The user's frame rate, and the highlight knee turned off (hard clip).
|
||
manual = manual with
|
||
{
|
||
Timing = manual.Timing with { FrameRate = manual.Timing.FrameRate.WithManual("25") },
|
||
Exposure = manual.Exposure with { HighlightKnee = null },
|
||
};
|
||
await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = manual.ToJson() }, null, fx.Ct);
|
||
await fx.StartEngineAsync();
|
||
|
||
var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id], sequence.DisplayTitle));
|
||
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
||
|
||
var folder = fx.Folder(sequence);
|
||
var project = ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct));
|
||
Assert.Equal((false, "25"), (project.Timing.FrameRate.IsAuto, project.Timing.FrameRate.Resolve("30")));
|
||
Assert.Null(project.Exposure.HighlightKnee);
|
||
Assert.True(project.Timing.Mode.HasAdvice, "timing mode advised");
|
||
Assert.True(project.Timing.FrameRate.HasAdvice, "the advice is kept next to the manual frame rate");
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(project.ToJson(), ProjectSettings.FromJson(row.SettingsJson!).ToJson());
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
|
||
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), "no deep status without a report");
|
||
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
|
||
Assert.Contains(Loc.L("Advisor.Log.AutoUpdated", sequence.DisplayTitle), done.LogTail);
|
||
|
||
// Idempotent: a second pass changes nothing.
|
||
var version = row.SettingsVersion;
|
||
var again = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(again.Id, JobState.Done);
|
||
Assert.Equal(version, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.SettingsVersion);
|
||
}
|
||
|
||
[Test]
|
||
public async Task LightAnalysisRecordsFormatTiersAndTheAdvisorCapsTheOutputAtThePreviews()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering(12));
|
||
// This host reads the first ten files only through 48×32 embedded previews and cannot decode the last one.
|
||
for (var i = 0; i < 10; i++)
|
||
fx.Decoder.Tiers[ProcessingFixture.FileName(i)] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", "test", 48, 32);
|
||
fx.Decoder.Tiers[ProcessingFixture.FileName(11)] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.Unavailable, null, "test", 0, 0);
|
||
await fx.StartEngineAsync();
|
||
|
||
var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||
|
||
var folder = fx.Folder(sequence);
|
||
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
|
||
Assert.Equal((DecodeTier.EmbeddedPreview, FrameFlags.PreviewOnly, (int?)48, (int?)32), (sidecar[0].Tier, sidecar[0].Flags, sidecar[0].PreviewW, sidecar[0].PreviewH));
|
||
Assert.Equal((DecodeTier.Native, FrameFlags.None), (sidecar[10].Tier, sidecar[10].Flags));
|
||
Assert.Equal((DecodeTier.Unavailable, FrameFlags.Undecodable), (sidecar[11].Tier, sidecar[11].Flags));
|
||
var rows = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
Assert.Equal((FrameFlags.PreviewOnly, 48, 32), (rows[3].Flags, rows[3].PreviewWidth, rows[3].PreviewHeight));
|
||
Assert.Equal(FrameFlags.Undecodable, rows[11].Flags);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(10, row.PreviewOnlyCount);
|
||
Assert.GreaterOrEqual(row.WarningCount, 1);
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.HasWarnings), "the undecodable frame is a sequence warning");
|
||
|
||
// The advisor reads the preview size from the index and caps the output there (BRIEF §8.2 tier 2).
|
||
var tiers = SequenceContext.FromFrames(rows);
|
||
Assert.Equal((10, 48, 32), (tiers.PreviewOnlyCount, tiers.PreviewWidth, tiers.PreviewHeight));
|
||
var light = LightAnalyzer.Analyze(rows);
|
||
var advice = Advisor.Recommend(light, null, tiers);
|
||
Assert.Equal(AdvisorKeys.ResolutionPreview, advice.Find(AdvisorParameters.Resolution)!.ReasonKey);
|
||
Assert.Contains(advice.Findings, f => f.Key == FindingKeys.PreviewOnly && Equals(f.Arguments[0], 10));
|
||
Assert.Contains(advice.Findings, f => f.Key == DeepFindingKeys.UnreadableFrames && f.Frames.SequenceEqual([11]));
|
||
|
||
// Probed once: a second pass opens no file for its tier and leaves the sidecar alone.
|
||
fx.Decoder.Tiers.Clear();
|
||
var written = File.GetLastWriteTimeUtc(folder.ManifestPath);
|
||
var again = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(again.Id, JobState.Done);
|
||
Assert.Equal(FrameFlags.PreviewOnly, ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].Flags);
|
||
Assert.Equal(written, File.GetLastWriteTimeUtc(folder.ManifestPath));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AHeifFrameTheServerReadsFromItsProxyCapsNothing()
|
||
{
|
||
// The server's decoders (BRIEF §4.2, §8.1): a HEIF gives them only the 160 × 120 JPEG thumbnail of its Exif.
|
||
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Archive = o.Archive with { Decoder = DecoderFactory.CreateServer() } });
|
||
var directory = Path.Combine(fx.Archive.Paths.Root, "Heif");
|
||
Directory.CreateDirectory(directory);
|
||
var start = new DateTime(2026, 9, 28, 18, 0, 0);
|
||
for (var i = 0; i < 4; i++)
|
||
await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024), fx.Ct);
|
||
await fx.Archive.ScanAsync(ScanMode.Full);
|
||
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif");
|
||
var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
|
||
var folder = fx.Folder(sequence);
|
||
await fx.StartEngineAsync();
|
||
|
||
// Without proxies the server has nothing better than those thumbnails: preview-only, and the advisor caps the output.
|
||
await LightAnalysisAsync(fx, sequence);
|
||
var probed = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
|
||
Assert.True(probed.All(f => f is { Tier: DecodeTier.EmbeddedPreview, PreviewW: 160, PreviewH: 120 } && f.Flags.HasFlag(FrameFlags.PreviewOnly)),
|
||
"preview-only at the thumbnail's size");
|
||
Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
var rows = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
var capped = Advisor.Recommend(LightAnalyzer.Analyze(rows), null, SequenceContext.FromFrames(rows));
|
||
Assert.Equal((AdvisorKeys.ResolutionPreview, "720p-preview"), (capped.Find(AdvisorParameters.Resolution)!.ReasonKey, capped.Find(AdvisorParameters.ExportPreset)!.Value as string));
|
||
Assert.True(capped.Findings.Any(f => f.Key == FindingKeys.PreviewOnly), "preview-only finding");
|
||
|
||
// A client sends their proxies (BRIEF §2.3) at the policy's 1920 px: the server renders the frames from them, so none
|
||
// is preview-only and the advice follows. The tier and preview size stay the originals'.
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
foreach (var frame in rows) await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct);
|
||
await LightAnalysisAsync(fx, sequence);
|
||
var proxied = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
|
||
Assert.True(proxied.All(f => f is { Tier: DecodeTier.EmbeddedPreview, PreviewW: 160 } && !f.Flags.HasFlag(FrameFlags.PreviewOnly)), "not preview-only");
|
||
Assert.Equal(0, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
rows = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
Assert.True(rows.All(f => !f.Flags.HasFlag(FrameFlags.PreviewOnly)), "index flags");
|
||
var advice = Advisor.Recommend(LightAnalyzer.Analyze(rows), null, SequenceContext.FromFrames(rows));
|
||
Assert.Equal((AdvisorKeys.ResolutionSource, "4k-hevc"), (advice.Find(AdvisorParameters.Resolution)!.ReasonKey, advice.Find(AdvisorParameters.ExportPreset)!.Value as string));
|
||
Assert.False(advice.Findings.Any(f => f.Key == FindingKeys.PreviewOnly), "no preview-only finding");
|
||
}
|
||
|
||
[Test]
|
||
public async Task ProxiesThatArriveDuringADeepAnalysisStillCount()
|
||
{
|
||
// A server, whose advisor rules no codec out: it reads these HEIF files only through their 160 × 120 Exif
|
||
// thumbnails until proxies come (BRIEF §2.3).
|
||
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server });
|
||
var directory = Path.Combine(fx.Archive.Paths.Root, "Heif");
|
||
Directory.CreateDirectory(directory);
|
||
var start = new DateTime(2026, 9, 28, 18, 0, 0);
|
||
for (var i = 0; i < 4; i++)
|
||
await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024), fx.Ct);
|
||
await fx.Archive.ScanAsync(ScanMode.Full);
|
||
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif");
|
||
var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
|
||
var folder = fx.Folder(sequence);
|
||
await fx.StartEngineAsync(workers: 2);
|
||
async Task<string?> PresetAsync() => ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).Export.Preset.AutoValue;
|
||
await LightAnalysisAsync(fx, sequence);
|
||
Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
Assert.Equal("720p-preview", await PresetAsync());
|
||
|
||
// A deep analysis starts on the originals; while it runs, the proxies arrive and a light analysis on the other
|
||
// worker counts them.
|
||
fx.Decoder.Hold = "IMG_0001.HEIC";
|
||
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))
|
||
await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct);
|
||
await LightAnalysisAsync(fx, sequence);
|
||
Assert.Equal(0, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
Assert.Equal("4k-hevc", await PresetAsync());
|
||
fx.Decoder.Release();
|
||
await fx.WaitForAsync(deep.Id, JobState.Done);
|
||
|
||
// The deep pass decoded the originals from their thumbnails, yet the frames are read from their proxies by now:
|
||
// it flags none preview-only again, and its advice caps nothing.
|
||
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Frames.All(f => f.Tier == DecodeTier.EmbeddedPreview && !f.Flags.HasFlag(FrameFlags.PreviewOnly)),
|
||
"not preview-only");
|
||
Assert.Equal(0, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
Assert.Equal("4k-hevc", await PresetAsync());
|
||
}
|
||
|
||
[Test]
|
||
public async Task ProxiesThatArriveDuringADeepAnalysisOfHeifsWithoutThumbnailsMakeItsUnreadableVerdictStale()
|
||
{
|
||
// A server, which finds no pixels at all in these HEIF files (they carry no Exif thumbnail) until proxies come (BRIEF §2.3).
|
||
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server });
|
||
var directory = Path.Combine(fx.Archive.Paths.Root, "Heif");
|
||
Directory.CreateDirectory(directory);
|
||
var start = new DateTime(2026, 9, 28, 18, 0, 0);
|
||
for (var i = 0; i < 4; i++)
|
||
{
|
||
await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024, thumbnailWidth: 0),
|
||
fx.Ct);
|
||
}
|
||
await fx.Archive.ScanAsync(ScanMode.Full);
|
||
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif");
|
||
var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
|
||
var folder = fx.Folder(sequence);
|
||
await fx.StartEngineAsync(workers: 2);
|
||
async Task<string> AutoValuesAsync() =>
|
||
File.Exists(folder.ProjectPath) ? ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).ToJson() : string.Empty;
|
||
|
||
// A deep analysis starts on the originals; while it runs, the proxies arrive and a light analysis on the other
|
||
// worker reads the frames from them.
|
||
fx.Decoder.Hold = "IMG_0001.HEIC";
|
||
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))
|
||
await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct);
|
||
await LightAnalysisAsync(fx, sequence);
|
||
var light = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
var advice = await AutoValuesAsync();
|
||
fx.Decoder.Release();
|
||
await fx.WaitForAsync(deep.Id, JobState.Done);
|
||
|
||
// The deep pass found no pixels in any original, a verdict the proxies made stale before it ended, as every reader
|
||
// of its report finds: it records no deep status, and neither warnings nor advice of its own.
|
||
Assert.Count(4, SequenceAnalyser.ReadReport(folder, sequence.FrameCount)!.Issues.Undecodable);
|
||
var now = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive.Archive, sequence, useProxies: false,
|
||
new ImageDecoder([fx.Decoder]), fx.Ct);
|
||
Assert.Null(SequenceAnalyser.ReadReport(folder, now));
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
|
||
Assert.False(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status");
|
||
Assert.Equal((light.WarningCount, light.Status.HasFlag(SequenceStatus.HasWarnings)), (row.WarningCount, row.Status.HasFlag(SequenceStatus.HasWarnings)));
|
||
Assert.Equal(advice, await AutoValuesAsync());
|
||
}
|
||
|
||
[Test]
|
||
public async Task ADeepAnalysisMadeStaleByProxiesLeavesTheReportALaterPassMadeFromThem()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server });
|
||
var directory = Path.Combine(fx.Archive.Paths.Root, "Heif");
|
||
Directory.CreateDirectory(directory);
|
||
var start = new DateTime(2026, 9, 28, 18, 0, 0);
|
||
for (var i = 0; i < 4; i++)
|
||
{
|
||
await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024, thumbnailWidth: 0),
|
||
fx.Ct);
|
||
}
|
||
await fx.Archive.ScanAsync(ScanMode.Full);
|
||
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif");
|
||
var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
|
||
var folder = fx.Folder(sequence);
|
||
await fx.StartEngineAsync(workers: 2);
|
||
|
||
// A deep analysis starts on the originals, in which the server finds no pixels; while it runs, the proxies arrive
|
||
// and a second deep analysis, on the other worker, measures every frame from them and records its report.
|
||
fx.Decoder.Hold = "IMG_0001.HEIC";
|
||
var stale = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
fx.Decoder.Hold = null;
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))
|
||
await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct);
|
||
var current = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(current.Id, JobState.Done);
|
||
var report = await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct);
|
||
var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.True(analysed.Status.HasFlag(SequenceStatus.AnalyzedDeep), analysed.Status.ToString());
|
||
fx.Decoder.Release();
|
||
await fx.WaitForAsync(stale.Id, JobState.Done);
|
||
|
||
// The first pass's verdict on the originals no longer describes the frames: it leaves the report made from the
|
||
// proxies, and with it the deep status, in place.
|
||
Assert.Equal(report, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct));
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
|
||
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status");
|
||
Assert.Equal(analysed.AnalysisJson, row.AnalysisJson);
|
||
}
|
||
|
||
[Test]
|
||
public async Task FramesAddedDuringADeepAnalysisKeepTheLightAnalysisOfTheFramesAsTheyAre()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering(12));
|
||
var folder = fx.Folder(sequence);
|
||
await fx.StartEngineAsync(workers: 2);
|
||
|
||
// A deep analysis starts on twelve frames; while it runs, a thirteenth photo this host cannot decode is indexed,
|
||
// and a light analysis on the other worker analyses all thirteen.
|
||
fx.Decoder.HoldFrame(0);
|
||
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
fx.Decoder.Tiers[ProcessingFixture.FileName(12)] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.Unavailable, null, "test", 0, 0);
|
||
await fx.AddAsync(Flickering(13), first: 12);
|
||
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(light.Id, JobState.Done);
|
||
var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
var advice = await ProjectJsonAsync(fx, folder);
|
||
Assert.Equal(13, analysed.FrameCount);
|
||
Assert.Contains("13", ProjectSettings.FromJson(advice).Timing.FrameRate.ReasonArgs!);
|
||
Assert.True(analysed.Status.HasFlag(SequenceStatus.HasWarnings), "the photo this host cannot decode is a warning");
|
||
fx.Decoder.Release();
|
||
var done = await fx.WaitForAsync(deep.Id, JobState.Done);
|
||
|
||
// The deep pass measured twelve of them: it saved its report but applied none of it (neither the Auto values nor
|
||
// the status and warnings of twelve frames), and queued a light analysis of the frames as they are.
|
||
Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail);
|
||
Assert.Equal(12, SequenceAnalyser.ReadReport(folder, 12)!.FrameCount);
|
||
var again = Assert.Contains(await LightJobsAsync(fx), j => j.Id != light.Id);
|
||
await fx.WaitForAsync(again.Id, JobState.Done);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal((analysed.WarningCount, analysed.Status), (row.WarningCount, row.Status));
|
||
Assert.Null(row.AnalysisJson);
|
||
Assert.Equal(advice, await ProjectJsonAsync(fx, folder));
|
||
}
|
||
|
||
[Test]
|
||
public async Task ADeepAnalysisOvertakenByOneOfTheFramesAsTheyAreLeavesThatOnesReport()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering(12));
|
||
var folder = fx.Folder(sequence);
|
||
await fx.StartEngineAsync(workers: 2);
|
||
|
||
// A deep analysis starts on twelve frames; while it runs, a thirteenth photo is indexed and a second deep analysis,
|
||
// on the other worker, analyses all thirteen and records its report.
|
||
fx.Decoder.HoldFrame(0);
|
||
var overtaken = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
fx.Decoder.Hold = null;
|
||
await fx.AddAsync(Flickering(13), first: 12);
|
||
var current = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(current.Id, JobState.Done);
|
||
var report = await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct);
|
||
Assert.Equal(13, SequenceAnalyser.ReadReport(folder, 13)!.FrameCount);
|
||
var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.True(analysed.Status.HasFlag(SequenceStatus.AnalyzedDeep), analysed.Status.ToString());
|
||
fx.Decoder.Release();
|
||
var done = await fx.WaitForAsync(overtaken.Id, JobState.Done);
|
||
|
||
// The first pass measured twelve: it leaves the report of the thirteen in place, so the light analysis it queues
|
||
// keeps the deep status and its summary.
|
||
Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail);
|
||
Assert.Equal(report, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct));
|
||
Assert.FileMissing(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
|
||
var light = Assert.Contains(await LightJobsAsync(fx), j => j.TargetIds.Contains(sequence.Id));
|
||
await fx.WaitForAsync(light.Id, JobState.Done);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
|
||
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status");
|
||
Assert.Equal(analysed.AnalysisJson, row.AnalysisJson);
|
||
}
|
||
|
||
[Test]
|
||
public async Task ADeepAnalysisThatLeavesTheReportInPlaceRecordsThatReport()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering(12));
|
||
var folder = fx.Folder(sequence);
|
||
await fx.StartEngineAsync();
|
||
var first = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(first.Id, JobState.Done);
|
||
var measured = SequenceAnalyser.ReadReport(folder, 12)!;
|
||
|
||
// A deep analysis starts on the twelve frames on the only worker. Meanwhile another Chronos sharing the archive files
|
||
// them again, at the next frames version, and saves its own analysis of them; this index's scan indexes them, and
|
||
// their light analysis waits in the queue.
|
||
fx.Decoder.HoldFrame(0);
|
||
var overtaken = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
fx.Decoder.Hold = null;
|
||
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
|
||
{
|
||
Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 },
|
||
UpdatedAt = DateTimeOffset.UtcNow,
|
||
});
|
||
Assert.Contains(sequence.Id, (await fx.Archive.ScanAsync()).Sequences);
|
||
var theirs = measured with { HeroIndex = measured.HeroIndex == 5 ? 6 : 5 };
|
||
theirs.Save(folder.AnalysisPath);
|
||
var report = await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct);
|
||
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
fx.Decoder.Release();
|
||
await fx.WaitForAsync(overtaken.Id, JobState.Done);
|
||
|
||
// The pass leaves that report in place, and records it rather than its own: the summary in the index is the one of
|
||
// the file every reader opens, and the light analysis that follows keeps it.
|
||
Assert.Equal(report, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct));
|
||
var summary = AnalysisSummary.From(theirs);
|
||
Assert.NotEqual(AnalysisSummary.From(measured), summary);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
|
||
Assert.Equal(summary, AnalysisSummary.FromJson(row.AnalysisJson!));
|
||
await fx.WaitForAsync(light.Id, JobState.Done);
|
||
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
|
||
Assert.Equal(summary, AnalysisSummary.FromJson(row.AnalysisJson!));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnAnalysisOvertakenWhileALightAnalysisWaitsRecordsAndThatOneReplacesIt()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering(12));
|
||
var folder = fx.Folder(sequence);
|
||
await fx.StartEngineAsync();
|
||
|
||
// A deep analysis starts on twelve frames on the only worker; while it runs a thirteenth photo is indexed, and its
|
||
// light analysis waits in the queue, as during an upload burst.
|
||
fx.Decoder.HoldFrame(0);
|
||
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
await fx.AddAsync(Flickering(13), first: 12);
|
||
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
fx.Decoder.Hold = null;
|
||
fx.Decoder.Release();
|
||
// The light analysis waits on the new photo's probe, so what the deep pass left can be seen.
|
||
fx.Decoder.HoldProbeOf(12);
|
||
var done = await fx.WaitForAsync(deep.Id, JobState.Done);
|
||
|
||
// The deep pass records the analysis of the frames it read meanwhile, and queues nothing more.
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Contains(Loc.L("Advisor.Log.DeepDone", sequence.DisplayTitle, 12, row.WarningCount), done.LogTail);
|
||
Assert.False(done.LogTail!.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), StringComparison.Ordinal), done.LogTail);
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
|
||
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
|
||
Assert.SequenceEqual([light.Id], (await LightJobsAsync(fx)).Select(j => j.Id));
|
||
|
||
// The light analysis that waited replaces it with the analysis of all thirteen.
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
fx.Decoder.Release();
|
||
await fx.WaitForAsync(light.Id, JobState.Done);
|
||
Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!);
|
||
}
|
||
|
||
[Test]
|
||
public async Task ADeepAnalysisWhoseFolderAnotherChronosMovedRecordsNothingAndCreatesNoFolder()
|
||
{
|
||
// One decode at a time, so the held one is the only photo being read while the folder moves.
|
||
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { DeepAnalysis = o.DeepAnalysis with { MaxDegreeOfParallelism = 1 } });
|
||
var sequence = await fx.AddAsync(Flickering(12));
|
||
var left = fx.Folder(sequence).Folder;
|
||
var (server, archive) = await fx.Archive.OtherWriterAsync();
|
||
await fx.StartEngineAsync();
|
||
|
||
// A server sharing this desktop's archive reorganizes the sequence while the deep pass reads its photos.
|
||
fx.Decoder.HoldFrame(5);
|
||
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
var plan = await server.PlanRenameAsync(archive.Id, [sequence.Id], "Moved/{date}", null, fx.Ct);
|
||
await server.ApplyRenameAsync(plan, null, fx.Ct);
|
||
fx.Decoder.Hold = null;
|
||
fx.Decoder.Release();
|
||
var done = await fx.WaitForAsync(deep.Id, JobState.Done);
|
||
|
||
Assert.Contains(Loc.L("Advisor.Log.FramesChangedElsewhere", sequence.DisplayTitle), done.LogTail);
|
||
Assert.False(Directory.Exists(left), "the folder it left is not created again");
|
||
var moved = fx.Archive.Paths.Sequence(plan.Steps.Single().ToFolder);
|
||
Assert.FileMissing(moved.AnalysisPath);
|
||
Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedDeep), "nothing is recorded");
|
||
}
|
||
|
||
[Test]
|
||
public async Task ALightAnalysisOvertakenByAScanMarksNothingAnalysedAndRunsAgain()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering(12));
|
||
var folder = fx.Folder(sequence);
|
||
await fx.StartEngineAsync();
|
||
|
||
// A light analysis reads twelve frames; before it records, a scan indexes a thirteenth photo copied into the folder.
|
||
fx.Decoder.HoldProbeOf(0);
|
||
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
await fx.AddAsync(Flickering(13), first: 12);
|
||
fx.Decoder.Release();
|
||
// The light analysis that follows waits on the new photo's probe, so what the first one left can be seen.
|
||
fx.Decoder.HoldProbeOf(12);
|
||
var done = await fx.WaitForAsync(light.Id, JobState.Done);
|
||
|
||
Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(13, row.FrameCount);
|
||
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
|
||
Assert.False(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
|
||
Assert.False(ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.HasAdvice, "no advice for twelve frames");
|
||
var again = Assert.Contains(await LightJobsAsync(fx), j => j.Id != light.Id);
|
||
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
fx.Decoder.Release();
|
||
await fx.WaitForAsync(again.Id, JobState.Done);
|
||
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
|
||
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
|
||
Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnAnalysisOfPhotosAnotherChronosChangedLeavesWhatThatChronosRecorded()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering(12));
|
||
var folder = fx.Folder(sequence);
|
||
await fx.StartEngineAsync();
|
||
// A server sharing this desktop's archive, with its own index and analysis.
|
||
await using var serverDb = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("server"), "chronos.db")), fx.Ct);
|
||
var serverOptions = fx.Processing.Options with
|
||
{
|
||
Archive = new ArchiveServiceOptions { Decoder = new ImageDecoder([new ManagedDecoderProvider()]), Owner = "server", LeaseWait = TimeSpan.FromSeconds(20) },
|
||
};
|
||
var server = new ArchiveService(serverDb, serverOptions.Archive);
|
||
var archive = await server.OpenAsync(fx.Archive.Archive.RootPath, fx.Ct);
|
||
await server.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
|
||
await using var serverEngine = new JobEngine(serverDb, new ProcessingService(serverDb, serverOptions).CreateJobHandlers(),
|
||
new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
|
||
await serverEngine.StartAsync(fx.Ct);
|
||
|
||
// A light analysis here reads twelve frames; before it records, the server takes a thirteenth photo and analyses
|
||
// the sequence as it is now, in the shared sidecar and project.
|
||
fx.Decoder.HoldProbeOf(0);
|
||
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
var staged = fx.Archive.Stage(new DateTime(2024, 6, 21, 19, 2, 0), seed: 90);
|
||
await server.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0013.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
var analysed = await serverEngine.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]), cancellationToken: fx.Ct);
|
||
for (var until = DateTime.UtcNow.AddSeconds(60); (await serverDb.Jobs.GetAsync(analysed.Id, fx.Ct))!.State != JobState.Done;)
|
||
{
|
||
Assert.True(DateTime.UtcNow < until, "the server's light analysis ends");
|
||
await Task.Delay(20, fx.Ct);
|
||
}
|
||
var advice = await ProjectJsonAsync(fx, folder);
|
||
Assert.Contains("13", ProjectSettings.FromJson(advice).Timing.FrameRate.ReasonArgs!);
|
||
var status = ManifestStore.ReadSequence(folder.ManifestPath).Status;
|
||
Assert.True(status.HasFlag(SequenceStatus.AnalyzedLight), "the server's status");
|
||
fx.Decoder.Release();
|
||
var done = await fx.WaitForAsync(light.Id, JobState.Done);
|
||
|
||
// This index still holds twelve frames: the analysis here records nothing over the server's, and queues nothing.
|
||
Assert.Contains(Loc.L("Advisor.Log.FramesChangedElsewhere", sequence.DisplayTitle), done.LogTail);
|
||
Assert.Equal(advice, await ProjectJsonAsync(fx, folder));
|
||
Assert.Equal(status, ManifestStore.ReadSequence(folder.ManifestPath).Status);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(12, row.FrameCount);
|
||
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
|
||
Assert.SequenceEqual([light.Id], (await LightJobsAsync(fx)).Select(j => j.Id));
|
||
|
||
// The next scan here indexes the photo with the server's analysis, which a desktop keeps.
|
||
Assert.Contains(sequence.Id, (await fx.Archive.ScanAsync()).Sequences);
|
||
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(13, row.FrameCount);
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
|
||
Assert.Equal(advice, await ProjectJsonAsync(fx, folder));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering());
|
||
var folder = fx.Folder(sequence);
|
||
// Frame 4 is a HEIF this host reads only from its proxy (BRIEF §2.3); the proxy a client sends shows the photo.
|
||
var original = Path.Combine(folder.Folder, ProcessingFixture.FileName(3));
|
||
var photo = await File.ReadAllBytesAsync(original, fx.Ct);
|
||
await File.WriteAllBytesAsync(original, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8], fx.Ct);
|
||
var frame = (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[3];
|
||
await using (var session = await fx.Db.Database.OpenAsync(fx.Ct))
|
||
await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE id = $2", [(int)ImageFileFormat.Heif, frame.Id], fx.Ct);
|
||
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
|
||
ManifestStore.WriteSequence(folder.ManifestPath, sidecar with
|
||
{
|
||
Frames = [.. sidecar.Frames.Select(f => f.Id == frame.Id ? f with { Format = ImageFileFormat.Heif } : f)],
|
||
});
|
||
fx.Decoder.HoldFrame(25);
|
||
await fx.StartEngineAsync();
|
||
|
||
// A deep analysis reads the original, finds no pixels, and is interrupted once its checkpoint holds that verdict.
|
||
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
await fx.Engine.CancelAsync(job.Id, fx.Ct);
|
||
await fx.WaitForAsync(job.Id, JobState.Cancelled);
|
||
|
||
// The proxy arrives meanwhile: the frame is no longer read as the checkpoint read it, so the resumed run starts over.
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
await File.WriteAllBytesAsync(folder.Proxy(3), photo, fx.Ct);
|
||
fx.Decoder.Hold = null;
|
||
await fx.Engine.ResumeAsync(job.Id, fx.Ct);
|
||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||
var report = SequenceAnalyser.ReadReport(folder, sequence.FrameCount)!;
|
||
Assert.Empty(report.Issues.Undecodable);
|
||
Assert.SequenceEqual([3], report.ProxyFrames!);
|
||
|
||
// Without its proxy, a deep analysis lists the frame as unreadable, a sequence warning...
|
||
File.Delete(folder.Proxy(3));
|
||
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(deep.Id, JobState.Done);
|
||
Assert.SequenceEqual([3], SequenceAnalyser.ReadReport(folder, sequence.FrameCount)!.Issues.Undecodable);
|
||
var unreadable = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.True(unreadable.Status.HasFlag(SequenceStatus.AnalyzedDeep | SequenceStatus.HasWarnings), unreadable.Status.ToString());
|
||
|
||
// ...a verdict that no longer holds once the proxy is there: the next light analysis drops the report and its warning.
|
||
await File.WriteAllBytesAsync(folder.Proxy(3), photo, fx.Ct);
|
||
await LightAnalysisAsync(fx, sequence);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), "the report is stale");
|
||
Assert.Less(row.WarningCount, unreadable.WarningCount);
|
||
}
|
||
|
||
[Test]
|
||
public async Task ACheckpointOfASequenceReadFromItsOriginalsKeepsTheKeyEarlierVersionsWrote()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering(4));
|
||
var frames = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive.Archive, sequence, useProxies: false, new ImageDecoder([fx.Decoder]), fx.Ct);
|
||
Assert.SequenceEqual([false, false, false, false], frames.Proxied!);
|
||
|
||
// A deep analysis interrupted before an upgrade resumes after it (a server restarted mid-run does not start over).
|
||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||
foreach (var f in frames.Frames)
|
||
hash.AppendData(Encoding.UTF8.GetBytes($"{f.Id:N}/{f.Sha256}/{f.Size}/{f.Flags & Chronos.Data.Processing.SequenceFrames.NotAnalysed};"));
|
||
var earlier = Convert.ToHexStringLower(hash.GetHashAndReset())[..16];
|
||
Assert.Equal(earlier, SequenceAnalyser.ContentKey(frames));
|
||
|
||
// A frame read from its proxy since makes the checkpoint stale.
|
||
Assert.NotEqual(earlier, SequenceAnalyser.ContentKey(frames with { Proxied = [false, true, false, false] }));
|
||
}
|
||
|
||
[Test]
|
||
public async Task DeepAnalysisFlagsFramesItCannotDecodeAndListsThemOnce()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering(12));
|
||
// Damaged after indexing: the header probe still says JPEG, only decoding finds out.
|
||
await File.WriteAllBytesAsync(Path.Combine(fx.Folder(sequence).Folder, ProcessingFixture.FileName(4)), new byte[3000], fx.Ct);
|
||
await fx.StartEngineAsync();
|
||
|
||
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
||
|
||
var sidecar = ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Frames;
|
||
Assert.Equal((DecodeTier.Unavailable, FrameFlags.Undecodable), (sidecar[4].Tier, sidecar[4].Flags));
|
||
Assert.Equal(DecodeTier.Native, sidecar[5].Tier);
|
||
Assert.Equal(FrameFlags.Undecodable, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[4].Flags);
|
||
var report = SequenceAnalyser.ReadReport(fx.Folder(sequence), sequence.FrameCount)!;
|
||
var rows = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
var findings = Advisor.Recommend(LightAnalyzer.Analyze(rows), report, SequenceContext.FromFrames(rows)).Findings;
|
||
Assert.Count(1, findings.Where(f => f.Key == DeepFindingKeys.UnreadableFrames));
|
||
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.WarningCount >= 1, "listed in the sequence warnings");
|
||
Assert.Equal(JobState.Done, done.State);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnImportedGoProRawDecodesThroughTheJpegItsImportKeptInAnotherSequence()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
// A GoPro card: each GPR (VC-5, without a usable embedded image here) has the camera's full-size JPEG beside it.
|
||
// An upload may send the GPRs first: they arrive as two cards here.
|
||
var raws = Path.Combine(context.CreateTempDirectory("card"), "DCIM", "100GOPRO");
|
||
var jpegs = Path.Combine(context.CreateTempDirectory("card"), "DCIM", "100GOPRO");
|
||
Directory.CreateDirectory(raws);
|
||
Directory.CreateDirectory(jpegs);
|
||
var start = new DateTime(2026, 9, 28, 18, 0, 0);
|
||
for (var i = 0; i < 4; i++)
|
||
{
|
||
var metadata = new FrameMetadata { Make = "GoPro", Model = "HERO12 Black", BodySerial = "C3501", CapturedAt = start.AddSeconds(5 * i) };
|
||
using var sensor = new ImageBuffer(64, 48);
|
||
File.WriteAllBytes(Path.Combine(raws, $"GOPR{i + 1:0000}.GPR"),
|
||
LinearDngWriter.Write(sensor, new LinearDngOptions { Exif = ExifWriter.BuildTiff(metadata), TileSize = 64 }));
|
||
using var picture = new ImageBuffer(160, 120);
|
||
picture.Plane(1).Fill(0.25f * (i + 1));
|
||
File.WriteAllBytes(Path.Combine(jpegs, $"GOPR{i + 1:0000}.JPG"),
|
||
JpegEncoder.Encode(picture, new JpegEncodeOptions { App1Segments = [ExifWriter.BuildJpegApp1(metadata)[4..]] }));
|
||
}
|
||
var service = fx.Archive.Service;
|
||
async Task<ImportPlan> ImportAsync(string card)
|
||
{
|
||
var plan = await service.PlanImportAsync(fx.Archive.Archive.Id, Path.GetDirectoryName(Path.GetDirectoryName(card))!, new ImportOptions(),
|
||
cancellationToken: fx.Ct);
|
||
await service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Archive.Id, plan, null, true, null), Guid.NewGuid(), 0,
|
||
_ => ValueTask.CompletedTask, null, fx.Ct);
|
||
return plan;
|
||
}
|
||
var gprSession = (await ImportAsync(raws)).Sessions.Single();
|
||
Assert.Equal(ImageFileFormat.Gpr, gprSession.Format);
|
||
var id = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Archive.Id, gprSession.SessionKey, fx.Ct);
|
||
var sequence = (await fx.Db.Sequences.GetAsync(id!.Value, fx.Ct))!;
|
||
await fx.StartEngineAsync();
|
||
|
||
// Alone, a GPR without an embedded preview gives no pixels here, which proves nothing about the file.
|
||
var first = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(first.Id, JobState.Done);
|
||
var alone = ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Frames;
|
||
Assert.True(alone.All(f => f is { Tier: DecodeTier.Unavailable, Flags: FrameFlags.None, PreviewW: null }), "unavailable, not flagged");
|
||
|
||
// The JPEGs arrive as a sequence of their own (the import splits per file format, DECISIONS 2026-10-02); the next
|
||
// probe finds each GPR's JPEG there, so the output cap is the JPEG's size.
|
||
Assert.Equal(ImageFileFormat.Jpeg, (await ImportAsync(jpegs)).Sessions.Single().Format);
|
||
var jpegFrame = (await fx.Db.Frames.ListByCaptureRangeAsync(fx.Archive.Archive.Id, ImageFileFormat.Jpeg, start, start.AddMinutes(1), fx.Ct))[0];
|
||
Assert.NotEqual(Path.GetDirectoryName(fx.Folder(sequence).File(alone[0].File)), Path.GetDirectoryName(fx.Archive.Paths.FromRelative(jpegFrame.RelativePath)),
|
||
"not beside the GPR");
|
||
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(light.Id, JobState.Done);
|
||
var sidecar = ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Frames;
|
||
Assert.True(sidecar.All(f => f is { Tier: DecodeTier.EmbeddedPreview, Flags: FrameFlags.PreviewOnly, PreviewW: 160, PreviewH: 120 }),
|
||
"preview-only at the JPEG's size");
|
||
|
||
// The deep pass decodes the GPRs through those JPEGs (decoder "managed-sidecar"), never failing a frame.
|
||
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(deep.Id, JobState.Done);
|
||
var gprs = fx.Decoder.Decoded.Where(d => d.File.EndsWith(".GPR", StringComparison.OrdinalIgnoreCase)).ToList();
|
||
Assert.NotEmpty(gprs);
|
||
Assert.True(gprs.All(d => d.Decoder == "managed-sidecar"), string.Join(", ", gprs.Select(d => d.Decoder).Distinct()));
|
||
Assert.Empty(SequenceAnalyser.ReadReport(fx.Folder(sequence), sequence.FrameCount)!.Issues.Undecodable);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The owner's archive (R4, owner review 1): GoPro DNGs converted by Adobe software were indexed as GPR and recorded
|
||
/// preview-only. The scan follow-up lists their sequences again, and the light analysis probes them: they become
|
||
/// DNGs decoded natively, in the sidecar and the index.
|
||
/// </summary>
|
||
[Test]
|
||
public async Task GoProDngsIndexedAsGprAreProbedAgainAndBecomeNativeDngs()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var card = Path.Combine(context.CreateTempDirectory("card"), "DCIM", "100GOPRO");
|
||
Directory.CreateDirectory(card);
|
||
var start = new DateTime(2024, 10, 11, 18, 0, 0);
|
||
for (var i = 0; i < 3; i++)
|
||
{
|
||
var metadata = new FrameMetadata { Make = "GoPro", Model = "HERO8 Black", BodySerial = "C3441", CapturedAt = start.AddSeconds(5 * i) };
|
||
using var sensor = new ImageBuffer(64, 48);
|
||
sensor.Plane(1).Fill(0.2f + 0.1f * i);
|
||
File.WriteAllBytes(Path.Combine(card, $"GOPR{i + 1:0000}.dng"),
|
||
LinearDngWriter.Write(sensor, new LinearDngOptions { Exif = ExifWriter.BuildTiff(metadata), TileSize = 64 }));
|
||
}
|
||
var service = fx.Archive.Service;
|
||
var plan = await service.PlanImportAsync(fx.Archive.Archive.Id, Path.GetDirectoryName(Path.GetDirectoryName(card))!, new ImportOptions(),
|
||
cancellationToken: fx.Ct);
|
||
await service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Archive.Id, plan, null, true, null), Guid.NewGuid(), 0,
|
||
_ => ValueTask.CompletedTask, null, fx.Ct);
|
||
var session = plan.Sessions.Single();
|
||
Assert.Equal(ImageFileFormat.Dng, session.Format);
|
||
var id = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Archive.Id, session.SessionKey, fx.Ct);
|
||
var sequence = (await fx.Db.Sequences.GetAsync(id!.Value, fx.Ct))!;
|
||
|
||
// What an index from before owner review 1 holds: GPR, preview-only, already light-analysed.
|
||
var folder = fx.Folder(sequence);
|
||
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
|
||
{
|
||
Frames = [.. manifest.Frames.Select(f => f with { Format = ImageFileFormat.Gpr, Tier = DecodeTier.EmbeddedPreview, Flags = f.Flags | FrameFlags.PreviewOnly, PreviewW = 64, PreviewH = 48 })],
|
||
});
|
||
var rows = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
await fx.Db.Frames.UpdateTiersAsync([.. rows.Select(r => new FrameTierFields(r.Id, r.Flags | FrameFlags.PreviewOnly, 64, 48, ImageFileFormat.Gpr))], fx.Ct);
|
||
await fx.Db.Sequences.UpsertAsync(sequence with { Status = sequence.Status | SequenceStatus.AnalyzedLight }, fx.Ct);
|
||
Assert.Contains(sequence.Id, await AnalysisFollowUp.UnanalysedAsync(fx.Db, fx.Archive.Archive.Id, fx.Ct));
|
||
|
||
await fx.StartEngineAsync();
|
||
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(light.Id, JobState.Done);
|
||
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
|
||
Assert.True(sidecar.All(f => f is { Format: ImageFileFormat.Dng, Tier: DecodeTier.Native } && (f.Flags & FrameFlags.PreviewOnly) == 0),
|
||
string.Join(", ", sidecar.Select(f => $"{f.Format}/{f.Tier}/{f.Flags}")));
|
||
var indexed = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
Assert.True(indexed.All(f => f.Format == ImageFileFormat.Dng && (f.Flags & FrameFlags.PreviewOnly) == 0), "the index follows");
|
||
Assert.Equal(0, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
Assert.DoesNotContain(sequence.Id, await AnalysisFollowUp.UnanalysedAsync(fx.Db, fx.Archive.Archive.Id, fx.Ct));
|
||
}
|
||
|
||
[Test]
|
||
public async Task DeepAnalysisNeverFlagsFramesThatVanishedOrCouldNotBeRead()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering(12));
|
||
// Deleted after indexing (a verify flags it missing), and held by another program during the pass.
|
||
File.Delete(Path.Combine(fx.Folder(sequence).Folder, ProcessingFixture.FileName(3)));
|
||
fx.Decoder.Unreadable[ProcessingFixture.FileName(6)] = 0;
|
||
await fx.StartEngineAsync();
|
||
|
||
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||
|
||
// The pass could not read them, so it lists them as unreadable, but nothing proves the files undecodable.
|
||
var report = SequenceAnalyser.ReadReport(fx.Folder(sequence), sequence.FrameCount)!;
|
||
Assert.SequenceEqual([3, 6], report.Issues.Undecodable);
|
||
var sidecar = ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Frames;
|
||
foreach (var index in new[] { 3, 6 })
|
||
{
|
||
Assert.Equal(FrameFlags.None, sidecar[index].Flags & FrameFlags.Undecodable);
|
||
Assert.NotEqual(DecodeTier.Unavailable, sidecar[index].Tier);
|
||
}
|
||
var rows = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
Assert.True(rows.All(f => (f.Flags & FrameFlags.Undecodable) == 0), "no frame flagged in the index");
|
||
var unreadable = Advisor.Recommend(LightAnalyzer.Analyze(rows), report, SequenceContext.FromFrames(rows)).Findings
|
||
.Single(f => f.Key == DeepFindingKeys.UnreadableFrames);
|
||
Assert.SequenceEqual([3, 6], unreadable.Frames);
|
||
|
||
// Back in place and readable, the next pass reads both frames.
|
||
fx.Decoder.Unreadable.Clear();
|
||
await File.WriteAllBytesAsync(Path.Combine(fx.Folder(sequence).Folder, ProcessingFixture.FileName(3)),
|
||
await File.ReadAllBytesAsync(Path.Combine(fx.Folder(sequence).Folder, ProcessingFixture.FileName(2)), fx.Ct), fx.Ct);
|
||
var again = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(again.Id, JobState.Done);
|
||
Assert.Empty(SequenceAnalyser.ReadReport(fx.Folder(sequence), sequence.FrameCount)!.Issues.Undecodable);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AResumedDeepAnalysisDecodesAgainWhatTheEarlierRunCouldNotRead()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering());
|
||
// The first run cannot read frame 12 (held by another program) and is cancelled later, at frame 26.
|
||
fx.Decoder.Unreadable[ProcessingFixture.FileName(11)] = 0;
|
||
fx.Decoder.HoldFrame(25);
|
||
await fx.StartEngineAsync();
|
||
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
await fx.Engine.CancelAsync(job.Id, fx.Ct);
|
||
await fx.WaitForAsync(job.Id, JobState.Cancelled);
|
||
|
||
// Readable again; the resumed run takes frame 12 from its checkpoint (it is not in the region sample).
|
||
fx.Decoder.Unreadable.Clear();
|
||
fx.Decoder.Hold = null;
|
||
while (fx.Decoder.Requests.TryDequeue(out _))
|
||
{
|
||
}
|
||
await fx.Engine.ResumeAsync(job.Id, fx.Ct);
|
||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||
|
||
Assert.Contains(11, SequenceAnalyser.ReadReport(fx.Folder(sequence), sequence.FrameCount)!.Issues.Undecodable);
|
||
// That verdict was a read error: decoding the frame again shows it is fine, so nothing flags it.
|
||
Assert.Count(1, fx.Decoder.Requests.Where(r => Path.GetFileName(r.Path) == ProcessingFixture.FileName(11)));
|
||
var frame = ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Frames[11];
|
||
Assert.Equal((DecodeTier.Native, FrameFlags.None), (frame.Tier, frame.Flags & FrameFlags.Undecodable));
|
||
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[11].Flags);
|
||
}
|
||
|
||
[Test]
|
||
public async Task DeepAnalysisResumesAfterCancellationWithTheSameReport()
|
||
{
|
||
await using var reference = await ProcessingFixture.CreateAsync(context);
|
||
var expectedSequence = await reference.AddAsync(Flickering());
|
||
await reference.StartEngineAsync();
|
||
var full = await reference.EnqueueAsync(ProcessingJobs.DeepAnalysis([expectedSequence.Id]));
|
||
await reference.WaitForAsync(full.Id, JobState.Done);
|
||
var expected = await File.ReadAllTextAsync(reference.Folder(expectedSequence).AnalysisPath, reference.Ct);
|
||
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering());
|
||
var folder = fx.Folder(sequence);
|
||
var partial = Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName);
|
||
// Frame 25 is not in the region sample, so the hold stops the pass inside the batch of frames 24–25.
|
||
fx.Decoder.HoldFrame(25);
|
||
await fx.StartEngineAsync();
|
||
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
await fx.Engine.CancelAsync(job.Id, fx.Ct);
|
||
await fx.WaitForAsync(job.Id, JobState.Cancelled);
|
||
var saved = File.ReadAllLines(partial).Length - 1;
|
||
Assert.Equal(24, saved);
|
||
Assert.FileMissing(folder.AnalysisPath);
|
||
|
||
fx.Decoder.Hold = null;
|
||
while (fx.Decoder.Requests.TryDequeue(out _))
|
||
{
|
||
}
|
||
await fx.Engine.ResumeAsync(job.Id, fx.Ct);
|
||
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
||
|
||
Assert.Equal(expected, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct));
|
||
Assert.FileMissing(partial);
|
||
// After the region sample, only the last checkpointed frame is decoded again (to restore the comparison state).
|
||
var analysed = fx.Decoder.Requests.Skip(ProcessingFixture.DeepOptions.RegionSampleFrames)
|
||
.Select(r => int.Parse(Path.GetFileNameWithoutExtension(r.Path)[4..]) - 1).ToList();
|
||
Assert.Equal(saved - 1, analysed.Min());
|
||
Assert.Equal(sequence.FrameCount, (int)done.ProgressDone);
|
||
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep | SequenceStatus.AnalyzedLight), "index status");
|
||
var summary = AnalysisSummary.FromJson(row.AnalysisJson!);
|
||
Assert.Equal(sequence.FrameCount, summary.FrameCount);
|
||
Assert.NotNull((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[5].MeanLogLuminance);
|
||
var project = ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct));
|
||
Assert.True(project.Exposure.Deflicker.HasAdvice, "deflicker advised from the pixels");
|
||
Assert.Contains(Loc.L("Advisor.Log.Presets", string.Empty), done.LogTail);
|
||
}
|
||
|
||
[Test]
|
||
public async Task EachDeepAnalysisTakesTheHostsTuningOfThatMoment()
|
||
{
|
||
// The desktop's Settings › Analysis: the width cap the user sets now applies to the next analysis.
|
||
var cap = 40;
|
||
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with
|
||
{
|
||
Tune = basis => basis with { DeepAnalysis = basis.DeepAnalysis with { MinAnalysisLongEdge = 16, MaxAnalysisLongEdge = cap } },
|
||
});
|
||
var first = await fx.AddAsync(Flickering(10), "A");
|
||
var second = await fx.AddAsync(Flickering(10), "B");
|
||
await fx.StartEngineAsync();
|
||
|
||
var job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([first.Id]));
|
||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||
Assert.Equal(40, DeepAnalysisReport.Load(fx.Folder(first).AnalysisPath).AnalysisLongEdge);
|
||
|
||
cap = 32;
|
||
job = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([second.Id]));
|
||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||
Assert.Equal(32, DeepAnalysisReport.Load(fx.Folder(second).AnalysisPath).AnalysisLongEdge);
|
||
}
|
||
|
||
[Test]
|
||
public async Task ASequenceThatFailsDoesNotStopTheOthersAndARetryRedoesOnlyIt()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var a = await fx.AddAsync(Flickering(10), "A");
|
||
var b = await fx.AddAsync(Flickering(10), "B");
|
||
var c = await fx.AddAsync(Flickering(10), "C");
|
||
// B's sidecar is gone for a while: its status cannot be recorded.
|
||
var sidecar = fx.Folder(b).ManifestPath;
|
||
var saved = await File.ReadAllBytesAsync(sidecar, fx.Ct);
|
||
File.Delete(sidecar);
|
||
await fx.StartEngineAsync();
|
||
|
||
var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([a.Id, b.Id, c.Id]));
|
||
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
|
||
Assert.True(failed.Error!.StartsWith(Loc.L("Advisor.Log.Failed", b.DisplayTitle, string.Empty), StringComparison.Ordinal), failed.Error);
|
||
Assert.Equal("3;1", failed.Checkpoint);
|
||
Assert.SequenceEqual([true, false, true], await AnalysedAsync(fx, a, b, c));
|
||
|
||
await File.WriteAllBytesAsync(sidecar, saved, fx.Ct);
|
||
await fx.Engine.ResumeAsync(job.Id, fx.Ct);
|
||
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
||
Assert.SequenceEqual([true, true, true], await AnalysedAsync(fx, a, b, c));
|
||
Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal));
|
||
// The retry analysed B alone.
|
||
var lightOfA = Loc.L("Advisor.Log.LightDone", a.DisplayTitle, 10, 0).Split(',')[0];
|
||
Assert.Equal(1, done.LogTail!.Split('\n').Count(line => line.Contains(lightOfA, StringComparison.Ordinal)));
|
||
}
|
||
|
||
[Test]
|
||
public void AnalysisCheckpointsKeepTheFailedTargets()
|
||
{
|
||
Assert.Equal("3", AnalyseJobHandler.Format(3, []));
|
||
Assert.Equal("4;0,2", AnalyseJobHandler.Format(4, [2, 0]));
|
||
var (next, retry) = AnalyseJobHandler.Parse("4;0,2", 5);
|
||
Assert.Equal(4, next);
|
||
Assert.SequenceEqual([0, 2], retry);
|
||
Assert.Equal(2, AnalyseJobHandler.Parse("2", 5).Next);
|
||
// Anything else starts over: no checkpoint, beyond the targets, a failed target not yet reached, garbage.
|
||
foreach (var text in new[] { null, "6", "2;3", "2;x", "a" }) Assert.Equal(0, AnalyseJobHandler.Parse(text, 5).Next);
|
||
}
|
||
|
||
private static async Task LightAnalysisAsync(ProcessingFixture fx, Sequence sequence)
|
||
{
|
||
var job = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||
}
|
||
|
||
/// <summary>A JPEG (320 × 240 unless sized) with <paramref name="captured"/> in its Exif, as a client makes a proxy.</summary>
|
||
private static byte[] Proxy(DateTime captured, int width = 320, int height = 240)
|
||
{
|
||
using var image = new ImageBuffer(width, height);
|
||
for (var c = 0; c < 3; c++) image.Plane(c).Fill(0.3f + c * 0.1f);
|
||
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Chronos" });
|
||
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
|
||
}
|
||
|
||
private static async Task<IReadOnlyList<Job>> LightJobsAsync(ProcessingFixture fx) =>
|
||
(await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, 100, null, fx.Ct)).Items;
|
||
|
||
/// <summary>The project as <c>project.json</c> holds it (the defaults before any analysis wrote it).</summary>
|
||
private static async Task<string> ProjectJsonAsync(ProcessingFixture fx, SequencePaths folder) =>
|
||
(File.Exists(folder.ProjectPath) ? ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)) : new ProjectSettings()).ToJson();
|
||
|
||
private static async Task<List<bool>> AnalysedAsync(ProcessingFixture fx, params Sequence[] sequences)
|
||
{
|
||
var analysed = new List<bool>();
|
||
foreach (var s in sequences) analysed.Add((await fx.Db.Sequences.GetAsync(s.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedLight));
|
||
return analysed;
|
||
}
|
||
|
||
[Test]
|
||
public async Task LightAnalysisAfterNewFramesDropsTheStaleDeepStatus()
|
||
{
|
||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||
var sequence = await fx.AddAsync(Flickering(12));
|
||
await fx.StartEngineAsync();
|
||
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(deep.Id, JobState.Done);
|
||
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedDeep), "deep status");
|
||
|
||
// A thirteenth frame arrives: the report describes 12 frames, so it no longer counts.
|
||
await fx.AddAsync(Flickering(13), first: 12);
|
||
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
|
||
await fx.WaitForAsync(light.Id, JobState.Done);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(13, row.FrameCount);
|
||
Assert.Equal((true, false), (row.Status.HasFlag(SequenceStatus.AnalyzedLight), row.Status.HasFlag(SequenceStatus.AnalyzedDeep)));
|
||
}
|
||
}
|