A light or deep analysis read its frames at the start and, under the sequence lock, recorded AnalyzedLight, the warning count and the advisor's Auto values for them whatever happened meanwhile. Frames added by an upload, an import or a scan's reconcile then kept advice and warnings counted on the old frame count, over a newer light analysis, and scans never queued another one because AnalyzedLight was set again. Under the lock the analysis now compares the row's frames version with the one it read. When it moved and no light analysis is queued for the sequence, it records nothing (a deep pass keeps only analysis.json and its per-frame measurements), and the job logs it and queues a light analysis of the frames as they are. A queued light analysis replaces what the pass records, as for image passes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
198 lines
9.6 KiB
C#
198 lines
9.6 KiB
C#
using System.Collections.Concurrent;
|
|
using Titano.Core.Analysis.Deep;
|
|
using Titano.Core.Archiving;
|
|
using Titano.Core.Codecs.Jpeg;
|
|
using Titano.Core.Diagnostics.Synthetic;
|
|
using Titano.Core.Imaging;
|
|
using Titano.Core.Imaging.Decoders;
|
|
using Titano.Core.Metadata;
|
|
using Titano.Core.Model;
|
|
using Titano.Data.Archiving;
|
|
using Titano.Data.Jobs;
|
|
using Titano.Data.Processing;
|
|
using Titano.Data.Tests.Archiving;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Data.Tests.Processing;
|
|
|
|
/// <summary>A temporary archive holding synthetic sequences (JPEGs with Exif), a processing engine and its job engine.</summary>
|
|
internal sealed class ProcessingFixture : IAsyncDisposable
|
|
{
|
|
/// <summary>Small sources are analysed at native size; batches of two frames between checkpoints.</summary>
|
|
public static readonly DeepAnalysisOptions DeepOptions = new() { MinAnalysisLongEdge = 64, FlowEvery = 5, MaxDegreeOfParallelism = 2, RegionSampleFrames = 9 };
|
|
|
|
private JobEngine? _engine;
|
|
|
|
private ProcessingFixture(ArchiveFixture archive, GateDecoder decoder, Func<ProcessingOptions, ProcessingOptions>? configure)
|
|
{
|
|
Archive = archive;
|
|
Decoder = decoder;
|
|
var options = new ProcessingOptions
|
|
{
|
|
Archive = new ArchiveServiceOptions { Decoder = new ImageDecoder([decoder]), Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) },
|
|
DeepAnalysis = DeepOptions,
|
|
ProducedBy = "test-pc",
|
|
};
|
|
Processing = new ProcessingService(archive.Db, configure?.Invoke(options) ?? options);
|
|
}
|
|
|
|
public ArchiveFixture Archive { get; }
|
|
public GateDecoder Decoder { get; }
|
|
public ProcessingService Processing { get; }
|
|
public TitanoDatabase Db => Archive.Db;
|
|
public CancellationToken Ct => Archive.Ct;
|
|
|
|
public JobEngine Engine => _engine ?? throw new InvalidOperationException("Start the engine first.");
|
|
|
|
/// <param name="configure">Changes to the processing options (a server's host and render-node check).</param>
|
|
public static async Task<ProcessingFixture> CreateAsync(TestContext context, Func<ProcessingOptions, ProcessingOptions>? configure = null) =>
|
|
new(await ArchiveFixture.CreateAsync(context), new GateDecoder(), configure);
|
|
|
|
/// <summary>A started engine with the processing handlers, and the archive's when asked (one worker unless asked, fast checkpoints).</summary>
|
|
public async Task<JobEngine> StartEngineAsync(bool withArchive = false, int workers = 1)
|
|
{
|
|
_engine = new JobEngine(Db, [.. withArchive ? Archive.Service.CreateJobHandlers() : [], .. Processing.CreateJobHandlers()],
|
|
new JobEngineOptions { Workers = workers, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
|
|
await _engine.StartAsync(Ct);
|
|
return _engine;
|
|
}
|
|
|
|
public Task<Job> EnqueueAsync(Titano.Data.Archiving.Jobs.ArchiveJobRequest request) => Engine.EnqueueAsync(request, cancellationToken: Ct);
|
|
|
|
public Task<Job> WaitForAsync(Guid jobId, JobState state) => Archive.WaitForAsync(jobId, state, 60);
|
|
|
|
/// <summary>Writes frames <paramref name="first"/>… of <paramref name="sequence"/> as JPEGs with their Exif into
|
|
/// <paramref name="folder"/> (adopted in place) and indexes the archive.</summary>
|
|
public async Task<Sequence> AddAsync(SyntheticSequence sequence, string folder = "Synthetic", int first = 0)
|
|
{
|
|
Write(sequence, folder, first);
|
|
await Archive.ScanAsync(ScanMode.Full);
|
|
var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder);
|
|
return (await Db.Sequences.GetAsync(location.Id, Ct))!;
|
|
}
|
|
|
|
/// <summary>Writes frames <paramref name="first"/>… of <paramref name="sequence"/> as JPEGs with their Exif into <paramref name="folder"/>.</summary>
|
|
public void Write(SyntheticSequence sequence, string folder = "Synthetic", int first = 0)
|
|
{
|
|
var directory = Path.Combine(Archive.Paths.Root, folder);
|
|
Directory.CreateDirectory(directory);
|
|
for (var i = first; i < sequence.Count; i++)
|
|
{
|
|
using var image = sequence.Render(i);
|
|
var exif = ExifWriter.BuildJpegApp1(sequence.Metadata[i]);
|
|
File.WriteAllBytes(Path.Combine(directory, FileName(i)), JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 95, App1Segments = [exif[4..]] }));
|
|
}
|
|
}
|
|
|
|
public static string FileName(int index) => $"IMG_{index + 1:0000}.JPG";
|
|
|
|
public SequencePaths Folder(Sequence sequence) => Archive.Paths.Sequence(sequence.Folder);
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
if (_engine is not null) await _engine.DisposeAsync();
|
|
await Archive.DisposeAsync();
|
|
}
|
|
}
|
|
|
|
/// <summary>Managed decoding that records every request and can hold one file's decode until the job is cancelled or the decode is released.</summary>
|
|
internal sealed class GateDecoder : IImageDecoderProvider
|
|
{
|
|
private readonly ManagedDecoderProvider _managed = new();
|
|
|
|
public string Name => "gate";
|
|
|
|
public ConcurrentQueue<DecodeRequest> Requests { get; } = new();
|
|
|
|
/// <summary>File name whose decode waits for cancellation; null decodes everything.</summary>
|
|
public string? Hold { get; set; }
|
|
|
|
/// <summary>Holds only decodes of this purpose (null: any).</summary>
|
|
public DecodePurpose? HoldPurpose { get; set; }
|
|
|
|
/// <summary>Holds only decodes with this long edge (null: any).</summary>
|
|
public int? HoldEdge { get; set; }
|
|
|
|
/// <summary>Completed when a decode of <see cref="Hold"/> started waiting.</summary>
|
|
public TaskCompletionSource Holding { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
private TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
public void HoldFrame(int index, DecodePurpose? purpose = null, int? edge = null)
|
|
{
|
|
Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
_released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
(HoldPurpose, HoldEdge) = (purpose, edge);
|
|
Hold = ProcessingFixture.FileName(index);
|
|
}
|
|
|
|
/// <summary>File name whose tier probe waits until <see cref="Release"/> (a light analysis probes frames whose tier is
|
|
/// not known yet, and decodes nothing); null probes everything.</summary>
|
|
public string? HoldProbe { get; private set; }
|
|
|
|
/// <summary>Holds the next tier probe of frame <paramref name="index"/>'s file, as <see cref="HoldFrame"/> holds a decode.</summary>
|
|
public void HoldProbeOf(int index)
|
|
{
|
|
Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
_released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
HoldProbe = ProcessingFixture.FileName(index);
|
|
}
|
|
|
|
/// <summary>Lets the held decode go on, as a slow read would.</summary>
|
|
public void Release() => _released.TrySetResult();
|
|
|
|
/// <summary>Tiers <see cref="Probe(string, ImageFileFormat)"/> reports per file name instead of the managed provider's (a
|
|
/// host that reads them only as previews, or not at all).</summary>
|
|
public ConcurrentDictionary<string, FileTier> Tiers { get; } = new();
|
|
|
|
/// <summary>Files (by name) whose decodes report the embedded-preview tier from a preview of this oriented size, as
|
|
/// a size-limited decode that picked a smaller preview would.</summary>
|
|
public ConcurrentDictionary<string, (int Width, int Height)> PreviewDecodes { get; } = new();
|
|
|
|
/// <summary>Files (by name) that cannot be read: their decodes throw <see cref="IOException"/> (a sharing violation).</summary>
|
|
public ConcurrentDictionary<string, byte> Unreadable { get; } = new();
|
|
|
|
/// <summary>File name and decoder of every decode that produced pixels.</summary>
|
|
public ConcurrentQueue<(string File, string Decoder)> Decoded { get; } = new();
|
|
|
|
public DecodeTier? TierFor(ImageFileFormat format) => _managed.TierFor(format);
|
|
|
|
public FileTier Probe(string path, ImageFileFormat format) => Probe(path, format, null);
|
|
|
|
public FileTier Probe(string path, ImageFileFormat format, string? companion)
|
|
{
|
|
var name = Path.GetFileName(path);
|
|
if (HoldProbe is { } hold && name == hold)
|
|
{
|
|
Holding.TrySetResult();
|
|
_released.Task.Wait(TimeSpan.FromSeconds(30));
|
|
}
|
|
return Tiers.TryGetValue(name, out var tier) ? tier : _managed.Probe(path, format, companion);
|
|
}
|
|
|
|
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default)
|
|
{
|
|
Requests.Enqueue(request);
|
|
var name = Path.GetFileName(request.Path);
|
|
if (Hold is { } hold && name == hold && (HoldPurpose ?? request.Purpose) == request.Purpose
|
|
&& (HoldEdge ?? request.MaxLongEdge) == request.MaxLongEdge)
|
|
{
|
|
Holding.TrySetResult();
|
|
try
|
|
{
|
|
_released.Task.Wait(TimeSpan.FromSeconds(30), cancellationToken);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
}
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
}
|
|
if (Unreadable.ContainsKey(name)) throw new IOException($"The process cannot access the file '{request.Path}' because it is being used by another process.");
|
|
var decoded = _managed.Decode(request, cancellationToken);
|
|
if (decoded is null) return null;
|
|
Decoded.Enqueue((name, decoded.Decoder));
|
|
if (!PreviewDecodes.TryGetValue(name, out var preview)) return decoded;
|
|
return new DecodedImage(decoded.Pixels, DecodeTier.EmbeddedPreview, "test-preview", preview.Width, preview.Height);
|
|
}
|
|
}
|