using System.Collections.Concurrent;
using Titano.Core.Analysis.Deep;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Diagnostics.Synthetic;
using Titano.Core.Imaging;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Jobs;
using Titano.Data.Processing;
using Titano.Data.Tests.Archiving;
using Titano.Testing;
namespace Titano.Data.Tests.Processing;
/// A temporary archive holding synthetic sequences (JPEGs with Exif), a processing engine and its job engine.
internal sealed class ProcessingFixture : IAsyncDisposable
{
/// Small sources are analysed at native size; batches of two frames between checkpoints.
public static readonly DeepAnalysisOptions DeepOptions = new() { MinAnalysisLongEdge = 64, FlowEvery = 5, MaxDegreeOfParallelism = 2, RegionSampleFrames = 9 };
private JobEngine? _engine;
private ProcessingFixture(ArchiveFixture archive, GateDecoder decoder, Func? configure)
{
Archive = archive;
Decoder = decoder;
var options = new ProcessingOptions
{
Archive = new ArchiveServiceOptions { Decoder = new ImageDecoder([decoder]), Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) },
DeepAnalysis = DeepOptions,
ProducedBy = "test-pc",
};
Processing = new ProcessingService(archive.Db, configure?.Invoke(options) ?? options);
}
public ArchiveFixture Archive { get; }
public GateDecoder Decoder { get; }
public ProcessingService Processing { get; }
public TitanoDatabase Db => Archive.Db;
public CancellationToken Ct => Archive.Ct;
public JobEngine Engine => _engine ?? throw new InvalidOperationException("Start the engine first.");
/// Changes to the processing options (a server's host and render-node check).
public static async Task CreateAsync(TestContext context, Func? configure = null) =>
new(await ArchiveFixture.CreateAsync(context), new GateDecoder(), configure);
/// A started engine with the processing handlers, and the archive's when asked (one worker unless asked, fast checkpoints).
public async Task StartEngineAsync(bool withArchive = false, int workers = 1)
{
_engine = new JobEngine(Db, [.. withArchive ? Archive.Service.CreateJobHandlers() : [], .. Processing.CreateJobHandlers()],
new JobEngineOptions { Workers = workers, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await _engine.StartAsync(Ct);
return _engine;
}
public Task EnqueueAsync(Titano.Data.Archiving.Jobs.ArchiveJobRequest request) => Engine.EnqueueAsync(request, cancellationToken: Ct);
public Task WaitForAsync(Guid jobId, JobState state) => Archive.WaitForAsync(jobId, state, 60);
/// Writes frames … of as JPEGs with their Exif into
/// (adopted in place) and indexes the archive.
public async Task AddAsync(SyntheticSequence sequence, string folder = "Synthetic", int first = 0)
{
Write(sequence, folder, first);
await Archive.ScanAsync(ScanMode.Full);
var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder);
return (await Db.Sequences.GetAsync(location.Id, Ct))!;
}
/// Writes frames … of as JPEGs with their Exif into .
public void Write(SyntheticSequence sequence, string folder = "Synthetic", int first = 0)
{
var directory = Path.Combine(Archive.Paths.Root, folder);
Directory.CreateDirectory(directory);
for (var i = first; i < sequence.Count; i++)
{
using var image = sequence.Render(i);
var exif = ExifWriter.BuildJpegApp1(sequence.Metadata[i]);
File.WriteAllBytes(Path.Combine(directory, FileName(i)), JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 95, App1Segments = [exif[4..]] }));
}
}
public static string FileName(int index) => $"IMG_{index + 1:0000}.JPG";
public SequencePaths Folder(Sequence sequence) => Archive.Paths.Sequence(sequence.Folder);
public async ValueTask DisposeAsync()
{
if (_engine is not null) await _engine.DisposeAsync();
await Archive.DisposeAsync();
}
}
/// Managed decoding that records every request and can hold one file's decode until the job is cancelled or the decode is released.
internal sealed class GateDecoder : IImageDecoderProvider
{
private readonly ManagedDecoderProvider _managed = new();
public string Name => "gate";
public ConcurrentQueue Requests { get; } = new();
/// File name whose decode waits for cancellation; null decodes everything.
public string? Hold { get; set; }
/// Holds only decodes of this purpose (null: any).
public DecodePurpose? HoldPurpose { get; set; }
/// Holds only decodes with this long edge (null: any).
public int? HoldEdge { get; set; }
/// Completed when a decode of started waiting.
public TaskCompletionSource Holding { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
private TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void HoldFrame(int index, DecodePurpose? purpose = null, int? edge = null)
{
Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
_released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
(HoldPurpose, HoldEdge) = (purpose, edge);
Hold = ProcessingFixture.FileName(index);
}
/// Lets the held decode go on, as a slow read would.
public void Release() => _released.TrySetResult();
/// Tiers reports per file name instead of the managed provider's (a
/// host that reads them only as previews, or not at all).
public ConcurrentDictionary Tiers { get; } = new();
/// Files (by name) whose decodes report the embedded-preview tier from a preview of this oriented size, as
/// a size-limited decode that picked a smaller preview would.
public ConcurrentDictionary PreviewDecodes { get; } = new();
/// Files (by name) that cannot be read: their decodes throw (a sharing violation).
public ConcurrentDictionary Unreadable { get; } = new();
/// File name and decoder of every decode that produced pixels.
public ConcurrentQueue<(string File, string Decoder)> Decoded { get; } = new();
public DecodeTier? TierFor(ImageFileFormat format) => _managed.TierFor(format);
public FileTier Probe(string path, ImageFileFormat format) => Probe(path, format, null);
public FileTier Probe(string path, ImageFileFormat format, string? companion) =>
Tiers.TryGetValue(Path.GetFileName(path), out var tier) ? tier : _managed.Probe(path, format, companion);
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default)
{
Requests.Enqueue(request);
var name = Path.GetFileName(request.Path);
if (Hold is { } hold && name == hold && (HoldPurpose ?? request.Purpose) == request.Purpose
&& (HoldEdge ?? request.MaxLongEdge) == request.MaxLongEdge)
{
Holding.TrySetResult();
try
{
_released.Task.Wait(TimeSpan.FromSeconds(30), cancellationToken);
}
catch (OperationCanceledException)
{
}
cancellationToken.ThrowIfCancellationRequested();
}
if (Unreadable.ContainsKey(name)) throw new IOException($"The process cannot access the file '{request.Path}' because it is being used by another process.");
var decoded = _managed.Decode(request, cancellationToken);
if (decoded is null) return null;
Decoded.Enqueue((name, decoded.Decoder));
if (!PreviewDecodes.TryGetValue(name, out var preview)) return decoded;
return new DecodedImage(decoded.Pixels, DecodeTier.EmbeddedPreview, "test-preview", preview.Width, preview.Height);
}
}