using System.Collections.Concurrent; using Titano.Core.Analysis.Deep; using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; using Titano.Core.Diagnostics.Synthetic; using Titano.Core.Imaging; using Titano.Core.Imaging.Decoders; using Titano.Core.Metadata; using Titano.Core.Model; using Titano.Data.Archiving; using Titano.Data.Jobs; using Titano.Data.Processing; using Titano.Data.Tests.Archiving; using Titano.Testing; namespace Titano.Data.Tests.Processing; /// A temporary archive holding synthetic sequences (JPEGs with Exif), a processing engine and its job engine. internal sealed class ProcessingFixture : IAsyncDisposable { /// Small sources are analysed at native size; batches of two frames between checkpoints. public static readonly DeepAnalysisOptions DeepOptions = new() { MinAnalysisLongEdge = 64, FlowEvery = 5, MaxDegreeOfParallelism = 2, RegionSampleFrames = 9 }; private JobEngine? _engine; private ProcessingFixture(ArchiveFixture archive, GateDecoder decoder, Func? configure) { Archive = archive; Decoder = decoder; var options = new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = new ImageDecoder([decoder]), Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }, DeepAnalysis = DeepOptions, ProducedBy = "test-pc", }; Processing = new ProcessingService(archive.Db, configure?.Invoke(options) ?? options); } public ArchiveFixture Archive { get; } public GateDecoder Decoder { get; } public ProcessingService Processing { get; } public TitanoDatabase Db => Archive.Db; public CancellationToken Ct => Archive.Ct; public JobEngine Engine => _engine ?? throw new InvalidOperationException("Start the engine first."); /// Changes to the processing options (a server's host and render-node check). public static async Task CreateAsync(TestContext context, Func? configure = null) => new(await ArchiveFixture.CreateAsync(context), new GateDecoder(), configure); /// A started engine with the processing handlers, and the archive's when asked (one worker unless asked, fast checkpoints). public async Task StartEngineAsync(bool withArchive = false, int workers = 1) { _engine = new JobEngine(Db, [.. withArchive ? Archive.Service.CreateJobHandlers() : [], .. Processing.CreateJobHandlers()], new JobEngineOptions { Workers = workers, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); await _engine.StartAsync(Ct); return _engine; } public Task EnqueueAsync(Titano.Data.Archiving.Jobs.ArchiveJobRequest request) => Engine.EnqueueAsync(request, cancellationToken: Ct); public Task WaitForAsync(Guid jobId, JobState state) => Archive.WaitForAsync(jobId, state, 60); /// Writes frames … of as JPEGs with their Exif into /// (adopted in place) and indexes the archive. public async Task AddAsync(SyntheticSequence sequence, string folder = "Synthetic", int first = 0) { Write(sequence, folder, first); await Archive.ScanAsync(ScanMode.Full); var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder); return (await Db.Sequences.GetAsync(location.Id, Ct))!; } /// Writes frames … of as JPEGs with their Exif into . public void Write(SyntheticSequence sequence, string folder = "Synthetic", int first = 0) { var directory = Path.Combine(Archive.Paths.Root, folder); Directory.CreateDirectory(directory); for (var i = first; i < sequence.Count; i++) { using var image = sequence.Render(i); var exif = ExifWriter.BuildJpegApp1(sequence.Metadata[i]); File.WriteAllBytes(Path.Combine(directory, FileName(i)), JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 95, App1Segments = [exif[4..]] })); } } public static string FileName(int index) => $"IMG_{index + 1:0000}.JPG"; public SequencePaths Folder(Sequence sequence) => Archive.Paths.Sequence(sequence.Folder); public async ValueTask DisposeAsync() { if (_engine is not null) await _engine.DisposeAsync(); await Archive.DisposeAsync(); } } /// Managed decoding that records every request and can hold one file's decode until the job is cancelled or the decode is released. internal sealed class GateDecoder : IImageDecoderProvider { private readonly ManagedDecoderProvider _managed = new(); public string Name => "gate"; public ConcurrentQueue Requests { get; } = new(); /// File name whose decode waits for cancellation; null decodes everything. public string? Hold { get; set; } /// Holds only decodes of this purpose (null: any). public DecodePurpose? HoldPurpose { get; set; } /// Holds only decodes with this long edge (null: any). public int? HoldEdge { get; set; } /// Completed when a decode of started waiting. public TaskCompletionSource Holding { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously); private TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously); public void HoldFrame(int index, DecodePurpose? purpose = null, int? edge = null) { Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); _released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); (HoldPurpose, HoldEdge) = (purpose, edge); Hold = ProcessingFixture.FileName(index); } /// File name whose tier probe waits until (a light analysis probes frames whose tier is /// not known yet, and decodes nothing); null probes everything. public string? HoldProbe { get; private set; } /// Holds the next tier probe of frame 's file, as holds a decode. public void HoldProbeOf(int index) { Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); _released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); HoldProbe = ProcessingFixture.FileName(index); } /// Lets the held decode go on, as a slow read would. public void Release() => _released.TrySetResult(); /// Tiers reports per file name instead of the managed provider's (a /// host that reads them only as previews, or not at all). public ConcurrentDictionary Tiers { get; } = new(); /// Files (by name) whose decodes report the embedded-preview tier from a preview of this oriented size, as /// a size-limited decode that picked a smaller preview would. public ConcurrentDictionary PreviewDecodes { get; } = new(); /// Files (by name) that cannot be read: their decodes throw (a sharing violation). public ConcurrentDictionary Unreadable { get; } = new(); /// File name and decoder of every decode that produced pixels. public ConcurrentQueue<(string File, string Decoder)> Decoded { get; } = new(); public DecodeTier? TierFor(ImageFileFormat format) => _managed.TierFor(format); public FileTier Probe(string path, ImageFileFormat format) => Probe(path, format, null); public FileTier Probe(string path, ImageFileFormat format, string? companion) { var name = Path.GetFileName(path); if (HoldProbe is { } hold && name == hold) { Holding.TrySetResult(); _released.Task.Wait(TimeSpan.FromSeconds(30)); } return Tiers.TryGetValue(name, out var tier) ? tier : _managed.Probe(path, format, companion); } public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) { Requests.Enqueue(request); var name = Path.GetFileName(request.Path); if (Hold is { } hold && name == hold && (HoldPurpose ?? request.Purpose) == request.Purpose && (HoldEdge ?? request.MaxLongEdge) == request.MaxLongEdge) { Holding.TrySetResult(); try { _released.Task.Wait(TimeSpan.FromSeconds(30), cancellationToken); } catch (OperationCanceledException) { } cancellationToken.ThrowIfCancellationRequested(); } if (Unreadable.ContainsKey(name)) throw new IOException($"The process cannot access the file '{request.Path}' because it is being used by another process."); var decoded = _managed.Decode(request, cancellationToken); if (decoded is null) return null; Decoded.Enqueue((name, decoded.Decoder)); if (!PreviewDecodes.TryGetValue(name, out var preview)) return decoded; return new DecodedImage(decoded.Pixels, DecodeTier.EmbeddedPreview, "test-preview", preview.Width, preview.Height); } }