Files
Chronos/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs
T
Claude 4014b0e669 fix(proxies): judge proxies by one rule and record tiers and reports as they stand
- ArchiveImages.IsProxyOf compares capture times to the microsecond the index
  keeps (sidecars and Exif keep 100 ns ticks), and for frames read through
  proxies also applies the policy's size rule, so the proxy pass, uploads,
  thumbnails, renders, analyses, tiers, manifests and downloads agree.
- Proxy headers are cached per file while length and write time are unchanged
  (and the file has settled), so manifests cost a stat per proxy.
- FrameTiers.RecordAsync checks proxy presence when it writes the sidecar and
  returns the sequence's frames as they are, so a deep analysis or render that
  ran while proxies arrived no longer flags the frames preview-only again, and
  the analysis advises from the current flags.
- Server manifests ignore ProxiesBuilt for frames the server cannot decode, so
  frames added since the last proxy pass are asked for their proxies.
- GET /sequences/{id}/analysis/report answers 404 for a report that found
  frames unreadable before their proxies arrived (DeepReports.ReadCurrentAsync).
- The deep checkpoint key marks only frames read from proxies, so checkpoints
  written by earlier versions still resume.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 16:52:30 +00:00

178 lines
8.7 KiB
C#

using System.Collections.Concurrent;
using Titano.Core.Analysis.Deep;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Diagnostics.Synthetic;
using Titano.Core.Imaging;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Jobs;
using Titano.Data.Processing;
using Titano.Data.Tests.Archiving;
using Titano.Testing;
namespace Titano.Data.Tests.Processing;
/// <summary>A temporary archive holding synthetic sequences (JPEGs with Exif), a processing engine and its job engine.</summary>
internal sealed class ProcessingFixture : IAsyncDisposable
{
/// <summary>Small sources are analysed at native size; batches of two frames between checkpoints.</summary>
public static readonly DeepAnalysisOptions DeepOptions = new() { MinAnalysisLongEdge = 64, FlowEvery = 5, MaxDegreeOfParallelism = 2, RegionSampleFrames = 9 };
private JobEngine? _engine;
private ProcessingFixture(ArchiveFixture archive, GateDecoder decoder, Func<ProcessingOptions, ProcessingOptions>? configure)
{
Archive = archive;
Decoder = decoder;
var options = new ProcessingOptions
{
Archive = new ArchiveServiceOptions { Decoder = new ImageDecoder([decoder]), Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) },
DeepAnalysis = DeepOptions,
ProducedBy = "test-pc",
};
Processing = new ProcessingService(archive.Db, configure?.Invoke(options) ?? options);
}
public ArchiveFixture Archive { get; }
public GateDecoder Decoder { get; }
public ProcessingService Processing { get; }
public TitanoDatabase Db => Archive.Db;
public CancellationToken Ct => Archive.Ct;
public JobEngine Engine => _engine ?? throw new InvalidOperationException("Start the engine first.");
/// <param name="configure">Changes to the processing options (a server's host and render-node check).</param>
public static async Task<ProcessingFixture> CreateAsync(TestContext context, Func<ProcessingOptions, ProcessingOptions>? configure = null) =>
new(await ArchiveFixture.CreateAsync(context), new GateDecoder(), configure);
/// <summary>A started engine with the processing handlers, and the archive's when asked (one worker unless asked, fast checkpoints).</summary>
public async Task<JobEngine> StartEngineAsync(bool withArchive = false, int workers = 1)
{
_engine = new JobEngine(Db, [.. withArchive ? Archive.Service.CreateJobHandlers() : [], .. Processing.CreateJobHandlers()],
new JobEngineOptions { Workers = workers, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await _engine.StartAsync(Ct);
return _engine;
}
public Task<Job> EnqueueAsync(Titano.Data.Archiving.Jobs.ArchiveJobRequest request) => Engine.EnqueueAsync(request, cancellationToken: Ct);
public Task<Job> WaitForAsync(Guid jobId, JobState state) => Archive.WaitForAsync(jobId, state, 60);
/// <summary>Writes frames <paramref name="first"/>… of <paramref name="sequence"/> as JPEGs with their Exif into
/// <paramref name="folder"/> (adopted in place) and indexes the archive.</summary>
public async Task<Sequence> AddAsync(SyntheticSequence sequence, string folder = "Synthetic", int first = 0)
{
Write(sequence, folder, first);
await Archive.ScanAsync(ScanMode.Full);
var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder);
return (await Db.Sequences.GetAsync(location.Id, Ct))!;
}
/// <summary>Writes frames <paramref name="first"/>… of <paramref name="sequence"/> as JPEGs with their Exif into <paramref name="folder"/>.</summary>
public void Write(SyntheticSequence sequence, string folder = "Synthetic", int first = 0)
{
var directory = Path.Combine(Archive.Paths.Root, folder);
Directory.CreateDirectory(directory);
for (var i = first; i < sequence.Count; i++)
{
using var image = sequence.Render(i);
var exif = ExifWriter.BuildJpegApp1(sequence.Metadata[i]);
File.WriteAllBytes(Path.Combine(directory, FileName(i)), JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 95, App1Segments = [exif[4..]] }));
}
}
public static string FileName(int index) => $"IMG_{index + 1:0000}.JPG";
public SequencePaths Folder(Sequence sequence) => Archive.Paths.Sequence(sequence.Folder);
public async ValueTask DisposeAsync()
{
if (_engine is not null) await _engine.DisposeAsync();
await Archive.DisposeAsync();
}
}
/// <summary>Managed decoding that records every request and can hold one file's decode until the job is cancelled or the decode is released.</summary>
internal sealed class GateDecoder : IImageDecoderProvider
{
private readonly ManagedDecoderProvider _managed = new();
public string Name => "gate";
public ConcurrentQueue<DecodeRequest> Requests { get; } = new();
/// <summary>File name whose decode waits for cancellation; null decodes everything.</summary>
public string? Hold { get; set; }
/// <summary>Holds only decodes of this purpose (null: any).</summary>
public DecodePurpose? HoldPurpose { get; set; }
/// <summary>Holds only decodes with this long edge (null: any).</summary>
public int? HoldEdge { get; set; }
/// <summary>Completed when a decode of <see cref="Hold"/> started waiting.</summary>
public TaskCompletionSource Holding { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
private TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void HoldFrame(int index, DecodePurpose? purpose = null, int? edge = null)
{
Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
_released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
(HoldPurpose, HoldEdge) = (purpose, edge);
Hold = ProcessingFixture.FileName(index);
}
/// <summary>Lets the held decode go on, as a slow read would.</summary>
public void Release() => _released.TrySetResult();
/// <summary>Tiers <see cref="Probe(string, ImageFileFormat)"/> reports per file name instead of the managed provider's (a
/// host that reads them only as previews, or not at all).</summary>
public ConcurrentDictionary<string, FileTier> Tiers { get; } = new();
/// <summary>Files (by name) whose decodes report the embedded-preview tier from a preview of this oriented size, as
/// a size-limited decode that picked a smaller preview would.</summary>
public ConcurrentDictionary<string, (int Width, int Height)> PreviewDecodes { get; } = new();
/// <summary>Files (by name) that cannot be read: their decodes throw <see cref="IOException"/> (a sharing violation).</summary>
public ConcurrentDictionary<string, byte> Unreadable { get; } = new();
/// <summary>File name and decoder of every decode that produced pixels.</summary>
public ConcurrentQueue<(string File, string Decoder)> Decoded { get; } = new();
public DecodeTier? TierFor(ImageFileFormat format) => _managed.TierFor(format);
public FileTier Probe(string path, ImageFileFormat format) => Probe(path, format, null);
public FileTier Probe(string path, ImageFileFormat format, string? companion) =>
Tiers.TryGetValue(Path.GetFileName(path), out var tier) ? tier : _managed.Probe(path, format, companion);
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default)
{
Requests.Enqueue(request);
var name = Path.GetFileName(request.Path);
if (Hold is { } hold && name == hold && (HoldPurpose ?? request.Purpose) == request.Purpose
&& (HoldEdge ?? request.MaxLongEdge) == request.MaxLongEdge)
{
Holding.TrySetResult();
try
{
_released.Task.Wait(TimeSpan.FromSeconds(30), cancellationToken);
}
catch (OperationCanceledException)
{
}
cancellationToken.ThrowIfCancellationRequested();
}
if (Unreadable.ContainsKey(name)) throw new IOException($"The process cannot access the file '{request.Path}' because it is being used by another process.");
var decoded = _managed.Decode(request, cancellationToken);
if (decoded is null) return null;
Decoded.Enqueue((name, decoded.Decoder));
if (!PreviewDecodes.TryGetValue(name, out var preview)) return decoded;
return new DecodedImage(decoded.Pixels, DecodeTier.EmbeddedPreview, "test-preview", preview.Width, preview.Height);
}
}