fix(proxies): judge proxies by one rule and record tiers and reports as they stand
- ArchiveImages.IsProxyOf compares capture times to the microsecond the index
keeps (sidecars and Exif keep 100 ns ticks), and for frames read through
proxies also applies the policy's size rule, so the proxy pass, uploads,
thumbnails, renders, analyses, tiers, manifests and downloads agree.
- Proxy headers are cached per file while length and write time are unchanged
(and the file has settled), so manifests cost a stat per proxy.
- FrameTiers.RecordAsync checks proxy presence when it writes the sidecar and
returns the sequence's frames as they are, so a deep analysis or render that
ran while proxies arrived no longer flags the frames preview-only again, and
the analysis advises from the current flags.
- Server manifests ignore ProxiesBuilt for frames the server cannot decode, so
frames added since the last proxy pass are asked for their proxies.
- GET /sequences/{id}/analysis/report answers 404 for a report that found
frames unreadable before their proxies arrived (DeepReports.ReadCurrentAsync).
- The deep checkpoint key marks only frames read from proxies, so checkpoints
written by earlier versions still resume.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
16 files changed
+514
-99
No files matched your search
@@ -57,7 +57,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S
|
||||
| `Repositories` | `TitanoDatabase` facade: archives, sequences, frames, tags, renders, jobs, cards, users, tokens, uploads, render nodes, settings, audit; `DatabaseBackups` (daily `VACUUM INTO` copies of an SQLite index kept ×3, restore staged for the next start and carried out before the index opens, so it also recovers one that no longer opens) |
|
||||
| `Jobs` | `JobEngine` (its number of workers can change while it runs), `IJobHandler`, `IJobHandlerProvider`, `JobContext` (progress, checkpoints, follow-up jobs) |
|
||||
| `Archiving` | `ArchiveService` (create/adopt/open, archive settings written to `archive.json` then the index), `ArchiveIndexer` (scan, rebuild), `ArchiveIngest` (uploads and imports), `ImportService`, `ArchiveVerifier`, `ArchiveImages`, `FrameTiers` (decode tier, `PreviewOnly`/`Undecodable` flags and preview size of frames, written to sidecar and index by the light analysis, deep analysis, renders and image passes), `GprCompanions` (the JPEG of each GoPro GPR, found in the index when an import kept it in another sequence), `ArchiveMaintenance`, `MassRenamer`, `SequenceTrash` (delete into the archive's hidden trash with a dry run, restore; the folder moves back when the index step fails), job handlers |
|
||||
| `Processing` | `ProcessingService`, `SequenceAnalyser`, `SequenceRender`, `ProjectStore`, `ProjectBundles` (a `.titano` file for a sequence, and the sequence a file names), `RenderNodeClient`, job handlers |
|
||||
| `Processing` | `ProcessingService`, `SequenceAnalyser`, `DeepReports` (a saved deep report while it still describes the frames), `SequenceRender`, `ProjectStore`, `ProjectBundles` (a `.titano` file for a sequence, and the sequence a file names), `RenderNodeClient`, job handlers |
|
||||
| `Sync` | `SyncService` (sync table of one server: compare, link by content, settle conflicts, enqueue transfers), `SyncRules` (status, settings merge and differences, no I/O), `SyncLinkStore` (`sync_links`, `sync_state`), `SyncTransfers` and `SyncJobHandler` (resumable, hash-verified uploads and downloads as jobs; downloads commit through the archive ingest), `SyncAgent` (background comparison, server events, backoff, offline proxies), `ProxyCache` and `RemoteSequence` (server sequences opened with proxies), `ISyncRemote` (the server calls; `TitanoSyncRemote` over `TitanoClient`) |
|
||||
|
||||
### 2.3 Titano.App areas
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
@@ -47,9 +48,26 @@ public sealed class ArchiveImages
|
||||
|
||||
private const int CommitFrames = 200;
|
||||
private const int ProbeLongEdge = 64;
|
||||
|
||||
/// <summary>Proxy headers kept at most (<see cref="Headers"/>): some 30 MB, the proxies of a few large sequences.</summary>
|
||||
private const int MaxHeaders = 1 << 17;
|
||||
|
||||
private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>
|
||||
/// Age of a proxy file under which its header is not kept: a file system's clock may give a file written again at
|
||||
/// once (with the same length) the same write time, at a granularity of up to two seconds.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan Settling = TimeSpan.FromSeconds(3);
|
||||
|
||||
private static readonly ImageDecoder ServerDecoders = DecoderFactory.CreateServer();
|
||||
|
||||
/// <summary>
|
||||
/// The capture time and size of proxies read lately, by path, while the file keeps its length and write time: the
|
||||
/// server's manifests and tier records check every proxy of a sequence, so an unchanged one costs a stat.
|
||||
/// </summary>
|
||||
private static readonly ConcurrentDictionary<string, ProxyHeader> Headers = new(StringComparer.Ordinal);
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
@@ -90,12 +108,18 @@ public sealed class ArchiveImages
|
||||
|
||||
/// <summary>
|
||||
/// Whether the file at <paramref name="proxy"/> (a sequence's <c>proxies/0001.jpg</c>) is the proxy of the frame
|
||||
/// captured at <paramref name="capturedAt"/>: its Exif capture time is the frame's, or the frame has none to compare
|
||||
/// with. Proxies are named by frame index, so one left by a frame renumbered since (frames inserted before it) is
|
||||
/// another photo's: renders, analyses and the server's manifests and downloads treat it as missing, and the proxy
|
||||
/// pass removes it (<see cref="BuildProxiesAsync"/>).
|
||||
/// captured at <paramref name="capturedAt"/>, of <paramref name="width"/> × <paramref name="height"/>: its Exif capture
|
||||
/// time is the frame's to the microsecond (the index's precision), or the frame has none to compare with; and, for a
|
||||
/// frame whose original this host takes from its proxy (<paramref name="policy"/> given: the archive's), it is not
|
||||
/// smaller than the policy's (<see cref="TooSmall"/>: one an earlier version made from an embedded preview). Proxies
|
||||
/// are named by frame index, so one left by a frame renumbered since (frames inserted before it) is another photo's.
|
||||
/// Renders, analyses, decode tiers and the server's manifests and downloads treat a proxy that fails this as missing,
|
||||
/// and the proxy pass removes it (<see cref="BuildProxiesAsync"/>).
|
||||
/// </summary>
|
||||
public static bool IsProxyOf(string proxy, DateTime? capturedAt) => ReadProxy(proxy, capturedAt).Check is ProxyCheck.Current or ProxyCheck.Unverifiable;
|
||||
/// <param name="policy">The archive's proxy policy for a frame whose original this host cannot decode (its proxy comes
|
||||
/// from a client or render node); null for one this host decodes and makes the proxy of itself.</param>
|
||||
public static bool IsProxyOf(string proxy, DateTime? capturedAt, int width, int height, ProxyPolicy? policy) =>
|
||||
Check(proxy, capturedAt, width, height, policy).Check is ProxyCheck.Current or ProxyCheck.Unverifiable;
|
||||
|
||||
/// <summary>
|
||||
/// A proxy of the original at <paramref name="path"/> for a server that cannot decode it (<see cref="StoreProxy"/>):
|
||||
@@ -151,11 +175,13 @@ public sealed class ArchiveImages
|
||||
if (frame.CapturedAt is { } captured)
|
||||
{
|
||||
using var stream = new MemoryStream(jpeg, writable: false);
|
||||
if (MetadataReader.Read(stream, "proxy.jpg").CapturedAt != captured) return ProxyStoreResult.WrongFrame;
|
||||
if (!SameCapture(MetadataReader.Read(stream, "proxy.jpg").CapturedAt, captured)) return ProxyStoreResult.WrongFrame;
|
||||
}
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
AtomicFile.WriteAllBytes(folder.Proxy(frame.Index), jpeg);
|
||||
var path = folder.Proxy(frame.Index);
|
||||
AtomicFile.WriteAllBytes(path, jpeg);
|
||||
Headers.TryRemove(path, out _);
|
||||
return ProxyStoreResult.Stored;
|
||||
}
|
||||
|
||||
@@ -167,7 +193,8 @@ public sealed class ArchiveImages
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ThumbnailPolicy;
|
||||
var archiveManifest = ManifestStore.ReadArchive(paths.ManifestPath);
|
||||
var policy = archiveManifest.ThumbnailPolicy;
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
int built = 0, reused = 0, undecodable = 0;
|
||||
@@ -198,7 +225,7 @@ public sealed class ArchiveImages
|
||||
// preview, whose smaller thumbnail the proxy's replaces.
|
||||
var fromProxy = false;
|
||||
if (NeedsProxy(frame.Format) && !(current && edge >= policy.LongEdge)
|
||||
&& ReadProxy(folder.Proxy(frame.I), frame.CapturedAt) is { Check: ProxyCheck.Current or ProxyCheck.Unverifiable } proxy)
|
||||
&& Check(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, archiveManifest.ProxyPolicy) is { Check: ProxyCheck.Current or ProxyCheck.Unverifiable } proxy)
|
||||
fromProxy = !current || edge < System.Math.Min(policy.LongEdge, proxy.LongEdge);
|
||||
if (current && !fromProxy)
|
||||
{
|
||||
@@ -227,7 +254,7 @@ public sealed class ArchiveImages
|
||||
}
|
||||
pack.Commit();
|
||||
}
|
||||
await _tiers.RecordAsync(archive, sequence, tiers, cancellationToken).ConfigureAwait(false);
|
||||
await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false);
|
||||
await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false);
|
||||
return new ImagePassResult(built, reused, undecodable);
|
||||
}
|
||||
@@ -252,7 +279,6 @@ public sealed class ArchiveImages
|
||||
int built = 0, reused = 0, undecodable = 0, waiting = 0;
|
||||
var complete = true;
|
||||
var tiers = new Dictionary<Guid, FileTier>();
|
||||
var proxied = new Dictionary<Guid, bool>();
|
||||
var companions = await GprCompanions.FindAsync(_db, archive, manifest.Frames, cancellationToken).ConfigureAwait(false);
|
||||
var cadence = new Cadence();
|
||||
for (var i = 0; i < manifest.Frames.Count; i++)
|
||||
@@ -266,11 +292,9 @@ public sealed class ArchiveImages
|
||||
// Only a client or render node that decodes the original makes this proxy: a preview would make one the
|
||||
// size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again, and so
|
||||
// does one smaller than the policy's (made here from a preview by an earlier version).
|
||||
var (check, longEdge) = ReadProxy(path, frame.CapturedAt);
|
||||
if (check is ProxyCheck.Current or ProxyCheck.Unverifiable && TooSmall(longEdge, frame.W, frame.H, policy)) check = ProxyCheck.Stale;
|
||||
var check = Check(path, frame.CapturedAt, frame.W, frame.H, policy).Check;
|
||||
var usable = check is ProxyCheck.Current or ProxyCheck.Unverifiable;
|
||||
if (check == ProxyCheck.Stale) File.Delete(path);
|
||||
proxied[frame.Id] = usable;
|
||||
if (check == ProxyCheck.Stale) Delete(path);
|
||||
complete &= usable;
|
||||
if (done) continue;
|
||||
if (usable) reused++;
|
||||
@@ -292,6 +316,7 @@ public sealed class ArchiveImages
|
||||
() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken, companions.GetValueOrDefault(frame.Id))) is { } image)
|
||||
{
|
||||
AtomicFile.WriteAllBytes(path, image.Jpeg);
|
||||
Headers.TryRemove(path, out _);
|
||||
built++;
|
||||
}
|
||||
else
|
||||
@@ -301,7 +326,7 @@ public sealed class ArchiveImages
|
||||
}
|
||||
if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
|
||||
}
|
||||
await _tiers.RecordAsync(archive, sequence, tiers, proxied, cancellationToken).ConfigureAwait(false);
|
||||
await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false);
|
||||
await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
|
||||
return new ImagePassResult(built, reused, undecodable, waiting);
|
||||
}
|
||||
@@ -317,27 +342,80 @@ public sealed class ArchiveImages
|
||||
/// <summary>The frame has no capture time to compare with.</summary>
|
||||
Unverifiable,
|
||||
|
||||
/// <summary>Made from another frame (frames were renumbered since).</summary>
|
||||
/// <summary>Made from another frame (frames were renumbered since), or too small (<see cref="Check"/>).</summary>
|
||||
Stale,
|
||||
}
|
||||
|
||||
/// <summary>A proxy's Exif capture time and oriented long edge, read when the file had this length and write time.</summary>
|
||||
private readonly record struct ProxyHeader(long Length, long WriteTicks, DateTime? CapturedAt, int LongEdge);
|
||||
|
||||
/// <summary>
|
||||
/// Where the proxy at <paramref name="proxy"/> stands for the frame captured at <paramref name="capturedAt"/>, of
|
||||
/// <paramref name="width"/> × <paramref name="height"/>, and its long edge: <see cref="ProxyCheck.Stale"/> also when
|
||||
/// <paramref name="policy"/> is given and the proxy is <see cref="TooSmall"/> for it (<see cref="IsProxyOf"/>).
|
||||
/// </summary>
|
||||
private static (ProxyCheck Check, int LongEdge) Check(string proxy, DateTime? capturedAt, int width, int height, ProxyPolicy? policy)
|
||||
{
|
||||
var (check, longEdge) = ReadProxy(proxy, capturedAt);
|
||||
if (check is ProxyCheck.Current or ProxyCheck.Unverifiable && policy is not null && TooSmall(longEdge, width, height, policy)) check = ProxyCheck.Stale;
|
||||
return (check, longEdge);
|
||||
}
|
||||
|
||||
/// <summary>Where the proxy at <paramref name="proxy"/> stands for the frame captured at <paramref name="capturedAt"/>, and its long edge.</summary>
|
||||
private static (ProxyCheck Check, int LongEdge) ReadProxy(string proxy, DateTime? capturedAt)
|
||||
{
|
||||
if (!File.Exists(proxy)) return (ProxyCheck.Missing, 0);
|
||||
if (ReadHeader(proxy) is not { } header) return (ProxyCheck.Missing, 0);
|
||||
var check = capturedAt is not { } captured ? ProxyCheck.Unverifiable : SameCapture(header.CapturedAt, captured) ? ProxyCheck.Current : ProxyCheck.Stale;
|
||||
return (check, header.LongEdge);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The header of the proxy at <paramref name="proxy"/>; null when there is none or it cannot be read. Kept in
|
||||
/// <see cref="Headers"/> while the file's length and write time stay as they were when it was read (taken first, so
|
||||
/// a file replaced meanwhile is read again next time), once the file is older than <see cref="Settling"/>; a full
|
||||
/// cache starts over.
|
||||
/// </summary>
|
||||
private static ProxyHeader? ReadHeader(string proxy)
|
||||
{
|
||||
try
|
||||
{
|
||||
var file = new FileInfo(proxy);
|
||||
if (!file.Exists) return null;
|
||||
var (length, written) = (file.Length, file.LastWriteTimeUtc);
|
||||
if (Headers.TryGetValue(proxy, out var cached) && cached.Length == length && cached.WriteTicks == written.Ticks) return cached;
|
||||
var metadata = MetadataReader.Read(proxy);
|
||||
var (width, height) = metadata.OrientedSize;
|
||||
var check = capturedAt is null ? ProxyCheck.Unverifiable : metadata.CapturedAt == capturedAt ? ProxyCheck.Current : ProxyCheck.Stale;
|
||||
return (check, System.Math.Max(width, height));
|
||||
var header = new ProxyHeader(length, written.Ticks, metadata.CapturedAt, System.Math.Max(width, height));
|
||||
if (DateTime.UtcNow - written < Settling)
|
||||
{
|
||||
Headers.TryRemove(proxy, out _);
|
||||
return header;
|
||||
}
|
||||
if (Headers.Count >= MaxHeaders) Headers.Clear();
|
||||
Headers[proxy] = header;
|
||||
return header;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return (ProxyCheck.Missing, 0);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a proxy's Exif capture time is the frame's <paramref name="captured"/> to the microsecond: the index keeps
|
||||
/// capture times to the microsecond, sidecars and Exif to the 100 ns tick, so a proxy made from either record of the
|
||||
/// frame (here from the sidecar, by a client from the server's index) matches both.
|
||||
/// </summary>
|
||||
private static bool SameCapture(DateTime? proxy, DateTime captured) =>
|
||||
proxy is { } time && time.Ticks / TimeSpan.TicksPerMicrosecond == captured.Ticks / TimeSpan.TicksPerMicrosecond;
|
||||
|
||||
/// <summary>Removes the proxy at <paramref name="path"/> and what is known of it.</summary>
|
||||
private static void Delete(string path)
|
||||
{
|
||||
File.Delete(path);
|
||||
Headers.TryRemove(path, out _);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a proxy whose long edge is <paramref name="longEdge"/> falls short of the policy's for a frame of
|
||||
/// <paramref name="width"/> × <paramref name="height"/> (the policy's long edge, or the frame's own when smaller or
|
||||
|
||||
@@ -51,25 +51,26 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
|
||||
return tiers;
|
||||
}
|
||||
|
||||
/// <inheritdoc cref="RecordAsync(Archive, Sequence, IReadOnlyDictionary{Guid, FileTier}, IReadOnlyDictionary{Guid, bool}?, CancellationToken)"/>
|
||||
public Task<IReadOnlyList<SidecarFrame>> RecordAsync(Archive archive, Sequence sequence, IReadOnlyDictionary<Guid, FileTier> tiers,
|
||||
CancellationToken cancellationToken) => RecordAsync(archive, sequence, tiers, null, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Records <paramref name="tiers"/> (by frame id) where they change what the sidecar says, and returns the frames that
|
||||
/// changed as they are now. Frames no longer in the sequence are ignored.
|
||||
/// Records <paramref name="tiers"/> (by frame id) where they change what the sidecar says, and returns the sequence's
|
||||
/// frames as the sidecar has them afterwards (none when there was nothing to record). Frames no longer in the
|
||||
/// sequence are ignored. The <see cref="FrameFlags.PreviewOnly"/> flag of a frame whose original this host reads
|
||||
/// through its proxy follows whether its own proxy is there when the sidecar is written
|
||||
/// (<see cref="ArchiveImages.IsProxyOf"/>, <see cref="SequenceFrames.WithProxy"/>), not when the caller looked: proxies
|
||||
/// arrive while analyses and renders run, and another worker may have recorded them already.
|
||||
/// </summary>
|
||||
/// <param name="proxied">By frame id, for frames whose original this host reads through its proxy, whether the frame's
|
||||
/// own proxy is there now (<see cref="SequenceFrames.WithProxy"/>); their <see cref="FrameFlags.PreviewOnly"/> follows
|
||||
/// it even where no tier is recorded.</param>
|
||||
/// <param name="proxies">Whether every such frame follows its proxy even where no tier is recorded (analyses and the
|
||||
/// proxy pass); otherwise only the frames in <paramref name="tiers"/> do.</param>
|
||||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
|
||||
public async Task<IReadOnlyList<SidecarFrame>> RecordAsync(Archive archive, Sequence sequence, IReadOnlyDictionary<Guid, FileTier> tiers,
|
||||
IReadOnlyDictionary<Guid, bool>? proxied, CancellationToken cancellationToken)
|
||||
bool proxies, CancellationToken cancellationToken)
|
||||
{
|
||||
if (tiers.Count == 0 && proxied is not { Count: > 0 }) return [];
|
||||
if (tiers.Count == 0 && !proxies) return [];
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var changed = new List<SidecarFrame>();
|
||||
ProxyPolicy? policy = null;
|
||||
var manifest = await writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
|
||||
{
|
||||
changed.Clear();
|
||||
@@ -79,10 +80,15 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
|
||||
{
|
||||
var frame = m.Frames[i];
|
||||
frames[i] = frame;
|
||||
bool? hasProxy = proxied is not null && proxied.TryGetValue(frame.Id, out var found) ? found : null;
|
||||
var tier = tiers.GetValueOrDefault(frame.Id);
|
||||
if (tier is null && !proxies) continue;
|
||||
bool? proxied = (frame.Flags & FrameFlags.Missing) == 0 && ArchiveImages.NeedsProxy(writer.Options.Decoder, frame.Format)
|
||||
? ArchiveImages.IsProxyOf(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H,
|
||||
policy ??= ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy)
|
||||
: null;
|
||||
SidecarFrame updated;
|
||||
if (tiers.TryGetValue(frame.Id, out var tier)) updated = SequenceFrames.WithTier(frame, tier, DecodesNatively(frame.Format), hasProxy == true);
|
||||
else if (hasProxy is { } proxy) updated = SequenceFrames.WithProxy(frame, proxy);
|
||||
if (tier is not null) updated = SequenceFrames.WithTier(frame, tier, DecodesNatively(frame.Format), proxied == true);
|
||||
else if (proxied is { } own) updated = SequenceFrames.WithProxy(frame, own);
|
||||
else continue;
|
||||
if (updated.Tier == frame.Tier && updated.Flags == frame.Flags && updated.PreviewW == frame.PreviewW && updated.PreviewH == frame.PreviewH)
|
||||
continue;
|
||||
@@ -98,14 +104,13 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
if (changed.Count == 0) return [];
|
||||
if (changed.Count == 0) return manifest.Frames;
|
||||
await db.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is { } existing)
|
||||
await db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, folder, existing), cancellationToken).ConfigureAwait(false);
|
||||
await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
return changed;
|
||||
return manifest.Frames;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
using Titano.Core.Analysis.Deep;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// Saved deep-analysis reports (<c>.titano/analysis.json</c>, BRIEF §10.2) for hosts that hand them on: a server serves
|
||||
/// one to the editors of its clients only while it still describes the sequence (§14.5).
|
||||
/// </summary>
|
||||
public static class DeepReports
|
||||
{
|
||||
/// <summary>
|
||||
/// The saved report of <paramref name="sequence"/> when it describes its frames as a host with
|
||||
/// <paramref name="decoder"/> reads them now; null when there is none, it covers another number of frames, or it
|
||||
/// found unreadable a frame that has been read from its proxy since (an original that host cannot decode, whose proxy
|
||||
/// arrived after the analysis): its verdict and its advice to exclude that frame no longer hold.
|
||||
/// </summary>
|
||||
public static async Task<DeepAnalysisReport?> ReadCurrentAsync(TitanoDatabase db, Archive archive, Sequence sequence, ImageDecoder decoder,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(archive);
|
||||
ArgumentNullException.ThrowIfNull(sequence);
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, decoder, cancellationToken).ConfigureAwait(false);
|
||||
return SequenceAnalyser.ReadReport(new ArchivePaths(archive.RootPath).Sequence(sequence.Folder), frames);
|
||||
}
|
||||
}
|
||||
@@ -94,11 +94,12 @@ internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions opti
|
||||
if (render.StandIns.Count > 0)
|
||||
context.Log(Loc.L("Export.Log.StandIns", string.Join(", ", render.StandIns.Select(i => (i + 1).ToString(Loc.Culture)))));
|
||||
// Recorded without the job's token: what decoding found holds even if the job is cancelled meanwhile, and a
|
||||
// sidecar that cannot be written now only leaves it to the next analysis or render.
|
||||
// sidecar that cannot be written now only leaves it to the next analysis or render. A frame decoded from its
|
||||
// preview whose proxy arrived during the render is not preview-only: FrameTiers looks for the proxy as it records.
|
||||
try
|
||||
{
|
||||
var decoded = render.Decoded.ToDictionary(kv => frames.Frames[kv.Key].Id, kv => kv.Value);
|
||||
await tiers.RecordAsync(archive, sequence, decoded, CancellationToken.None).ConfigureAwait(false);
|
||||
await tiers.RecordAsync(archive, sequence, decoded, proxies: false, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or LeaseUnavailableException
|
||||
or ArchiveOfflineException)
|
||||
|
||||
@@ -46,17 +46,18 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
|
||||
// Format tiers (BRIEF §10.1): frames indexed since the last analysis are probed with this host's decoders. A frame
|
||||
// read from its proxy is not preview-only whatever its original's tier, so it caps nothing (BRIEF §8.1, §8.2).
|
||||
// read from its proxy is not preview-only whatever its original's tier, so it caps nothing (BRIEF §8.1, §8.2); the
|
||||
// flags are taken as the sidecar has them once recorded, so ones another worker recorded meanwhile count too.
|
||||
var probed = await _tiers.ProbeUnknownAsync(archive, folder, ManifestStore.ReadSequence(folder.ManifestPath), cancellationToken).ConfigureAwait(false);
|
||||
var proxied = frames.ProxyStates(options.Archive.Decoder);
|
||||
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, probed, proxied, cancellationToken).ConfigureAwait(false));
|
||||
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, probed, proxies: true, cancellationToken).ConfigureAwait(false));
|
||||
var metadata = frames.LightFrames();
|
||||
|
||||
DeepAnalysisReport? report;
|
||||
if (deep)
|
||||
{
|
||||
// Proxies may have arrived during the pass: the advice below follows the frames as they are at its end.
|
||||
(report, var decoded) = await AnalyseDeepAsync(archive, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false);
|
||||
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxied, cancellationToken).ConfigureAwait(false));
|
||||
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxies: true, cancellationToken).ConfigureAwait(false));
|
||||
metadata = [.. metadata.Select((m, i) => m with { Flags = frames.Frames[i].Flags })];
|
||||
}
|
||||
else
|
||||
@@ -139,11 +140,11 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The frame rows with the tier fields of <paramref name="changed"/> sidecar frames.</summary>
|
||||
private static SequenceFrames WithTiers(SequenceFrames frames, IReadOnlyList<SidecarFrame> changed)
|
||||
/// <summary>The frame rows with the flags and preview sizes of the <paramref name="current"/> sidecar frames.</summary>
|
||||
private static SequenceFrames WithTiers(SequenceFrames frames, IReadOnlyList<SidecarFrame> current)
|
||||
{
|
||||
if (changed.Count == 0) return frames;
|
||||
var byId = changed.ToDictionary(f => f.Id);
|
||||
if (current.Count == 0) return frames;
|
||||
var byId = current.ToDictionary(f => f.Id);
|
||||
var rows = frames.Frames.Select(f => byId.TryGetValue(f.Id, out var s)
|
||||
? f with { Flags = s.Flags, PreviewWidth = s.PreviewW ?? 0, PreviewHeight = s.PreviewH ?? 0 }
|
||||
: f).ToList();
|
||||
@@ -215,15 +216,16 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
|
||||
|
||||
/// <summary>Identity of the analysed content: frames replaced or renumbered since a checkpoint was written (with the
|
||||
/// same count), or now read from their proxies where they were read from their originals (or the other way round),
|
||||
/// make it stale, like changed options do.</summary>
|
||||
private static string ContentKey(SequenceFrames frames)
|
||||
/// make it stale, like changed options do. Only frames read from proxies are marked, so the key of a sequence read
|
||||
/// from its originals is what it was before proxies counted, and checkpoints written then still resume.</summary>
|
||||
internal static string ContentKey(SequenceFrames frames)
|
||||
{
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
for (var i = 0; i < frames.Count; i++)
|
||||
{
|
||||
var f = frames.Frames[i];
|
||||
var source = frames.Proxied is { } proxied && proxied[i] ? "proxy" : "original";
|
||||
hash.AppendData(Encoding.UTF8.GetBytes($"{f.Id:N}/{f.Sha256}/{f.Size}/{f.Flags & SequenceFrames.NotAnalysed}/{source};"));
|
||||
var source = frames.Proxied is { } proxied && proxied[i] ? "/proxy" : "";
|
||||
hash.AppendData(Encoding.UTF8.GetBytes($"{f.Id:N}/{f.Sha256}/{f.Size}/{f.Flags & SequenceFrames.NotAnalysed}{source};"));
|
||||
}
|
||||
return Convert.ToHexStringLower(hash.GetHashAndReset())[..16];
|
||||
}
|
||||
|
||||
@@ -35,7 +35,8 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList<Frame> Fr
|
||||
/// Every frame row of <paramref name="sequence"/> with its archive file: the proxy when asked, and whenever the
|
||||
/// original is one <paramref name="decoder"/> cannot decode (<see cref="ArchiveImages.NeedsProxy(ImageDecoder, ImageFileFormat)"/>:
|
||||
/// HEIF on the server, BRIEF §2.3, §8.5), as long as the frame's own proxy is there (<see cref="ArchiveImages.IsProxyOf"/>:
|
||||
/// not one left at its index by a frame renumbered since); else the original.
|
||||
/// not one left at its index by a frame renumbered since, nor, for an original this host cannot decode, one smaller
|
||||
/// than the archive's policy); else the original.
|
||||
/// </summary>
|
||||
public static async Task<SequenceFrames> LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies, ImageDecoder decoder,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -46,13 +47,16 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList<Frame> Fr
|
||||
var files = new string?[frames.Count];
|
||||
var originals = new string?[frames.Count];
|
||||
var proxied = new bool[frames.Count];
|
||||
ProxyPolicy? policy = null;
|
||||
for (var i = 0; i < files.Length; i++)
|
||||
{
|
||||
var frame = frames[i];
|
||||
if ((frame.Flags & FrameFlags.Missing) != 0) continue;
|
||||
originals[i] = paths.FromRelative(frame.RelativePath);
|
||||
var proxy = useProxies || ArchiveImages.NeedsProxy(decoder, frame.Format) ? folder.Proxy(frame.Index) : null;
|
||||
proxied[i] = proxy is not null && ArchiveImages.IsProxyOf(proxy, frame.CapturedAt);
|
||||
var needs = ArchiveImages.NeedsProxy(decoder, frame.Format);
|
||||
var proxy = useProxies || needs ? folder.Proxy(frame.Index) : null;
|
||||
proxied[i] = proxy is not null && ArchiveImages.IsProxyOf(proxy, frame.CapturedAt, frame.Width, frame.Height,
|
||||
needs ? policy ??= ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy : null);
|
||||
files[i] = proxied[i] ? proxy : originals[i];
|
||||
}
|
||||
return new SequenceFrames(sequence, frames, files) { Proxied = proxied, Originals = originals };
|
||||
@@ -95,22 +99,6 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList<Frame> Fr
|
||||
return light;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// By frame id, whether each frame whose original <paramref name="decoder"/> cannot decode (it is read from its proxy,
|
||||
/// <see cref="LoadAsync"/>) has its own proxy now: what its <see cref="FrameFlags.PreviewOnly"/> follows
|
||||
/// (<see cref="FrameTiers.RecordAsync(Archive, Sequence, IReadOnlyDictionary{Guid, FileTier}, IReadOnlyDictionary{Guid, bool}?, CancellationToken)"/>).
|
||||
/// Missing frames are left out.
|
||||
/// </summary>
|
||||
public Dictionary<Guid, bool> ProxyStates(ImageDecoder decoder)
|
||||
{
|
||||
var states = new Dictionary<Guid, bool>();
|
||||
for (var i = 0; i < Frames.Count; i++)
|
||||
{
|
||||
if (Files[i] is not null && ArchiveImages.NeedsProxy(decoder, Frames[i].Format)) states[Frames[i].Id] = Proxied is { } proxied && proxied[i];
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
/// <summary>Positions of the frames a render must skip besides the project's exclusions.</summary>
|
||||
public IEnumerable<int> NotRenderable()
|
||||
{
|
||||
|
||||
@@ -254,7 +254,11 @@ public static class SequenceEndpoints
|
||||
_ => ApiErrors.NotFound(context, "analysis"),
|
||||
};
|
||||
|
||||
/// <summary>The full deep-analysis report (<c>.titano/analysis.json</c>) a client's editor shows for a server sequence.</summary>
|
||||
/// <summary>
|
||||
/// The full deep-analysis report (<c>.titano/analysis.json</c>) a client's editor shows for a server sequence, while it
|
||||
/// describes the frames as the server reads them now (<see cref="DeepReports.ReadCurrentAsync"/>): one that found
|
||||
/// frames unreadable before their proxies arrived answers 404 <c>analysis</c>, so no editor advises excluding them.
|
||||
/// </summary>
|
||||
private static async Task<IResult> ReportAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive)
|
||||
{
|
||||
if (await db.Sequences.GetAsync(id, context.RequestAborted) is not { } sequence) return ApiErrors.NotFound(context, "sequence");
|
||||
@@ -262,6 +266,16 @@ public static class SequenceEndpoints
|
||||
var path = new Core.Archiving.ArchivePaths(archive.Root).Sequence(sequence.Folder).AnalysisPath;
|
||||
var file = new FileInfo(path);
|
||||
if (!file.Exists || file.LinkTarget is not null || !ArchivePathGuard.IsInside(archive.Root, path)) return ApiErrors.NotFound(context, "analysis");
|
||||
try
|
||||
{
|
||||
if (await DeepReports.ReadCurrentAsync(db, archive.Current, sequence, Core.Imaging.Decoders.DecoderFactory.CreateServer(), context.RequestAborted) is null)
|
||||
return ApiErrors.NotFound(context, "analysis");
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
|
||||
{
|
||||
// archive.json cannot be read now, so whether the proxies count is not known.
|
||||
return ApiErrors.NotFound(context, "analysis");
|
||||
}
|
||||
var etag = string.Create(CultureInfo.InvariantCulture, $"a-{file.Length}-{file.LastWriteTimeUtc.Ticks}");
|
||||
return FileResponses.Serve(context, file, 0, file.Length, "application/json; charset=utf-8", "no-cache", etag) ?? ApiErrors.NotFound(context, "analysis");
|
||||
}
|
||||
|
||||
@@ -7,8 +7,8 @@ namespace Titano.Server.Storage;
|
||||
/// <summary>
|
||||
/// Proxies are files under the sequence's <c>proxies/</c> folder; a thumbnail is the JPEG byte range of its entry in
|
||||
/// <c>thumbs.pack</c>. A thumbnail made from a different original (the frame was renumbered or replaced) counts as not
|
||||
/// built yet, like a missing one, and so does a proxy whose capture time is another frame's
|
||||
/// (<see cref="ArchiveImages.IsProxyOf"/>).
|
||||
/// built yet, like a missing one, and so does a proxy whose capture time is another frame's, or one smaller than the
|
||||
/// archive's policy sent for a frame the server cannot decode (<see cref="ServerProxies.HasProxy"/>).
|
||||
/// </summary>
|
||||
internal sealed class ArchiveAssetLocator : IFrameAssetLocator
|
||||
{
|
||||
@@ -17,13 +17,17 @@ internal sealed class ArchiveAssetLocator : IFrameAssetLocator
|
||||
public ValueTask<AssetLocation?> LocateAsync(Archive archive, Sequence sequence, Frame frame, FrameAsset asset, CancellationToken cancellationToken)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath).Sequence(sequence.Folder);
|
||||
return ValueTask.FromResult(asset == FrameAsset.Proxy ? Proxy(paths, frame) : Thumbnail(paths, frame));
|
||||
return ValueTask.FromResult(asset == FrameAsset.Proxy ? Proxy(archive, paths, frame) : Thumbnail(paths, frame));
|
||||
}
|
||||
|
||||
private static AssetLocation? Proxy(SequencePaths paths, Frame frame)
|
||||
private static AssetLocation? Proxy(Archive archive, SequencePaths paths, Frame frame)
|
||||
{
|
||||
var file = new FileInfo(paths.Proxy(frame.Index));
|
||||
return file.Exists && ArchiveImages.IsProxyOf(file.FullName, frame.CapturedAt) ? new AssetLocation(file.FullName, 0, file.Length, Jpeg) : null;
|
||||
if (!file.Exists) return null;
|
||||
var own = ServerProxies.CannotDecode(frame.Format)
|
||||
? ServerProxies.HasProxy(paths, frame, ServerProxies.Spec(archive))
|
||||
: ArchiveImages.IsProxyOf(file.FullName, frame.CapturedAt, frame.Width, frame.Height, policy: null);
|
||||
return own ? new AssetLocation(file.FullName, 0, file.Length, Jpeg) : null;
|
||||
}
|
||||
|
||||
private static AssetLocation? Thumbnail(SequencePaths paths, Frame frame)
|
||||
|
||||
@@ -34,10 +34,20 @@ public static class ServerProxies
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Every frame of <paramref name="sequence"/> in index order, as a manifest lists it: a proxy is there when the server
|
||||
/// built them all or one was sent for the frame (<see cref="ArchiveImages.IsProxyOf"/>: one left at its index by a
|
||||
/// frame renumbered since is another photo's); it is wanted for a frame the server cannot decode that has none (and is
|
||||
/// not missing), unless <paramref name="spec"/> is null.
|
||||
/// Whether the proxy of <paramref name="frame"/>, one the server cannot decode, is in <paramref name="folder"/>: the
|
||||
/// frame's own (<see cref="ArchiveImages.IsProxyOf"/>: one left at its index by a frame renumbered since is another
|
||||
/// photo's) and as large as <paramref name="spec"/> asks (null: the archive makes none, or its manifest cannot be read
|
||||
/// now, so the size is not judged).
|
||||
/// </summary>
|
||||
public static bool HasProxy(SequencePaths folder, Frame frame, ProxySpec? spec) =>
|
||||
ArchiveImages.IsProxyOf(folder.Proxy(frame.Index), frame.CapturedAt, frame.Width, frame.Height,
|
||||
spec is null ? null : new ProxyPolicy { LongEdge = spec.LongEdge, Quality = spec.Quality });
|
||||
|
||||
/// <summary>
|
||||
/// Every frame of <paramref name="sequence"/> in index order, as a manifest lists it. A frame the server cannot decode
|
||||
/// has its proxy when its own is there (<see cref="HasProxy"/>), whatever the sequence's built status says (frames
|
||||
/// added or renumbered since keep it until the next proxy pass), and wants one otherwise (unless it is missing or
|
||||
/// damaged, or <paramref name="spec"/> is null); any other frame has one when the server built them all.
|
||||
/// </summary>
|
||||
public static async Task<List<ManifestFrame>> FramesAsync(TitanoDatabase db, Archive archive, Sequence sequence, ProxySpec? spec,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -52,9 +62,10 @@ public static class ServerProxies
|
||||
var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, Page, cancellationToken);
|
||||
foreach (var f in batch)
|
||||
{
|
||||
var sent = !proxies && CannotDecode(f.Format) && ArchiveImages.IsProxyOf(folder.Proxy(f.Index), f.CapturedAt);
|
||||
var wanted = spec is not null && !proxies && !sent && CannotDecode(f.Format) && (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0;
|
||||
frames.Add(new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, proxies || sent, thumbs, (long)f.Flags, wanted));
|
||||
var elsewhere = CannotDecode(f.Format);
|
||||
var has = elsewhere ? HasProxy(folder, f, spec) : proxies;
|
||||
var wanted = spec is not null && elsewhere && !has && (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0;
|
||||
frames.Add(new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, has, thumbs, (long)f.Flags, wanted));
|
||||
}
|
||||
if (batch.Count < Page) return frames;
|
||||
after = batch[^1].Index;
|
||||
|
||||
@@ -129,10 +129,11 @@ public sealed class ProxyTests(TestContext context)
|
||||
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1), fx.Ct);
|
||||
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct);
|
||||
|
||||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt));
|
||||
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(1), frames[1].CapturedAt), "another photo's");
|
||||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(1), null), "a frame without a capture time cannot tell");
|
||||
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(2), frames[1].CapturedAt), "none there");
|
||||
var (width, height) = (frames[0].Width, frames[0].Height);
|
||||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, width, height, new ProxyPolicy()));
|
||||
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(1), frames[1].CapturedAt, width, height, new ProxyPolicy()), "another photo's");
|
||||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(1), null, width, height, new ProxyPolicy()), "a frame without a capture time cannot tell");
|
||||
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(2), frames[1].CapturedAt, width, height, null), "none there");
|
||||
|
||||
// Renders and analyses read frame 1 from its original, whether the host takes HEIF from proxies or was asked to.
|
||||
foreach (var useProxies in new[] { false, true })
|
||||
@@ -143,6 +144,120 @@ public sealed class ProxyTests(TestContext context)
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AProxyMatchesItsFrameToTheMicrosecondTheIndexKeeps()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
// A frame timed to the 100 ns tick (seven sub-second digits, or a file time): the sidecar keeps every digit, the
|
||||
// index six.
|
||||
var captured = Start.AddTicks(1_234_567);
|
||||
var staged = fx.Stage(captured, 50);
|
||||
var id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "ticks", staged.Path, "IMG_0001.JPG",
|
||||
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
|
||||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||||
var frame = (await fx.Db.Frames.ListBySequenceAsync(id))[0];
|
||||
Assert.Equal(captured, ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].CapturedAt);
|
||||
Assert.Equal(captured.AddTicks(-7), frame.CapturedAt, "the index keeps microseconds");
|
||||
|
||||
// The proxy pass writes the sidecar's time into the proxy it makes; a render asked to use proxies takes it.
|
||||
var images = fx.Service.Images;
|
||||
var made = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal((1, 0), (made.Built, made.Reused));
|
||||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frame.CapturedAt, frame.Width, frame.Height, null));
|
||||
var loaded = await Titano.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: true, new ImageDecoder([]), fx.Ct);
|
||||
Assert.SequenceEqual([folder.Proxy(0)], loaded.Files);
|
||||
|
||||
// A client makes one from the index's time: it is kept, and the next pass reuses it rather than making its own.
|
||||
var sent = Proxy(frame.CapturedAt!.Value, 1);
|
||||
Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frame, sent));
|
||||
var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal((0, 1), (again.Built, again.Reused));
|
||||
Assert.SequenceEqual(sent, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AThumbnailSizedProxyIsNoProxyOfAFrameReadThroughProxies()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
|
||||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||||
// An earlier version made frame 0's proxy here from the original's 160 × 120 Exif thumbnail.
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1, 160, 120), fx.Ct);
|
||||
|
||||
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, 4032, 3024, new ProxyPolicy()), "under the policy's 1920 px");
|
||||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, 4032, 3024, null), "a host that decodes the original judges no size");
|
||||
// Renders and analyses read the original as the proxy pass would see it: no proxy.
|
||||
var loaded = await Titano.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: false, new ImageDecoder([]), fx.Ct);
|
||||
Assert.SequenceEqual([false, false], loaded.Proxied!);
|
||||
|
||||
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 2, 1920, 1440), fx.Ct);
|
||||
loaded = await Titano.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: false, new ImageDecoder([]), fx.Ct);
|
||||
Assert.SequenceEqual([true, false], loaded.Proxied!);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task PreviewOnlyFollowsTheProxyThereWhenTiersAreRecorded()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
|
||||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||||
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
||||
var tiers = new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
|
||||
// A render decoded both originals from their thumbnails. Frame 0's proxy arrived meanwhile; frame 1's place holds
|
||||
// a thumbnail-sized one an earlier version made.
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1, 1920, 1440), fx.Ct);
|
||||
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[1].CapturedAt!.Value, 2, 160, 120), fx.Ct);
|
||||
var preview = new FileTier(ImageFileFormat.Heif, DecodeTier.EmbeddedPreview, "managed", null, 160, 120);
|
||||
await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier> { [frames[0].Id] = preview, [frames[1].Id] = preview },
|
||||
proxies: false, fx.Ct);
|
||||
|
||||
var recorded = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
|
||||
Assert.Equal((DecodeTier.EmbeddedPreview, (int?)160, FrameFlags.None), (recorded[0].Tier, recorded[0].PreviewW, recorded[0].Flags & FrameFlags.PreviewOnly));
|
||||
Assert.Equal(FrameFlags.PreviewOnly, recorded[1].Flags & FrameFlags.PreviewOnly);
|
||||
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||||
|
||||
// Once frame 0's proxy is gone, an analysis (every frame follows its proxy) makes it preview-only again, and
|
||||
// hands back every frame as the sidecar has it.
|
||||
File.Delete(folder.Proxy(0));
|
||||
var current = await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier>(), proxies: true, fx.Ct);
|
||||
Assert.True(current.Count == 2 && current.All(f => f.Flags.HasFlag(FrameFlags.PreviewOnly)), "both preview-only");
|
||||
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||||
Assert.Count(2, await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier>(), proxies: true, fx.Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUnchangedProxyIsNotReadAgain()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||||
var path = fx.Paths.Sequence(sequence.Folder).Proxy(0);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
var proxy = Proxy(frames[0].CapturedAt!.Value, 1);
|
||||
bool Own() => ArchiveImages.IsProxyOf(path, frames[0].CapturedAt, frames[0].Width, frames[0].Height, null);
|
||||
await File.WriteAllBytesAsync(path, proxy, fx.Ct);
|
||||
var written = DateTime.UtcNow.AddMinutes(-5);
|
||||
File.SetLastWriteTimeUtc(path, written);
|
||||
Assert.True(Own());
|
||||
|
||||
// Same length and write time: a manifest listing thousands of proxies costs a stat per proxy, not a read.
|
||||
await File.WriteAllBytesAsync(path, new byte[proxy.Length], fx.Ct);
|
||||
File.SetLastWriteTimeUtc(path, written);
|
||||
Assert.True(Own(), "known");
|
||||
File.SetLastWriteTimeUtc(path, written.AddSeconds(1));
|
||||
Assert.False(Own(), "written since: read again");
|
||||
|
||||
// A file written moments ago is read every time: its clock may give a quick second write the same time.
|
||||
await File.WriteAllBytesAsync(path, proxy, fx.Ct);
|
||||
var now = File.GetLastWriteTimeUtc(path);
|
||||
Assert.True(Own());
|
||||
await File.WriteAllBytesAsync(path, new byte[proxy.Length], fx.Ct);
|
||||
File.SetLastWriteTimeUtc(path, now);
|
||||
Assert.False(Own(), "read again");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AResumedProxyPassSetsTheBuiltStatusOnlyWhenEveryFrameHasItsProxy()
|
||||
{
|
||||
@@ -242,7 +357,7 @@ public sealed class ProxyTests(TestContext context)
|
||||
|
||||
/// <summary>
|
||||
/// Two ingested frames whose originals the archive engine takes for HEIF (its decoders read none), their size in the
|
||||
/// sidecar <paramref name="width"/> × <paramref name="height"/> when given.
|
||||
/// sidecar and the index <paramref name="width"/> × <paramref name="height"/> when given.
|
||||
/// </summary>
|
||||
private static async Task<(Sequence Sequence, IReadOnlyList<Frame> Frames)> HeifSequenceAsync(ArchiveFixture fx, int? width = null, int? height = null)
|
||||
{
|
||||
@@ -261,7 +376,11 @@ public sealed class ProxyTests(TestContext context)
|
||||
Frames = [.. sidecar.Frames.Select(f => f with { Format = ImageFileFormat.Heif, W = width ?? f.W, H = height ?? f.H })],
|
||||
});
|
||||
await using (var session = await fx.Db.Database.OpenAsync(fx.Ct))
|
||||
{
|
||||
await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE sequence_id = $2", [(int)ImageFileFormat.Heif, id], fx.Ct);
|
||||
if (width is { } w && height is { } h)
|
||||
await session.ExecuteAsync("UPDATE frames SET width = $1, height = $2 WHERE sequence_id = $3", [w, h, id], fx.Ct);
|
||||
}
|
||||
return (sequence, await fx.Db.Frames.ListBySequenceAsync(id));
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Titano.Core.Advice;
|
||||
using Titano.Core.Analysis;
|
||||
using Titano.Core.Analysis.Deep;
|
||||
@@ -5,6 +7,7 @@ using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Dng;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Diagnostics.Synthetic;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Import;
|
||||
@@ -145,10 +148,10 @@ public sealed class AnalysisJobTests(TestContext context)
|
||||
Assert.Equal((AdvisorKeys.ResolutionPreview, "720p-preview"), (capped.Find(AdvisorParameters.Resolution)!.ReasonKey, capped.Find(AdvisorParameters.ExportPreset)!.Value as string));
|
||||
Assert.True(capped.Findings.Any(f => f.Key == FindingKeys.PreviewOnly), "preview-only finding");
|
||||
|
||||
// A client sends their proxies (BRIEF §2.3): the server renders the frames from them, so none is preview-only and
|
||||
// the advice follows. The tier and preview size stay the originals'.
|
||||
// A client sends their proxies (BRIEF §2.3) at the policy's 1920 px: the server renders the frames from them, so none
|
||||
// is preview-only and the advice follows. The tier and preview size stay the originals'.
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
foreach (var frame in rows) await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value), fx.Ct);
|
||||
foreach (var frame in rows) await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct);
|
||||
await LightAnalysisAsync(fx, sequence);
|
||||
var proxied = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
|
||||
Assert.True(proxied.All(f => f is { Tier: DecodeTier.EmbeddedPreview, PreviewW: 160 } && !f.Flags.HasFlag(FrameFlags.PreviewOnly)), "not preview-only");
|
||||
@@ -160,6 +163,49 @@ public sealed class AnalysisJobTests(TestContext context)
|
||||
Assert.False(advice.Findings.Any(f => f.Key == FindingKeys.PreviewOnly), "no preview-only finding");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ProxiesThatArriveDuringADeepAnalysisStillCount()
|
||||
{
|
||||
// A server, whose advisor rules no codec out: it reads these HEIF files only through their 160 × 120 Exif
|
||||
// thumbnails until proxies come (BRIEF §2.3).
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server });
|
||||
var directory = Path.Combine(fx.Archive.Paths.Root, "Heif");
|
||||
Directory.CreateDirectory(directory);
|
||||
var start = new DateTime(2026, 9, 28, 18, 0, 0);
|
||||
for (var i = 0; i < 4; i++)
|
||||
await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024), fx.Ct);
|
||||
await fx.Archive.ScanAsync(ScanMode.Full);
|
||||
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif");
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
|
||||
var folder = fx.Folder(sequence);
|
||||
await fx.StartEngineAsync(workers: 2);
|
||||
async Task<string?> PresetAsync() => ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).Export.Preset.AutoValue;
|
||||
await LightAnalysisAsync(fx, sequence);
|
||||
Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||||
Assert.Equal("720p-preview", await PresetAsync());
|
||||
|
||||
// A deep analysis starts on the originals; while it runs, the proxies arrive and a light analysis on the other
|
||||
// worker counts them.
|
||||
fx.Decoder.Hold = "IMG_0001.HEIC";
|
||||
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))
|
||||
await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct);
|
||||
await LightAnalysisAsync(fx, sequence);
|
||||
Assert.Equal(0, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||||
Assert.Equal("4k-hevc", await PresetAsync());
|
||||
fx.Decoder.Release();
|
||||
await fx.WaitForAsync(deep.Id, JobState.Done);
|
||||
|
||||
// The deep pass decoded the originals from their thumbnails, yet the frames are read from their proxies by now:
|
||||
// it flags none preview-only again, and its advice caps nothing.
|
||||
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Frames.All(f => f.Tier == DecodeTier.EmbeddedPreview && !f.Flags.HasFlag(FrameFlags.PreviewOnly)),
|
||||
"not preview-only");
|
||||
Assert.Equal(0, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||||
Assert.Equal("4k-hevc", await PresetAsync());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives()
|
||||
{
|
||||
@@ -213,6 +259,25 @@ public sealed class AnalysisJobTests(TestContext context)
|
||||
Assert.Less(row.WarningCount, unreadable.WarningCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACheckpointOfASequenceReadFromItsOriginalsKeepsTheKeyEarlierVersionsWrote()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(Flickering(4));
|
||||
var frames = await Titano.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive.Archive, sequence, useProxies: false, new ImageDecoder([fx.Decoder]), fx.Ct);
|
||||
Assert.SequenceEqual([false, false, false, false], frames.Proxied!);
|
||||
|
||||
// A deep analysis interrupted before an upgrade resumes after it (a server restarted mid-run does not start over).
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
foreach (var f in frames.Frames)
|
||||
hash.AppendData(Encoding.UTF8.GetBytes($"{f.Id:N}/{f.Sha256}/{f.Size}/{f.Flags & Titano.Data.Processing.SequenceFrames.NotAnalysed};"));
|
||||
var earlier = Convert.ToHexStringLower(hash.GetHashAndReset())[..16];
|
||||
Assert.Equal(earlier, SequenceAnalyser.ContentKey(frames));
|
||||
|
||||
// A frame read from its proxy since makes the checkpoint stale.
|
||||
Assert.NotEqual(earlier, SequenceAnalyser.ContentKey(frames with { Proxied = [false, true, false, false] }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DeepAnalysisFlagsFramesItCannotDecodeAndListsThemOnce()
|
||||
{
|
||||
@@ -490,10 +555,10 @@ public sealed class AnalysisJobTests(TestContext context)
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
}
|
||||
|
||||
/// <summary>A 320 × 240 JPEG with <paramref name="captured"/> in its Exif, as a client makes a proxy.</summary>
|
||||
private static byte[] Proxy(DateTime captured)
|
||||
/// <summary>A JPEG (320 × 240 unless sized) with <paramref name="captured"/> in its Exif, as a client makes a proxy.</summary>
|
||||
private static byte[] Proxy(DateTime captured, int width = 320, int height = 240)
|
||||
{
|
||||
using var image = new ImageBuffer(320, 240);
|
||||
using var image = new ImageBuffer(width, height);
|
||||
for (var c = 0; c < 3; c++) image.Plane(c).Fill(0.3f + c * 0.1f);
|
||||
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Titano" });
|
||||
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
|
||||
|
||||
@@ -95,7 +95,7 @@ internal sealed class ProcessingFixture : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Managed decoding that records every request and can hold one file's decode until the job is cancelled.</summary>
|
||||
/// <summary>Managed decoding that records every request and can hold one file's decode until the job is cancelled or the decode is released.</summary>
|
||||
internal sealed class GateDecoder : IImageDecoderProvider
|
||||
{
|
||||
private readonly ManagedDecoderProvider _managed = new();
|
||||
@@ -116,13 +116,19 @@ internal sealed class GateDecoder : IImageDecoderProvider
|
||||
/// <summary>Completed when a decode of <see cref="Hold"/> started waiting.</summary>
|
||||
public TaskCompletionSource Holding { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
private TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public void HoldFrame(int index, DecodePurpose? purpose = null, int? edge = null)
|
||||
{
|
||||
Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
(HoldPurpose, HoldEdge) = (purpose, edge);
|
||||
Hold = ProcessingFixture.FileName(index);
|
||||
}
|
||||
|
||||
/// <summary>Lets the held decode go on, as a slow read would.</summary>
|
||||
public void Release() => _released.TrySetResult();
|
||||
|
||||
/// <summary>Tiers <see cref="Probe(string, ImageFileFormat)"/> reports per file name instead of the managed provider's (a
|
||||
/// host that reads them only as previews, or not at all).</summary>
|
||||
public ConcurrentDictionary<string, FileTier> Tiers { get; } = new();
|
||||
@@ -152,7 +158,13 @@ internal sealed class GateDecoder : IImageDecoderProvider
|
||||
&& (HoldEdge ?? request.MaxLongEdge) == request.MaxLongEdge)
|
||||
{
|
||||
Holding.TrySetResult();
|
||||
cancellationToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(30));
|
||||
try
|
||||
{
|
||||
_released.Task.Wait(TimeSpan.FromSeconds(30), cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
}
|
||||
if (Unreadable.ContainsKey(name)) throw new IOException($"The process cannot access the file '{request.Path}' because it is being used by another process.");
|
||||
|
||||
@@ -32,10 +32,14 @@ public sealed class RemoteEditingTests(TestContext context)
|
||||
var sequence = (await server.Database.Sequences.GetAsync(frame.SequenceId))!;
|
||||
var path = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder).AnalysisPath;
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
const string report = """{"version":1,"frameCount":1}""";
|
||||
var report = Reports.Minimal(1).ToJson();
|
||||
await File.WriteAllTextAsync(path, report, context.Cancellation);
|
||||
Assert.Equal(report, await client.GetAnalysisReportAsync(frame.SequenceId, context.Cancellation));
|
||||
|
||||
// One made for other frames is not this sequence's report.
|
||||
await File.WriteAllTextAsync(path, Reports.Minimal(2).ToJson(), context.Cancellation);
|
||||
Assert.Null(await client.GetAnalysisReportAsync(frame.SequenceId, context.Cancellation), "another frame count");
|
||||
|
||||
var gone = await Assert.ThrowsAsync<TitanoApiException>(() => client.GetAnalysisReportAsync(Guid.CreateVersion7(), context.Cancellation));
|
||||
Assert.Equal(HttpStatusCode.NotFound, gone.Status);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
using Titano.Core.Analysis.Deep;
|
||||
|
||||
namespace Titano.Server.Tests;
|
||||
|
||||
/// <summary>Deep-analysis reports as a server keeps them in <c>.titano/analysis.json</c>.</summary>
|
||||
internal static class Reports
|
||||
{
|
||||
/// <summary>A report of <paramref name="frames"/> frames that found <paramref name="undecodable"/> unreadable and measured nothing else.</summary>
|
||||
public static DeepAnalysisReport Minimal(int frames, params int[] undecodable) => new()
|
||||
{
|
||||
FrameCount = frames,
|
||||
Flicker = new FlickerSummary(0, 0, null, 0, 0, 1),
|
||||
Exposure = new ExposureSummary(0, 0, null, null, 0, []),
|
||||
WhiteBalance = new WhiteBalanceSummary(0, 0, null, null),
|
||||
Shake = new ShakeSummary(0, 0, 0, 0, false, 0),
|
||||
Motion = new MotionSummary(0, null, null, null),
|
||||
Sky = new SkySummary(DayNight.Unknown, 0, 0, 0, null, false, null),
|
||||
Issues = new FrameIssues(undecodable, [], [], [], []),
|
||||
};
|
||||
}
|
||||
@@ -3,6 +3,7 @@ using System.Net.Http.Headers;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Protocol;
|
||||
using Titano.Server.Storage;
|
||||
using Titano.Testing;
|
||||
@@ -87,6 +88,69 @@ public sealed class ProxyUploadTests(TestContext context)
|
||||
await Http.BytesEqualAsync(proxy, editor.GetByteArrayAsync($"/api/v1/frames/{frame.Id}/proxy"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FramesAddedAfterEveryProxyArrivedAreAskedForTheirs()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var first = await Uploads.UploadAsync(editor, Heif(1), "IMG_0001.HEIC", sessionKey: "roof");
|
||||
var frame = Assert.NotNull(await server.Database.Frames.GetAsync(first.FrameId));
|
||||
using (var stored = await PutAsync(editor, frame.Id, Proxy(frame.CapturedAt!.Value)))
|
||||
await stored.ExpectAsync(HttpStatusCode.NoContent);
|
||||
|
||||
// A frame comes after the proxy pass found every proxy there: until the next pass the status still says so, yet
|
||||
// the new frame is asked for its proxy and the first keeps its own.
|
||||
var second = await Uploads.UploadAsync(editor, Heif(2), "IMG_0002.HEIC", sessionKey: "roof");
|
||||
Assert.Equal(first.SequenceId, second.SequenceId);
|
||||
await server.Database.Sequences.UpdateStatusAsync(first.SequenceId, SequenceStatus.ProxiesBuilt, SequenceStatus.None, context.Cancellation);
|
||||
var frames = (await ManifestAsync(editor, first.SequenceId)).Frames;
|
||||
Assert.Equal((false, true), Wanted(frames.Single(f => f.Id == first.FrameId)));
|
||||
Assert.Equal((true, false), Wanted(frames.Single(f => f.Id == second.FrameId)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AProxyTheServerMadeIsServedForAFrameTimedToTheTick()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
// Seven sub-second digits: the sidecar, and the proxy made from it, keep them all; the index keeps six.
|
||||
var upload = await Uploads.UploadAsync(editor, Proxy(new DateTime(2026, 9, 28, 18, 2, 11).AddTicks(1_234_567)), "IMG_0001.JPG", sessionKey: "roof");
|
||||
|
||||
// The proxy pass the upload queued makes the frame's proxy, and the server serves it.
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
while (true)
|
||||
{
|
||||
using var proxy = await editor.GetAsync($"/api/v1/frames/{upload.FrameId}/proxy");
|
||||
if (proxy.StatusCode == HttpStatusCode.OK) break;
|
||||
Assert.True(DateTime.UtcNow < until, $"proxy not served: {(int)proxy.StatusCode}");
|
||||
await Task.Delay(100);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AReportThatFoundAFrameUnreadableBeforeItsProxyCameIsNotServed()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var upload = await Uploads.UploadAsync(editor, Heif(1), "IMG_0001.HEIC", sessionKey: "roof");
|
||||
var frame = Assert.NotNull(await server.Database.Frames.GetAsync(upload.FrameId));
|
||||
var sequence = Assert.NotNull(await server.Database.Sequences.GetAsync(upload.SequenceId));
|
||||
// A deep analysis ran before the proxy came and found no pixels in the HEIF original.
|
||||
var path = new Core.Archiving.ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder).AnalysisPath;
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
Reports.Minimal(1, undecodable: 0).Save(path);
|
||||
var url = $"/api/v1/sequences/{sequence.Id}/analysis/report";
|
||||
using (var before = await editor.GetAsync(url))
|
||||
await before.ExpectAsync(HttpStatusCode.OK);
|
||||
|
||||
// Once the frame is read from its proxy, that verdict and its advice to exclude the frame no longer hold, even
|
||||
// before a light analysis has dropped the deep status.
|
||||
using (var stored = await PutAsync(editor, frame.Id, Proxy(frame.CapturedAt!.Value)))
|
||||
await stored.ExpectAsync(HttpStatusCode.NoContent);
|
||||
using var after = await editor.GetAsync(url);
|
||||
Assert.Equal("analysis", (await after.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found")).Details);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FramesTheServerDecodesAskForNoProxy()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user