Files
Chronos/src/Chronos.Data/Archiving/ArchiveImages.cs
T
Alby96andClaude Opus 5.5 7e4a3b481d
CI / Build and test (push) Successful in 9m15s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 15m35s
CI / PostgreSQL 16 integration (push) Successful in 7m26s
CI / PostgreSQL 17 integration (push) Successful in 6m47s
CI / Docker image (push) Successful in 59s
Release / Server image (push) Successful in 1m24s
feat: owner review 2 phase 4a: Reorganize sequences after import (#12)
- Reorganize… on the web panel and the desktop: folder, file and title templates applied to existing sequences, with a dry run and an explicit Apply that runs as a job.
- One naming rule (FrameNaming) for import, upload and Reorganize.
- Crash-safe plan and file journals, held plans, stale-plan refusal; exports move with their folder.
- Writers refuse a folder another program moved or gave to another sequence.
- Server: POST /api/v1/archive/reorganize with a bounded plan cache; template rules.
- Phase 3 docs and translations merged; strings in all 38 languages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-10-27 21:58:39 +01:00

720 lines
43 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Collections.Concurrent;
using System.Diagnostics;
using Chronos.Core.Archiving;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.IO;
using Chronos.Core.Localization;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
namespace Chronos.Data.Archiving;
/// <summary>Frames processed by one thumbnail or proxy pass over a sequence.</summary>
/// <param name="Waiting">Frames of a proxy pass whose originals this host cannot decode and whose proxy it waits for from a
/// client or render node that can (BRIEF §2.3): unlike <paramref name="Undecodable"/> ones, nothing is wrong with them.</param>
public sealed record ImagePassResult(int Built, int Reused, int Undecodable, int Waiting = 0);
/// <summary>What became of a proxy sent by a client or render node (<see cref="ArchiveImages.StoreProxyAsync"/>).</summary>
public enum ProxyStoreResult
{
Stored,
/// <summary>The archive's proxy policy makes no proxies.</summary>
Disabled,
/// <summary>Not a JPEG that decodes, or larger than <see cref="ArchiveImages.MaxProxyBytes"/>.</summary>
Invalid,
/// <summary>Its Exif capture time is not the frame's: it was made from another photo.</summary>
WrongFrame,
/// <summary>Smaller than the archive's proxies of that frame: made from an embedded preview, not the original.</summary>
TooSmall,
/// <summary>The frame's sequence left the index meanwhile (deleted), or its folder (moved by another index): nothing was written.</summary>
Gone,
}
/// <summary>
/// Thumbnails (<c>thumbs.pack</c>) and proxies (<c>proxies/0001.jpg</c>) of sequences (BRIEF §8.5). Both are
/// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after
/// frames were renumbered the stale ones are rebuilt and the rest reused. A frame whose original this host cannot
/// decode (<see cref="NeedsProxy(ImageDecoder, ImageFileFormat)"/>) gets its proxy from a client or render node that can
/// (<see cref="StoreProxyAsync"/>, BRIEF §2.3) and its thumbnail from that proxy. A frame nothing decodes is flagged
/// undecodable (BRIEF §8.2 tier 3); one flagged so that decodes after all gets its tier probed again.
/// </summary>
public sealed class ArchiveImages
{
/// <summary>Largest proxy a client or render node may send: a full-size proxy of a large sensor at q92 fits.</summary>
public const long MaxProxyBytes = 64L << 20;
private const int CommitFrames = 200;
private const int ProbeLongEdge = 64;
/// <summary>Proxy headers kept at most (<see cref="Headers"/>): some 30 MB, the proxies of a few large sequences.</summary>
private const int MaxHeaders = 1 << 17;
private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10);
/// <summary>How long a proxy pass lets the thumbnail lease go between two batches: longer than a waiting writer sleeps between its tries.</summary>
private static readonly TimeSpan BatchGap = LeaseFile.LongestPoll * 1.5;
/// <summary>
/// Age of a proxy file under which its header is not kept: a file system's clock may give a file written again at
/// once (with the same length) the same write time, at a granularity of up to two seconds.
/// </summary>
private static readonly TimeSpan Settling = TimeSpan.FromSeconds(3);
private static readonly ImageDecoder ServerDecoders = DecoderFactory.CreateServer();
/// <summary>
/// The capture time and size of proxies read lately, by path, while the file keeps its length and write time: the
/// server's manifests and tier records check every proxy of a sequence, so an unchanged one costs a stat.
/// </summary>
private static readonly ConcurrentDictionary<string, ProxyHeader> Headers = new(StringComparer.Ordinal);
private readonly ChronosDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
private readonly FrameImageRenderer _renderer;
private readonly FrameTiers _tiers;
internal ArchiveImages(ChronosDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
_renderer = new FrameImageRenderer(writer.Options.Decoder);
_tiers = new FrameTiers(db, writer, indexer);
}
/// <summary>
/// Whether a host with <paramref name="decoder"/> takes the pixels of a <paramref name="format"/> original from its
/// proxy: no decoder reads the format, or only previews of a HEIF, which embeds thumbnails at best (BRIEF §2.3,
/// §8.1: the server keeps HEIF originals and relies on proxies made by a Windows client or render node). Never for a
/// JPEG (proxies are JPEGs: a host that reads them reads it) or a file no format was recognised in.
/// </summary>
public static bool NeedsProxy(ImageDecoder decoder, ImageFileFormat format)
{
ArgumentNullException.ThrowIfNull(decoder);
if (format is ImageFileFormat.Unknown or ImageFileFormat.Jpeg) return false;
var tier = decoder.TierFor(format);
return tier == DecodeTier.Unavailable || (tier != DecodeTier.Native && ImageFormatInfo.Of(format).Family == ImageFamily.Heif);
}
/// <summary><see cref="NeedsProxy(ImageDecoder, ImageFileFormat)"/> with this archive engine's decoders.</summary>
public bool NeedsProxy(ImageFileFormat format) => NeedsProxy(_writer.Options.Decoder, format);
/// <summary>
/// <see cref="NeedsProxy(ImageDecoder, ImageFileFormat)"/> on a Chronos Server, whose decoders are managed only (BRIEF
/// §4.2): it asks clients and render nodes for the proxies of those originals.
/// </summary>
public static bool ServerNeedsProxy(ImageFileFormat format) => NeedsProxy(ServerDecoders, format);
/// <summary>
/// Whether the file at <paramref name="proxy"/> (a sequence's <c>proxies/0001.jpg</c>) is the proxy of the frame
/// captured at <paramref name="capturedAt"/>, of <paramref name="width"/> × <paramref name="height"/>: its Exif capture
/// time is the frame's to the microsecond (the index's precision), or the frame has none to compare with; and, for a
/// frame whose original this host takes from its proxy (<paramref name="policy"/> given: the archive's), it is not
/// smaller than the policy's (<see cref="TooSmall"/>: one an earlier version made from an embedded preview). Proxies
/// are named by frame index, so one left by a frame renumbered since (frames inserted before it) is another photo's.
/// Renders, analyses, decode tiers and the server's manifests and downloads treat a proxy that fails this as missing,
/// and the proxy pass removes it (<see cref="BuildProxiesAsync"/>).
/// </summary>
/// <param name="policy">The archive's proxy policy for a frame whose original this host cannot decode (its proxy comes
/// from a client or render node); null for one this host decodes and makes the proxy of itself.</param>
public static bool IsProxyOf(string proxy, DateTime? capturedAt, int width, int height, ProxyPolicy? policy) =>
Check(proxy, capturedAt, width, height, policy).Check is ProxyCheck.Current or ProxyCheck.Unverifiable;
/// <summary>
/// A proxy of the original at <paramref name="path"/> for a server that cannot decode it (<see cref="StoreProxyAsync"/>):
/// at <paramref name="longEdge"/> (0 = full size) and <paramref name="quality"/>, with <paramref name="frame"/>'s capture
/// data (the server's record, which it checks) and <paramref name="camera"/> in its Exif. Null when
/// <paramref name="decoder"/> cannot decode the original either.
/// </summary>
/// <exception cref="IOException">The original cannot be read.</exception>
public static byte[]? MakeProxy(ImageDecoder decoder, string path, Frame frame, SidecarCamera? camera, int longEdge, int quality,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(frame);
if (NeedsProxy(decoder, frame.Format)) return null;
// The original's folder stands in for a sequence folder: the renderer reads frames relative to one.
var folder = new ArchivePaths(Path.GetDirectoryName(Path.GetFullPath(path))!).Sequence(string.Empty);
var source = new SidecarFrame
{
Id = frame.Id,
I = frame.Index,
File = Path.GetFileName(path),
Size = frame.Size,
CapturedAt = frame.CapturedAt,
UtcOffset = frame.UtcOffsetMinutes,
Exposure = frame.ExposureTime,
FNumber = frame.FNumber,
Iso = frame.Iso,
Focal = frame.FocalLength,
Bias = frame.ExposureBias,
Orientation = frame.Orientation,
Format = frame.Format,
};
var policy = new ProxyPolicy { LongEdge = longEdge, Quality = quality };
return new FrameImageRenderer(decoder).Proxy(folder, source, camera, policy, cancellationToken)?.Jpeg;
}
/// <summary>
/// Files a proxy made elsewhere (a client or render node whose decoders read the original) as the frame's proxy,
/// replacing the one there: a JPEG of at most <see cref="MaxProxyBytes"/> that decodes, whose Exif capture time is the
/// frame's (proxies carry it, so the proxy counts as current and cannot belong to another photo) and whose size is
/// the policy's (<see cref="TooSmall"/>), written atomically under the sequence's lease into its folder as the index has
/// it then: a Reorganize moves the folder under that lease, and the folder it left is never created again (one another
/// index moved, which this index finds at its next scan, takes nothing, nor does that folder when it holds another
/// sequence now: <see cref="ProxyStoreResult.Gone"/>).
/// </summary>
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
/// <exception cref="LeaseUnavailableException">Another writer held the sequence for longer than the lease wait.</exception>
public async Task<ProxyStoreResult> StoreProxyAsync(Archive archive, Sequence sequence, Frame frame, byte[] jpeg, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(jpeg);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
if (policy.LongEdge == ProxyPolicy.Disabled) return ProxyStoreResult.Disabled;
if (jpeg.Length is 0 or > (int)MaxProxyBytes || !Decodes(jpeg)) return ProxyStoreResult.Invalid;
var info = JpegDecoder.ReadInfo(jpeg);
if (TooSmall(System.Math.Max(info.Width, info.Height), frame.Width, frame.Height, policy)) return ProxyStoreResult.TooSmall;
if (frame.CapturedAt is { } captured)
{
using var stream = new MemoryStream(jpeg, writable: false);
if (!SameCapture(MetadataReader.Read(stream, "proxy.jpg").CapturedAt, captured)) return ProxyStoreResult.WrongFrame;
}
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
if (await _db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not { } row) return ProxyStoreResult.Gone;
var folder = paths.Sequence(row.Folder);
if (!SequenceFolders.Holds(folder, row.Id)) return ProxyStoreResult.Gone;
Directory.CreateDirectory(folder.Proxies);
var path = folder.Proxy(frame.Index);
AtomicFile.WriteAllBytes(path, jpeg);
Headers.TryRemove(path, out _);
}
return ProxyStoreResult.Stored;
}
/// <param name="startFrame">Resume point inside the sequence (frames before it were done by an earlier run: the
/// sequence's built status counts them as they are now).</param>
/// <param name="checkpoint">Called with the next frame index whenever the pack has been committed.</param>
/// <remarks>
/// Several frames are drawn at once (<see cref="ThumbnailParallelism"/>); the pack takes them in frame order
/// (<see cref="OrderedWork{T}"/>), so a checkpoint still means that every frame before it is in the pack.
/// </remarks>
public async Task<ImagePassResult> BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame,
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var archiveManifest = ManifestStore.ReadArchive(paths.ManifestPath);
var policy = archiveManifest.ThumbnailPolicy;
SequenceManifest manifest;
int built = 0, reused = 0, undecodable = 0;
var complete = true;
var tiers = new Dictionary<Guid, FileTier>();
// Image passes exclude each other only (a proxy pass holds this lease a batch at a time); ingest keeps committing
// meanwhile. A Reorganize moves the folder under this lease too, so the folder is the one the index has once it is
// held: the pack never opens in a folder the sequence left, which opening it would create again.
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
{
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
manifest = ManifestStore.ReadSequence(folder.ManifestPath);
// Another sequence's folder now (another index moved this one away and that one in): its pack is never opened.
if (manifest.Id != sequence.Id) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
var companions = await GprCompanions.FindAsync(_db, archive, manifest.Frames, cancellationToken).ConfigureAwait(false);
using var pack = ThumbPackWriter.Open(folder.ThumbsPackPath);
for (var i = 0; i < startFrame && i < manifest.Frames.Count; i++)
{
var frame = manifest.Frames[i];
complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
}
var cadence = new Cadence();
await using (var work = new OrderedWork<Drawn>(ThumbnailParallelism(manifest.Frames, policy), drawn =>
{
if (drawn.Image is { } image)
{
pack.Append(drawn.Frame.I, image.Width, image.Height, drawn.Fingerprint, image.Jpeg);
built++;
}
else
{
undecodable++;
}
if (drawn.Tier is { } tier) tiers[drawn.Frame.Id] = tier;
if (!cadence.Due()) return ValueTask.CompletedTask;
pack.Commit();
return checkpoint(drawn.Index + 1, manifest.Frames.Count);
}, cancellationToken))
{
for (var i = startFrame; i < manifest.Frames.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var frame = manifest.Frames[i];
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
var entry = pack.Entries.GetValueOrDefault(frame.I);
var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height);
var current = !force && entry is not null && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
// A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded
// preview, whose smaller thumbnail the proxy's replaces.
var fromProxy = false;
if (NeedsProxy(frame.Format) && !(current && edge >= policy.LongEdge)
&& Check(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, archiveManifest.ProxyPolicy) is { Check: ProxyCheck.Current or ProxyCheck.Unverifiable } proxy)
fromProxy = !current || edge < System.Math.Min(policy.LongEdge, proxy.LongEdge);
if (current && !fromProxy)
{
reused++;
continue;
}
var companion = companions.GetValueOrDefault(frame.Id);
var index = i;
// A thumbnail drawn from the proxy says nothing about how this host decodes the original.
await work.AddAsync(fromProxy
? ct => Drawn.Of(index, frame, fingerprint, Render(folder, frame, null, recordTier: false,
() => _renderer.Thumbnail(folder, ProxySource(frame), policy, ct)))
: ct => Drawn.Of(index, frame, fingerprint, Render(folder, frame, companion, recordTier: true,
() => _renderer.Thumbnail(folder, frame, policy, ct, companion)))).ConfigureAwait(false);
}
await work.CompleteAsync().ConfigureAwait(false);
}
pack.Commit();
}
await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, manifest, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken)
.ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable);
}
/// <inheritdoc cref="BuildThumbnailsAsync"/>
/// <remarks>
/// <see cref="SequenceStatus.ProxiesBuilt"/> is set only when every frame has its proxy: a server asks clients and
/// render nodes for none while it is set (BRIEF §2.3), so a pass resumed at <paramref name="startFrame"/> checks the
/// proxies of the frames an earlier run did again (their headers only). The <see cref="FrameFlags.PreviewOnly"/> flag
/// of frames this host reads through their proxies follows whether they have one (<see cref="FrameTiers"/>). Several
/// originals are decoded at once (<see cref="PassParallelism"/>), and a checkpoint still means that every frame before
/// it has its proxy written (<see cref="OrderedWork{T}"/>). The pass works in batches (<see cref="ProxyBatch"/>), each
/// under the thumbnail lease with the sequence's folder and sidecar read again; a folder that is not where the index
/// has it (another index moved it, until this index's next scan), or that holds another sequence now, stops the pass
/// with an <see cref="InvalidOperationException"/>, nothing written there.
/// </remarks>
public async Task<ImagePassResult> BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame,
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0);
SequenceManifest? manifest = null;
int built = 0, reused = 0, undecodable = 0, waiting = 0;
var complete = true;
var tiers = new Dictionary<Guid, FileTier>();
var cadence = new Cadence();
// Under the thumbnail lease, as the thumbnail pass: image passes exclude each other only, ingest keeps committing
// meanwhile, and a Reorganize waits for the batch, so the proxies go to the folder the index has once the lease is
// held, never into one the sequence left, which writing them would create again. Between two batches the lease is let
// go for longer than a waiting writer sleeps between its tries, so another process's Reorganize, sequence deletion or
// pack compaction waits for one batch, not for the whole pass.
for (var next = 0; ;)
{
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
{
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
if (!SequenceFolders.Holds(folder, sequence.Id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
var read = ManifestStore.ReadSequence(folder.ManifestPath);
// Photos added, removed or renumbered since the last batch: what the earlier batches made may not be theirs.
if (manifest is not null && !SameFrames(read.Frames, manifest.Frames, names: false)) complete = false;
manifest = read;
Directory.CreateDirectory(folder.Proxies);
var companions = await GprCompanions.FindAsync(_db, archive, read.Frames, cancellationToken).ConfigureAwait(false);
var batch = Stopwatch.StartNew();
await using (var work = new OrderedWork<Made>(ProxyParallelism(read.Frames, policy), made =>
{
switch (made.Outcome)
{
case ProxyOutcome.Built:
built++;
break;
case ProxyOutcome.Reused:
reused++;
break;
case ProxyOutcome.Waiting:
waiting++;
break;
case ProxyOutcome.Failed:
undecodable++;
complete = false;
break;
default:
undecodable++;
break;
}
if (made.Tier is { } tier) tiers[made.FrameId] = tier;
return cadence.Due() ? checkpoint(made.Index + 1, read.Frames.Count) : ValueTask.CompletedTask;
}, cancellationToken))
{
// At least one frame per batch, so the pass always moves on.
for (var first = next; next < read.Frames.Count && (next == first || batch.Elapsed < ProxyBatch); next++)
{
cancellationToken.ThrowIfCancellationRequested();
var frame = read.Frames[next];
var path = folder.Proxy(frame.I);
var done = next < startFrame;
if (NeedsProxy(frame.Format))
{
// Only a client or render node that decodes the original makes this proxy: a preview would make one the
// size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again, and so
// does one smaller than the policy's (made here from a preview by an earlier version).
var check = Check(path, frame.CapturedAt, frame.W, frame.H, policy).Check;
var usable = check is ProxyCheck.Current or ProxyCheck.Unverifiable;
if (check == ProxyCheck.Stale) Delete(path);
complete &= usable;
if (done) continue;
var outcome = usable ? ProxyOutcome.Reused
: (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0 ? ProxyOutcome.Unreadable
: ProxyOutcome.Waiting;
await work.AddAsync(new Made(next, frame.Id, outcome, null)).ConfigureAwait(false);
continue;
}
if (done)
{
// Made by the earlier run, unless the frame did not decode then.
complete &= ReadProxy(path, frame.CapturedAt).Check is ProxyCheck.Current or ProxyCheck.Unverifiable;
continue;
}
if (!force && ReadProxy(path, frame.CapturedAt).Check == ProxyCheck.Current)
{
reused++;
continue;
}
var index = next;
var companion = companions.GetValueOrDefault(frame.Id);
await work.AddAsync(ct =>
{
var (image, tier) = Render(folder, frame, companion, recordTier: true,
() => _renderer.Proxy(folder, frame, read.Camera, policy, ct, companion));
if (image is null) return new Made(index, frame.Id, ProxyOutcome.Failed, tier);
AtomicFile.WriteAllBytes(path, image.Jpeg);
Headers.TryRemove(path, out _);
return new Made(index, frame.Id, ProxyOutcome.Built, tier);
}).ConfigureAwait(false);
}
await work.CompleteAsync().ConfigureAwait(false);
}
if (next >= read.Frames.Count) break;
}
await Task.Delay(BatchGap, cancellationToken).ConfigureAwait(false);
}
await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, manifest, JobKind.Proxies, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable, waiting);
}
/// <summary>
/// How long a proxy pass holds the thumbnail lease for one batch of frames: a quarter of the lease wait, so a writer of
/// another process waiting for the lease meanwhile gets it well before it gives up (the frames the batch started finish
/// first).
/// </summary>
private TimeSpan ProxyBatch => _writer.Options.LeaseWait / 4;
/// <summary>Photos a thumbnail pass draws at once (<see cref="PassParallelism"/>): previews of the thumbnail's size serve it.</summary>
private int ThumbnailParallelism(IReadOnlyList<SidecarFrame> frames, ThumbnailPolicy policy) =>
PassParallelism(_writer.Options.Parallelism(), _writer.Options.MemoryBudget(), frames, policy.LongEdge, preferPreview: true, _writer.Options.Decoder);
/// <summary>Originals a proxy pass decodes at once (<see cref="PassParallelism"/>): it develops RAWs, and decodes whole for a full-size policy.</summary>
private int ProxyParallelism(IReadOnlyList<SidecarFrame> frames, ProxyPolicy policy) =>
PassParallelism(_writer.Options.Parallelism(), _writer.Options.MemoryBudget(), frames, System.Math.Max(policy.LongEdge, 0), preferPreview: false,
_writer.Options.Decoder);
/// <summary>
/// Photos a pass decoding at <paramref name="edge"/> (0: full size) works on at once: the whole parallelism while no
/// frame is developed whole (a JPEG at least twice the edge decodes scaled down; a frame this host reads through its
/// proxy or preview develops nothing here, nor one whose embedded preview reaches the edge when previews serve), else
/// <see cref="DecodeBudget.PhotosAtOnce"/> for the largest original developed (a TIFF, a PNG, a RAW).
/// </summary>
internal static int PassParallelism(int parallelism, long memoryBudget, IReadOnlyList<SidecarFrame> frames, int edge, bool preferPreview, ImageDecoder decoder)
{
parallelism = Math.Max(1, parallelism);
long largest = 0;
foreach (var frame in frames)
{
if (NeedsProxy(decoder, frame.Format)) continue;
largest = System.Math.Max(largest, DecodeBudget.Developed(frame.Format, frame.W, frame.H, System.Math.Max(frame.PreviewW ?? 0, frame.PreviewH ?? 0),
edge, preferPreview));
}
return largest == 0 ? parallelism : DecodeBudget.PhotosAtOnce(parallelism, memoryBudget, largest);
}
/// <summary>A thumbnail drawn for the frame at <see cref="Index"/> of the pass (null: nothing decoded it) and what decoding found.</summary>
private sealed record Drawn(int Index, SidecarFrame Frame, byte[] Fingerprint, EncodedImage? Image, FileTier? Tier)
{
public static Drawn Of(int index, SidecarFrame frame, byte[] fingerprint, (EncodedImage? Image, FileTier? Tier) rendered) =>
new(index, frame, fingerprint, rendered.Image, rendered.Tier);
}
/// <summary>What became of the proxy of a frame of a proxy pass.</summary>
private enum ProxyOutcome
{
Built,
/// <summary>The frame this host cannot decode has its proxy from a client or render node.</summary>
Reused,
/// <summary>The frame this host cannot decode waits for its proxy from a client or render node.</summary>
Waiting,
/// <summary>The frame this host cannot decode is missing or damaged.</summary>
Unreadable,
/// <summary>No decoder read the original.</summary>
Failed,
}
/// <summary>The proxy outcome of the frame at <see cref="Index"/> of the pass, and what decoding found.</summary>
private sealed record Made(int Index, Guid FrameId, ProxyOutcome Outcome, FileTier? Tier);
/// <summary>Where a frame's proxy stands; only a capture time tells whether it is the frame's.</summary>
private enum ProxyCheck
{
Missing,
/// <summary>Its Exif capture time is the frame's.</summary>
Current,
/// <summary>The frame has no capture time to compare with.</summary>
Unverifiable,
/// <summary>Made from another frame (frames were renumbered since), or too small (<see cref="Check"/>).</summary>
Stale,
}
/// <summary>A proxy's Exif capture time and oriented long edge, read when the file had this length and write time.</summary>
private readonly record struct ProxyHeader(long Length, long WriteTicks, DateTime? CapturedAt, int LongEdge);
/// <summary>
/// Where the proxy at <paramref name="proxy"/> stands for the frame captured at <paramref name="capturedAt"/>, of
/// <paramref name="width"/> × <paramref name="height"/>, and its long edge: <see cref="ProxyCheck.Stale"/> also when
/// <paramref name="policy"/> is given and the proxy is <see cref="TooSmall"/> for it (<see cref="IsProxyOf"/>).
/// </summary>
private static (ProxyCheck Check, int LongEdge) Check(string proxy, DateTime? capturedAt, int width, int height, ProxyPolicy? policy)
{
var (check, longEdge) = ReadProxy(proxy, capturedAt);
if (check is ProxyCheck.Current or ProxyCheck.Unverifiable && policy is not null && TooSmall(longEdge, width, height, policy)) check = ProxyCheck.Stale;
return (check, longEdge);
}
/// <summary>Where the proxy at <paramref name="proxy"/> stands for the frame captured at <paramref name="capturedAt"/>, and its long edge.</summary>
private static (ProxyCheck Check, int LongEdge) ReadProxy(string proxy, DateTime? capturedAt)
{
if (ReadHeader(proxy) is not { } header) return (ProxyCheck.Missing, 0);
var check = capturedAt is not { } captured ? ProxyCheck.Unverifiable : SameCapture(header.CapturedAt, captured) ? ProxyCheck.Current : ProxyCheck.Stale;
return (check, header.LongEdge);
}
/// <summary>
/// The header of the proxy at <paramref name="proxy"/>; null when there is none or it cannot be read. Kept in
/// <see cref="Headers"/> while the file's length and write time stay as they were when it was read (taken first, so
/// a file replaced meanwhile is read again next time), once the file is older than <see cref="Settling"/>; a full
/// cache starts over.
/// </summary>
private static ProxyHeader? ReadHeader(string proxy)
{
try
{
var file = new FileInfo(proxy);
if (!file.Exists) return null;
var (length, written) = (file.Length, file.LastWriteTimeUtc);
if (Headers.TryGetValue(proxy, out var cached) && cached.Length == length && cached.WriteTicks == written.Ticks) return cached;
var metadata = MetadataReader.Read(proxy);
var (width, height) = metadata.OrientedSize;
var header = new ProxyHeader(length, written.Ticks, metadata.CapturedAt, System.Math.Max(width, height));
if (DateTime.UtcNow - written < Settling)
{
Headers.TryRemove(proxy, out _);
return header;
}
if (Headers.Count >= MaxHeaders) Headers.Clear();
Headers[proxy] = header;
return header;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
return null;
}
}
/// <summary>
/// Whether a proxy's Exif capture time is the frame's <paramref name="captured"/> to the microsecond: the index keeps
/// capture times to the microsecond, sidecars and Exif to the 100 ns tick, so a proxy made from either record of the
/// frame (here from the sidecar, by a client from the server's index) matches both.
/// </summary>
private static bool SameCapture(DateTime? proxy, DateTime captured) =>
proxy is { } time && time.Ticks / TimeSpan.TicksPerMicrosecond == captured.Ticks / TimeSpan.TicksPerMicrosecond;
/// <summary>Removes the proxy at <paramref name="path"/> and what is known of it.</summary>
private static void Delete(string path)
{
File.Delete(path);
Headers.TryRemove(path, out _);
}
/// <summary>
/// Whether a proxy whose long edge is <paramref name="longEdge"/> falls short of the policy's for a frame of
/// <paramref name="width"/> × <paramref name="height"/> (the policy's long edge, or the frame's own when smaller or
/// at full size): one made from an embedded preview. A frame of unknown size is never judged (a small original's
/// proxy would be refused again and again); a tenth of slack spares decoders that crop a few pixels.
/// </summary>
private static bool TooSmall(int longEdge, int width, int height, ProxyPolicy policy)
{
var own = System.Math.Max(width, height);
if (own <= 0) return false;
var expected = policy.LongEdge == ProxyPolicy.FullSize ? own : System.Math.Min(own, policy.LongEdge);
return longEdge * 10L < expected * 9L;
}
/// <summary>The frame's proxy as the frame to draw: a JPEG already oriented, under <c>proxies/</c>.</summary>
private static SidecarFrame ProxySource(SidecarFrame frame) => frame with
{
File = ArchiveLayout.ProxiesFolder + "/" + ArchiveLayout.ProxyFileName(frame.I),
Format = ImageFileFormat.Jpeg,
Orientation = 1,
PreviewOffset = null,
PreviewLength = null,
};
/// <summary>A colour or grey JPEG whose headers parse and whose scans decode (at a small DCT-scaled size).</summary>
private static bool Decodes(byte[] jpeg)
{
try
{
var info = JpegDecoder.ReadInfo(jpeg);
if (info.Process == JpegProcess.Lossless || info.Components is not (1 or 3) || info.Width < 1 || info.Height < 1) return false;
using var probe = JpegDecoder.DecodeToLinear(jpeg, new JpegDecodeOptions { MaxLongEdge = ProbeLongEdge });
return true;
}
catch (Exception ex) when (ex is InvalidDataException or NotSupportedException)
{
return false;
}
}
/// <summary>
/// A frame no decoder can read is skipped and counted; it never stops the pass. With <paramref name="recordTier"/>,
/// what decoding found comes back with the image: unavailable when nothing decoded a file that is there (a read error
/// proves nothing), and the probed tier when a frame flagged undecodable decoded after all. Runs on the pass's
/// workers, so it shares nothing with the others.
/// </summary>
private (EncodedImage? Image, FileTier? Tier) Render(SequencePaths folder, SidecarFrame frame, string? companion, bool recordTier, Func<EncodedImage?> render)
{
var path = folder.File(frame.File);
EncodedImage? image;
try
{
image = render();
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex);
return (null, null);
}
catch (Exception ex) when (ex is InvalidDataException or NotSupportedException)
{
Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex);
image = null;
}
if (!recordTier) return (image, null);
if (image is null)
return (null, File.Exists(path) ? new FileTier(frame.Format, DecodeTier.Unavailable, null, "no decoder produced pixels", 0, 0) : null);
return (image, (frame.Flags & FrameFlags.Undecodable) != 0 ? Reprobe(path, frame, companion, image.Tier) : null);
}
/// <summary>The tier of a frame that decoded after it was flagged undecodable (the image's own tier when probing finds none).</summary>
private FileTier Reprobe(string path, SidecarFrame frame, string? companion, DecodeTier decoded)
{
try
{
if (_writer.Options.Decoder.Probe(path, frame.Format, companion) is { Tier: not DecodeTier.Unavailable } tier) return tier;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("archive", $"{path} could not be probed", ex);
}
return new FileTier(frame.Format, decoded, null, null, 0, 0);
}
/// <summary>Commit/checkpoint rhythm: every <see cref="CommitFrames"/> rendered frames or <see cref="CommitInterval"/>.</summary>
private sealed class Cadence
{
private readonly Stopwatch _clock = Stopwatch.StartNew();
private int _frames;
public bool Due()
{
if (++_frames < CommitFrames && _clock.Elapsed < CommitInterval) return false;
_frames = 0;
_clock.Restart();
return true;
}
}
/// <summary>
/// Sets <paramref name="flag"/> in the sidecar and the index when the pass covered every frame it read
/// (<paramref name="complete"/>) and the sidecar, as it is under the lease, still lists those frames
/// (<see cref="SameFrames"/>); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or
/// renumbered after the pass read <paramref name="read"/> have nothing of it, and a scan that cleared the flag for
/// them queues the next pass only while the flag stays clear. A <paramref name="pass"/> of the same kind already
/// queued for the sequence (an upload's, or a scan's) draws them when it runs, so the flag stays set meanwhile, as
/// between an upload and its pass: a camera uploading faster than a pass ends would otherwise keep it clear, and a
/// server would offer no thumbnail or proxy of the sequence at all until uploads paused. The index is written under
/// the same lease, so the flag never lands there after a scan cleared it in both.
/// </summary>
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, JobKind pass, SequenceStatus flag,
bool complete, CancellationToken cancellationToken)
{
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var built = complete && (SameFrames(manifest.Frames, read.Frames)
|| await _db.Jobs.IsQueuedAsync(pass, sequence.Id, cancellationToken).ConfigureAwait(false));
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var status = built ? manifest.Status | flag : manifest.Status & ~flag;
if (status != manifest.Status)
{
manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
}
await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken)
.ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
}
/// <summary>
/// Whether <paramref name="current"/> lists the photos <paramref name="read"/> did, each at the same index with the
/// same file, size, capture time and content hash: thumbnails and proxies are kept by frame index, so photos added,
/// removed, renumbered or replaced since have none of what the pass made. A hash learnt since (a verify hashing a
/// photo the scan adopted) changes nothing about the photo, and its thumbnail stays its own (<see cref="ThumbPack.IsFrom"/>).
/// </summary>
/// <param name="names">Whether a file renamed since counts as a change (a pass's own batches: it does not, a proxy is
/// kept by frame index and a Reorganize renames files).</param>
private static bool SameFrames(IReadOnlyList<SidecarFrame> current, IReadOnlyList<SidecarFrame> read, bool names = true)
{
if (current.Count != read.Count) return false;
for (var i = 0; i < current.Count; i++)
{
var (now, then) = (current[i], read[i]);
if (now.Id != then.Id || now.I != then.I || (names && now.File != then.File) || now.Size != then.Size || now.CapturedAt != then.CapturedAt
|| (then.Sha256 is not null && !string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase))) return false;
}
return true;
}
}