CI / Build and test (push) Successful in 9m15s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 15m35s
CI / PostgreSQL 16 integration (push) Successful in 7m26s
CI / PostgreSQL 17 integration (push) Successful in 6m47s
CI / Docker image (push) Successful in 59s
Release / Server image (push) Successful in 1m24s
- Reorganize… on the web panel and the desktop: folder, file and title templates applied to existing sequences, with a dry run and an explicit Apply that runs as a job. - One naming rule (FrameNaming) for import, upload and Reorganize. - Crash-safe plan and file journals, held plans, stale-plan refusal; exports move with their folder. - Writers refuse a folder another program moved or gave to another sequence. - Server: POST /api/v1/archive/reorganize with a bounded plan cache; template rules. - Phase 3 docs and translations merged; strings in all 38 languages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
720 lines
43 KiB
C#
720 lines
43 KiB
C#
using System.Collections.Concurrent;
|
||
using System.Diagnostics;
|
||
using Chronos.Core.Archiving;
|
||
using Chronos.Core.Codecs.Jpeg;
|
||
using Chronos.Core.Imaging;
|
||
using Chronos.Core.Imaging.Decoders;
|
||
using Chronos.Core.IO;
|
||
using Chronos.Core.Localization;
|
||
using Chronos.Core.Metadata;
|
||
using Chronos.Core.Model;
|
||
|
||
namespace Chronos.Data.Archiving;
|
||
|
||
/// <summary>Frames processed by one thumbnail or proxy pass over a sequence.</summary>
|
||
/// <param name="Waiting">Frames of a proxy pass whose originals this host cannot decode and whose proxy it waits for from a
|
||
/// client or render node that can (BRIEF §2.3): unlike <paramref name="Undecodable"/> ones, nothing is wrong with them.</param>
|
||
public sealed record ImagePassResult(int Built, int Reused, int Undecodable, int Waiting = 0);
|
||
|
||
/// <summary>What became of a proxy sent by a client or render node (<see cref="ArchiveImages.StoreProxyAsync"/>).</summary>
|
||
public enum ProxyStoreResult
|
||
{
|
||
Stored,
|
||
|
||
/// <summary>The archive's proxy policy makes no proxies.</summary>
|
||
Disabled,
|
||
|
||
/// <summary>Not a JPEG that decodes, or larger than <see cref="ArchiveImages.MaxProxyBytes"/>.</summary>
|
||
Invalid,
|
||
|
||
/// <summary>Its Exif capture time is not the frame's: it was made from another photo.</summary>
|
||
WrongFrame,
|
||
|
||
/// <summary>Smaller than the archive's proxies of that frame: made from an embedded preview, not the original.</summary>
|
||
TooSmall,
|
||
|
||
/// <summary>The frame's sequence left the index meanwhile (deleted), or its folder (moved by another index): nothing was written.</summary>
|
||
Gone,
|
||
}
|
||
|
||
/// <summary>
|
||
/// Thumbnails (<c>thumbs.pack</c>) and proxies (<c>proxies/0001.jpg</c>) of sequences (BRIEF §8.5). Both are
|
||
/// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after
|
||
/// frames were renumbered the stale ones are rebuilt and the rest reused. A frame whose original this host cannot
|
||
/// decode (<see cref="NeedsProxy(ImageDecoder, ImageFileFormat)"/>) gets its proxy from a client or render node that can
|
||
/// (<see cref="StoreProxyAsync"/>, BRIEF §2.3) and its thumbnail from that proxy. A frame nothing decodes is flagged
|
||
/// undecodable (BRIEF §8.2 tier 3); one flagged so that decodes after all gets its tier probed again.
|
||
/// </summary>
|
||
public sealed class ArchiveImages
|
||
{
|
||
/// <summary>Largest proxy a client or render node may send: a full-size proxy of a large sensor at q92 fits.</summary>
|
||
public const long MaxProxyBytes = 64L << 20;
|
||
|
||
private const int CommitFrames = 200;
|
||
private const int ProbeLongEdge = 64;
|
||
|
||
/// <summary>Proxy headers kept at most (<see cref="Headers"/>): some 30 MB, the proxies of a few large sequences.</summary>
|
||
private const int MaxHeaders = 1 << 17;
|
||
|
||
private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10);
|
||
|
||
/// <summary>How long a proxy pass lets the thumbnail lease go between two batches: longer than a waiting writer sleeps between its tries.</summary>
|
||
private static readonly TimeSpan BatchGap = LeaseFile.LongestPoll * 1.5;
|
||
|
||
/// <summary>
|
||
/// Age of a proxy file under which its header is not kept: a file system's clock may give a file written again at
|
||
/// once (with the same length) the same write time, at a granularity of up to two seconds.
|
||
/// </summary>
|
||
private static readonly TimeSpan Settling = TimeSpan.FromSeconds(3);
|
||
|
||
private static readonly ImageDecoder ServerDecoders = DecoderFactory.CreateServer();
|
||
|
||
/// <summary>
|
||
/// The capture time and size of proxies read lately, by path, while the file keeps its length and write time: the
|
||
/// server's manifests and tier records check every proxy of a sequence, so an unchanged one costs a stat.
|
||
/// </summary>
|
||
private static readonly ConcurrentDictionary<string, ProxyHeader> Headers = new(StringComparer.Ordinal);
|
||
|
||
private readonly ChronosDatabase _db;
|
||
private readonly SequenceWriter _writer;
|
||
private readonly ArchiveIndexer _indexer;
|
||
private readonly FrameImageRenderer _renderer;
|
||
private readonly FrameTiers _tiers;
|
||
|
||
internal ArchiveImages(ChronosDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
||
{
|
||
_db = db;
|
||
_writer = writer;
|
||
_indexer = indexer;
|
||
_renderer = new FrameImageRenderer(writer.Options.Decoder);
|
||
_tiers = new FrameTiers(db, writer, indexer);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Whether a host with <paramref name="decoder"/> takes the pixels of a <paramref name="format"/> original from its
|
||
/// proxy: no decoder reads the format, or only previews of a HEIF, which embeds thumbnails at best (BRIEF §2.3,
|
||
/// §8.1: the server keeps HEIF originals and relies on proxies made by a Windows client or render node). Never for a
|
||
/// JPEG (proxies are JPEGs: a host that reads them reads it) or a file no format was recognised in.
|
||
/// </summary>
|
||
public static bool NeedsProxy(ImageDecoder decoder, ImageFileFormat format)
|
||
{
|
||
ArgumentNullException.ThrowIfNull(decoder);
|
||
if (format is ImageFileFormat.Unknown or ImageFileFormat.Jpeg) return false;
|
||
var tier = decoder.TierFor(format);
|
||
return tier == DecodeTier.Unavailable || (tier != DecodeTier.Native && ImageFormatInfo.Of(format).Family == ImageFamily.Heif);
|
||
}
|
||
|
||
/// <summary><see cref="NeedsProxy(ImageDecoder, ImageFileFormat)"/> with this archive engine's decoders.</summary>
|
||
public bool NeedsProxy(ImageFileFormat format) => NeedsProxy(_writer.Options.Decoder, format);
|
||
|
||
/// <summary>
|
||
/// <see cref="NeedsProxy(ImageDecoder, ImageFileFormat)"/> on a Chronos Server, whose decoders are managed only (BRIEF
|
||
/// §4.2): it asks clients and render nodes for the proxies of those originals.
|
||
/// </summary>
|
||
public static bool ServerNeedsProxy(ImageFileFormat format) => NeedsProxy(ServerDecoders, format);
|
||
|
||
/// <summary>
|
||
/// Whether the file at <paramref name="proxy"/> (a sequence's <c>proxies/0001.jpg</c>) is the proxy of the frame
|
||
/// captured at <paramref name="capturedAt"/>, of <paramref name="width"/> × <paramref name="height"/>: its Exif capture
|
||
/// time is the frame's to the microsecond (the index's precision), or the frame has none to compare with; and, for a
|
||
/// frame whose original this host takes from its proxy (<paramref name="policy"/> given: the archive's), it is not
|
||
/// smaller than the policy's (<see cref="TooSmall"/>: one an earlier version made from an embedded preview). Proxies
|
||
/// are named by frame index, so one left by a frame renumbered since (frames inserted before it) is another photo's.
|
||
/// Renders, analyses, decode tiers and the server's manifests and downloads treat a proxy that fails this as missing,
|
||
/// and the proxy pass removes it (<see cref="BuildProxiesAsync"/>).
|
||
/// </summary>
|
||
/// <param name="policy">The archive's proxy policy for a frame whose original this host cannot decode (its proxy comes
|
||
/// from a client or render node); null for one this host decodes and makes the proxy of itself.</param>
|
||
public static bool IsProxyOf(string proxy, DateTime? capturedAt, int width, int height, ProxyPolicy? policy) =>
|
||
Check(proxy, capturedAt, width, height, policy).Check is ProxyCheck.Current or ProxyCheck.Unverifiable;
|
||
|
||
/// <summary>
|
||
/// A proxy of the original at <paramref name="path"/> for a server that cannot decode it (<see cref="StoreProxyAsync"/>):
|
||
/// at <paramref name="longEdge"/> (0 = full size) and <paramref name="quality"/>, with <paramref name="frame"/>'s capture
|
||
/// data (the server's record, which it checks) and <paramref name="camera"/> in its Exif. Null when
|
||
/// <paramref name="decoder"/> cannot decode the original either.
|
||
/// </summary>
|
||
/// <exception cref="IOException">The original cannot be read.</exception>
|
||
public static byte[]? MakeProxy(ImageDecoder decoder, string path, Frame frame, SidecarCamera? camera, int longEdge, int quality,
|
||
CancellationToken cancellationToken = default)
|
||
{
|
||
ArgumentNullException.ThrowIfNull(frame);
|
||
if (NeedsProxy(decoder, frame.Format)) return null;
|
||
// The original's folder stands in for a sequence folder: the renderer reads frames relative to one.
|
||
var folder = new ArchivePaths(Path.GetDirectoryName(Path.GetFullPath(path))!).Sequence(string.Empty);
|
||
var source = new SidecarFrame
|
||
{
|
||
Id = frame.Id,
|
||
I = frame.Index,
|
||
File = Path.GetFileName(path),
|
||
Size = frame.Size,
|
||
CapturedAt = frame.CapturedAt,
|
||
UtcOffset = frame.UtcOffsetMinutes,
|
||
Exposure = frame.ExposureTime,
|
||
FNumber = frame.FNumber,
|
||
Iso = frame.Iso,
|
||
Focal = frame.FocalLength,
|
||
Bias = frame.ExposureBias,
|
||
Orientation = frame.Orientation,
|
||
Format = frame.Format,
|
||
};
|
||
var policy = new ProxyPolicy { LongEdge = longEdge, Quality = quality };
|
||
return new FrameImageRenderer(decoder).Proxy(folder, source, camera, policy, cancellationToken)?.Jpeg;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Files a proxy made elsewhere (a client or render node whose decoders read the original) as the frame's proxy,
|
||
/// replacing the one there: a JPEG of at most <see cref="MaxProxyBytes"/> that decodes, whose Exif capture time is the
|
||
/// frame's (proxies carry it, so the proxy counts as current and cannot belong to another photo) and whose size is
|
||
/// the policy's (<see cref="TooSmall"/>), written atomically under the sequence's lease into its folder as the index has
|
||
/// it then: a Reorganize moves the folder under that lease, and the folder it left is never created again (one another
|
||
/// index moved, which this index finds at its next scan, takes nothing, nor does that folder when it holds another
|
||
/// sequence now: <see cref="ProxyStoreResult.Gone"/>).
|
||
/// </summary>
|
||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
|
||
/// <exception cref="LeaseUnavailableException">Another writer held the sequence for longer than the lease wait.</exception>
|
||
public async Task<ProxyStoreResult> StoreProxyAsync(Archive archive, Sequence sequence, Frame frame, byte[] jpeg, CancellationToken cancellationToken = default)
|
||
{
|
||
ArgumentNullException.ThrowIfNull(jpeg);
|
||
var paths = new ArchivePaths(archive.RootPath);
|
||
ArchiveVolume.EnsureWritable(paths);
|
||
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
|
||
if (policy.LongEdge == ProxyPolicy.Disabled) return ProxyStoreResult.Disabled;
|
||
if (jpeg.Length is 0 or > (int)MaxProxyBytes || !Decodes(jpeg)) return ProxyStoreResult.Invalid;
|
||
var info = JpegDecoder.ReadInfo(jpeg);
|
||
if (TooSmall(System.Math.Max(info.Width, info.Height), frame.Width, frame.Height, policy)) return ProxyStoreResult.TooSmall;
|
||
if (frame.CapturedAt is { } captured)
|
||
{
|
||
using var stream = new MemoryStream(jpeg, writable: false);
|
||
if (!SameCapture(MetadataReader.Read(stream, "proxy.jpg").CapturedAt, captured)) return ProxyStoreResult.WrongFrame;
|
||
}
|
||
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||
{
|
||
if (await _db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not { } row) return ProxyStoreResult.Gone;
|
||
var folder = paths.Sequence(row.Folder);
|
||
if (!SequenceFolders.Holds(folder, row.Id)) return ProxyStoreResult.Gone;
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
var path = folder.Proxy(frame.Index);
|
||
AtomicFile.WriteAllBytes(path, jpeg);
|
||
Headers.TryRemove(path, out _);
|
||
}
|
||
return ProxyStoreResult.Stored;
|
||
}
|
||
|
||
/// <param name="startFrame">Resume point inside the sequence (frames before it were done by an earlier run: the
|
||
/// sequence's built status counts them as they are now).</param>
|
||
/// <param name="checkpoint">Called with the next frame index whenever the pack has been committed.</param>
|
||
/// <remarks>
|
||
/// Several frames are drawn at once (<see cref="ThumbnailParallelism"/>); the pack takes them in frame order
|
||
/// (<see cref="OrderedWork{T}"/>), so a checkpoint still means that every frame before it is in the pack.
|
||
/// </remarks>
|
||
public async Task<ImagePassResult> BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame,
|
||
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
|
||
{
|
||
var paths = new ArchivePaths(archive.RootPath);
|
||
ArchiveVolume.EnsureWritable(paths);
|
||
var archiveManifest = ManifestStore.ReadArchive(paths.ManifestPath);
|
||
var policy = archiveManifest.ThumbnailPolicy;
|
||
SequenceManifest manifest;
|
||
int built = 0, reused = 0, undecodable = 0;
|
||
var complete = true;
|
||
var tiers = new Dictionary<Guid, FileTier>();
|
||
// Image passes exclude each other only (a proxy pass holds this lease a batch at a time); ingest keeps committing
|
||
// meanwhile. A Reorganize moves the folder under this lease too, so the folder is the one the index has once it is
|
||
// held: the pack never opens in a folder the sequence left, which opening it would create again.
|
||
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
|
||
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
|
||
{
|
||
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
|
||
manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||
// Another sequence's folder now (another index moved this one away and that one in): its pack is never opened.
|
||
if (manifest.Id != sequence.Id) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||
var companions = await GprCompanions.FindAsync(_db, archive, manifest.Frames, cancellationToken).ConfigureAwait(false);
|
||
using var pack = ThumbPackWriter.Open(folder.ThumbsPackPath);
|
||
for (var i = 0; i < startFrame && i < manifest.Frames.Count; i++)
|
||
{
|
||
var frame = manifest.Frames[i];
|
||
complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
|
||
}
|
||
var cadence = new Cadence();
|
||
await using (var work = new OrderedWork<Drawn>(ThumbnailParallelism(manifest.Frames, policy), drawn =>
|
||
{
|
||
if (drawn.Image is { } image)
|
||
{
|
||
pack.Append(drawn.Frame.I, image.Width, image.Height, drawn.Fingerprint, image.Jpeg);
|
||
built++;
|
||
}
|
||
else
|
||
{
|
||
undecodable++;
|
||
}
|
||
if (drawn.Tier is { } tier) tiers[drawn.Frame.Id] = tier;
|
||
if (!cadence.Due()) return ValueTask.CompletedTask;
|
||
pack.Commit();
|
||
return checkpoint(drawn.Index + 1, manifest.Frames.Count);
|
||
}, cancellationToken))
|
||
{
|
||
for (var i = startFrame; i < manifest.Frames.Count; i++)
|
||
{
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
var frame = manifest.Frames[i];
|
||
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
|
||
var entry = pack.Entries.GetValueOrDefault(frame.I);
|
||
var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height);
|
||
var current = !force && entry is not null && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
|
||
// A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded
|
||
// preview, whose smaller thumbnail the proxy's replaces.
|
||
var fromProxy = false;
|
||
if (NeedsProxy(frame.Format) && !(current && edge >= policy.LongEdge)
|
||
&& Check(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, archiveManifest.ProxyPolicy) is { Check: ProxyCheck.Current or ProxyCheck.Unverifiable } proxy)
|
||
fromProxy = !current || edge < System.Math.Min(policy.LongEdge, proxy.LongEdge);
|
||
if (current && !fromProxy)
|
||
{
|
||
reused++;
|
||
continue;
|
||
}
|
||
var companion = companions.GetValueOrDefault(frame.Id);
|
||
var index = i;
|
||
// A thumbnail drawn from the proxy says nothing about how this host decodes the original.
|
||
await work.AddAsync(fromProxy
|
||
? ct => Drawn.Of(index, frame, fingerprint, Render(folder, frame, null, recordTier: false,
|
||
() => _renderer.Thumbnail(folder, ProxySource(frame), policy, ct)))
|
||
: ct => Drawn.Of(index, frame, fingerprint, Render(folder, frame, companion, recordTier: true,
|
||
() => _renderer.Thumbnail(folder, frame, policy, ct, companion)))).ConfigureAwait(false);
|
||
}
|
||
await work.CompleteAsync().ConfigureAwait(false);
|
||
}
|
||
pack.Commit();
|
||
}
|
||
await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false);
|
||
await MarkAsync(archive, paths, sequence, manifest, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken)
|
||
.ConfigureAwait(false);
|
||
return new ImagePassResult(built, reused, undecodable);
|
||
}
|
||
|
||
/// <inheritdoc cref="BuildThumbnailsAsync"/>
|
||
/// <remarks>
|
||
/// <see cref="SequenceStatus.ProxiesBuilt"/> is set only when every frame has its proxy: a server asks clients and
|
||
/// render nodes for none while it is set (BRIEF §2.3), so a pass resumed at <paramref name="startFrame"/> checks the
|
||
/// proxies of the frames an earlier run did again (their headers only). The <see cref="FrameFlags.PreviewOnly"/> flag
|
||
/// of frames this host reads through their proxies follows whether they have one (<see cref="FrameTiers"/>). Several
|
||
/// originals are decoded at once (<see cref="PassParallelism"/>), and a checkpoint still means that every frame before
|
||
/// it has its proxy written (<see cref="OrderedWork{T}"/>). The pass works in batches (<see cref="ProxyBatch"/>), each
|
||
/// under the thumbnail lease with the sequence's folder and sidecar read again; a folder that is not where the index
|
||
/// has it (another index moved it, until this index's next scan), or that holds another sequence now, stops the pass
|
||
/// with an <see cref="InvalidOperationException"/>, nothing written there.
|
||
/// </remarks>
|
||
public async Task<ImagePassResult> BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame,
|
||
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
|
||
{
|
||
var paths = new ArchivePaths(archive.RootPath);
|
||
ArchiveVolume.EnsureWritable(paths);
|
||
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
|
||
if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0);
|
||
SequenceManifest? manifest = null;
|
||
int built = 0, reused = 0, undecodable = 0, waiting = 0;
|
||
var complete = true;
|
||
var tiers = new Dictionary<Guid, FileTier>();
|
||
var cadence = new Cadence();
|
||
// Under the thumbnail lease, as the thumbnail pass: image passes exclude each other only, ingest keeps committing
|
||
// meanwhile, and a Reorganize waits for the batch, so the proxies go to the folder the index has once the lease is
|
||
// held, never into one the sequence left, which writing them would create again. Between two batches the lease is let
|
||
// go for longer than a waiting writer sleeps between its tries, so another process's Reorganize, sequence deletion or
|
||
// pack compaction waits for one batch, not for the whole pass.
|
||
for (var next = 0; ;)
|
||
{
|
||
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
|
||
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
|
||
{
|
||
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
|
||
if (!SequenceFolders.Holds(folder, sequence.Id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||
var read = ManifestStore.ReadSequence(folder.ManifestPath);
|
||
// Photos added, removed or renumbered since the last batch: what the earlier batches made may not be theirs.
|
||
if (manifest is not null && !SameFrames(read.Frames, manifest.Frames, names: false)) complete = false;
|
||
manifest = read;
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
var companions = await GprCompanions.FindAsync(_db, archive, read.Frames, cancellationToken).ConfigureAwait(false);
|
||
var batch = Stopwatch.StartNew();
|
||
await using (var work = new OrderedWork<Made>(ProxyParallelism(read.Frames, policy), made =>
|
||
{
|
||
switch (made.Outcome)
|
||
{
|
||
case ProxyOutcome.Built:
|
||
built++;
|
||
break;
|
||
case ProxyOutcome.Reused:
|
||
reused++;
|
||
break;
|
||
case ProxyOutcome.Waiting:
|
||
waiting++;
|
||
break;
|
||
case ProxyOutcome.Failed:
|
||
undecodable++;
|
||
complete = false;
|
||
break;
|
||
default:
|
||
undecodable++;
|
||
break;
|
||
}
|
||
if (made.Tier is { } tier) tiers[made.FrameId] = tier;
|
||
return cadence.Due() ? checkpoint(made.Index + 1, read.Frames.Count) : ValueTask.CompletedTask;
|
||
}, cancellationToken))
|
||
{
|
||
// At least one frame per batch, so the pass always moves on.
|
||
for (var first = next; next < read.Frames.Count && (next == first || batch.Elapsed < ProxyBatch); next++)
|
||
{
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
var frame = read.Frames[next];
|
||
var path = folder.Proxy(frame.I);
|
||
var done = next < startFrame;
|
||
if (NeedsProxy(frame.Format))
|
||
{
|
||
// Only a client or render node that decodes the original makes this proxy: a preview would make one the
|
||
// size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again, and so
|
||
// does one smaller than the policy's (made here from a preview by an earlier version).
|
||
var check = Check(path, frame.CapturedAt, frame.W, frame.H, policy).Check;
|
||
var usable = check is ProxyCheck.Current or ProxyCheck.Unverifiable;
|
||
if (check == ProxyCheck.Stale) Delete(path);
|
||
complete &= usable;
|
||
if (done) continue;
|
||
var outcome = usable ? ProxyOutcome.Reused
|
||
: (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0 ? ProxyOutcome.Unreadable
|
||
: ProxyOutcome.Waiting;
|
||
await work.AddAsync(new Made(next, frame.Id, outcome, null)).ConfigureAwait(false);
|
||
continue;
|
||
}
|
||
if (done)
|
||
{
|
||
// Made by the earlier run, unless the frame did not decode then.
|
||
complete &= ReadProxy(path, frame.CapturedAt).Check is ProxyCheck.Current or ProxyCheck.Unverifiable;
|
||
continue;
|
||
}
|
||
if (!force && ReadProxy(path, frame.CapturedAt).Check == ProxyCheck.Current)
|
||
{
|
||
reused++;
|
||
continue;
|
||
}
|
||
var index = next;
|
||
var companion = companions.GetValueOrDefault(frame.Id);
|
||
await work.AddAsync(ct =>
|
||
{
|
||
var (image, tier) = Render(folder, frame, companion, recordTier: true,
|
||
() => _renderer.Proxy(folder, frame, read.Camera, policy, ct, companion));
|
||
if (image is null) return new Made(index, frame.Id, ProxyOutcome.Failed, tier);
|
||
AtomicFile.WriteAllBytes(path, image.Jpeg);
|
||
Headers.TryRemove(path, out _);
|
||
return new Made(index, frame.Id, ProxyOutcome.Built, tier);
|
||
}).ConfigureAwait(false);
|
||
}
|
||
await work.CompleteAsync().ConfigureAwait(false);
|
||
}
|
||
if (next >= read.Frames.Count) break;
|
||
}
|
||
await Task.Delay(BatchGap, cancellationToken).ConfigureAwait(false);
|
||
}
|
||
await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false);
|
||
await MarkAsync(archive, paths, sequence, manifest, JobKind.Proxies, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
|
||
return new ImagePassResult(built, reused, undecodable, waiting);
|
||
}
|
||
|
||
/// <summary>
|
||
/// How long a proxy pass holds the thumbnail lease for one batch of frames: a quarter of the lease wait, so a writer of
|
||
/// another process waiting for the lease meanwhile gets it well before it gives up (the frames the batch started finish
|
||
/// first).
|
||
/// </summary>
|
||
private TimeSpan ProxyBatch => _writer.Options.LeaseWait / 4;
|
||
|
||
/// <summary>Photos a thumbnail pass draws at once (<see cref="PassParallelism"/>): previews of the thumbnail's size serve it.</summary>
|
||
private int ThumbnailParallelism(IReadOnlyList<SidecarFrame> frames, ThumbnailPolicy policy) =>
|
||
PassParallelism(_writer.Options.Parallelism(), _writer.Options.MemoryBudget(), frames, policy.LongEdge, preferPreview: true, _writer.Options.Decoder);
|
||
|
||
/// <summary>Originals a proxy pass decodes at once (<see cref="PassParallelism"/>): it develops RAWs, and decodes whole for a full-size policy.</summary>
|
||
private int ProxyParallelism(IReadOnlyList<SidecarFrame> frames, ProxyPolicy policy) =>
|
||
PassParallelism(_writer.Options.Parallelism(), _writer.Options.MemoryBudget(), frames, System.Math.Max(policy.LongEdge, 0), preferPreview: false,
|
||
_writer.Options.Decoder);
|
||
|
||
/// <summary>
|
||
/// Photos a pass decoding at <paramref name="edge"/> (0: full size) works on at once: the whole parallelism while no
|
||
/// frame is developed whole (a JPEG at least twice the edge decodes scaled down; a frame this host reads through its
|
||
/// proxy or preview develops nothing here, nor one whose embedded preview reaches the edge when previews serve), else
|
||
/// <see cref="DecodeBudget.PhotosAtOnce"/> for the largest original developed (a TIFF, a PNG, a RAW).
|
||
/// </summary>
|
||
internal static int PassParallelism(int parallelism, long memoryBudget, IReadOnlyList<SidecarFrame> frames, int edge, bool preferPreview, ImageDecoder decoder)
|
||
{
|
||
parallelism = Math.Max(1, parallelism);
|
||
long largest = 0;
|
||
foreach (var frame in frames)
|
||
{
|
||
if (NeedsProxy(decoder, frame.Format)) continue;
|
||
largest = System.Math.Max(largest, DecodeBudget.Developed(frame.Format, frame.W, frame.H, System.Math.Max(frame.PreviewW ?? 0, frame.PreviewH ?? 0),
|
||
edge, preferPreview));
|
||
}
|
||
return largest == 0 ? parallelism : DecodeBudget.PhotosAtOnce(parallelism, memoryBudget, largest);
|
||
}
|
||
|
||
/// <summary>A thumbnail drawn for the frame at <see cref="Index"/> of the pass (null: nothing decoded it) and what decoding found.</summary>
|
||
private sealed record Drawn(int Index, SidecarFrame Frame, byte[] Fingerprint, EncodedImage? Image, FileTier? Tier)
|
||
{
|
||
public static Drawn Of(int index, SidecarFrame frame, byte[] fingerprint, (EncodedImage? Image, FileTier? Tier) rendered) =>
|
||
new(index, frame, fingerprint, rendered.Image, rendered.Tier);
|
||
}
|
||
|
||
/// <summary>What became of the proxy of a frame of a proxy pass.</summary>
|
||
private enum ProxyOutcome
|
||
{
|
||
Built,
|
||
|
||
/// <summary>The frame this host cannot decode has its proxy from a client or render node.</summary>
|
||
Reused,
|
||
|
||
/// <summary>The frame this host cannot decode waits for its proxy from a client or render node.</summary>
|
||
Waiting,
|
||
|
||
/// <summary>The frame this host cannot decode is missing or damaged.</summary>
|
||
Unreadable,
|
||
|
||
/// <summary>No decoder read the original.</summary>
|
||
Failed,
|
||
}
|
||
|
||
/// <summary>The proxy outcome of the frame at <see cref="Index"/> of the pass, and what decoding found.</summary>
|
||
private sealed record Made(int Index, Guid FrameId, ProxyOutcome Outcome, FileTier? Tier);
|
||
|
||
/// <summary>Where a frame's proxy stands; only a capture time tells whether it is the frame's.</summary>
|
||
private enum ProxyCheck
|
||
{
|
||
Missing,
|
||
|
||
/// <summary>Its Exif capture time is the frame's.</summary>
|
||
Current,
|
||
|
||
/// <summary>The frame has no capture time to compare with.</summary>
|
||
Unverifiable,
|
||
|
||
/// <summary>Made from another frame (frames were renumbered since), or too small (<see cref="Check"/>).</summary>
|
||
Stale,
|
||
}
|
||
|
||
/// <summary>A proxy's Exif capture time and oriented long edge, read when the file had this length and write time.</summary>
|
||
private readonly record struct ProxyHeader(long Length, long WriteTicks, DateTime? CapturedAt, int LongEdge);
|
||
|
||
/// <summary>
|
||
/// Where the proxy at <paramref name="proxy"/> stands for the frame captured at <paramref name="capturedAt"/>, of
|
||
/// <paramref name="width"/> × <paramref name="height"/>, and its long edge: <see cref="ProxyCheck.Stale"/> also when
|
||
/// <paramref name="policy"/> is given and the proxy is <see cref="TooSmall"/> for it (<see cref="IsProxyOf"/>).
|
||
/// </summary>
|
||
private static (ProxyCheck Check, int LongEdge) Check(string proxy, DateTime? capturedAt, int width, int height, ProxyPolicy? policy)
|
||
{
|
||
var (check, longEdge) = ReadProxy(proxy, capturedAt);
|
||
if (check is ProxyCheck.Current or ProxyCheck.Unverifiable && policy is not null && TooSmall(longEdge, width, height, policy)) check = ProxyCheck.Stale;
|
||
return (check, longEdge);
|
||
}
|
||
|
||
/// <summary>Where the proxy at <paramref name="proxy"/> stands for the frame captured at <paramref name="capturedAt"/>, and its long edge.</summary>
|
||
private static (ProxyCheck Check, int LongEdge) ReadProxy(string proxy, DateTime? capturedAt)
|
||
{
|
||
if (ReadHeader(proxy) is not { } header) return (ProxyCheck.Missing, 0);
|
||
var check = capturedAt is not { } captured ? ProxyCheck.Unverifiable : SameCapture(header.CapturedAt, captured) ? ProxyCheck.Current : ProxyCheck.Stale;
|
||
return (check, header.LongEdge);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The header of the proxy at <paramref name="proxy"/>; null when there is none or it cannot be read. Kept in
|
||
/// <see cref="Headers"/> while the file's length and write time stay as they were when it was read (taken first, so
|
||
/// a file replaced meanwhile is read again next time), once the file is older than <see cref="Settling"/>; a full
|
||
/// cache starts over.
|
||
/// </summary>
|
||
private static ProxyHeader? ReadHeader(string proxy)
|
||
{
|
||
try
|
||
{
|
||
var file = new FileInfo(proxy);
|
||
if (!file.Exists) return null;
|
||
var (length, written) = (file.Length, file.LastWriteTimeUtc);
|
||
if (Headers.TryGetValue(proxy, out var cached) && cached.Length == length && cached.WriteTicks == written.Ticks) return cached;
|
||
var metadata = MetadataReader.Read(proxy);
|
||
var (width, height) = metadata.OrientedSize;
|
||
var header = new ProxyHeader(length, written.Ticks, metadata.CapturedAt, System.Math.Max(width, height));
|
||
if (DateTime.UtcNow - written < Settling)
|
||
{
|
||
Headers.TryRemove(proxy, out _);
|
||
return header;
|
||
}
|
||
if (Headers.Count >= MaxHeaders) Headers.Clear();
|
||
Headers[proxy] = header;
|
||
return header;
|
||
}
|
||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Whether a proxy's Exif capture time is the frame's <paramref name="captured"/> to the microsecond: the index keeps
|
||
/// capture times to the microsecond, sidecars and Exif to the 100 ns tick, so a proxy made from either record of the
|
||
/// frame (here from the sidecar, by a client from the server's index) matches both.
|
||
/// </summary>
|
||
private static bool SameCapture(DateTime? proxy, DateTime captured) =>
|
||
proxy is { } time && time.Ticks / TimeSpan.TicksPerMicrosecond == captured.Ticks / TimeSpan.TicksPerMicrosecond;
|
||
|
||
/// <summary>Removes the proxy at <paramref name="path"/> and what is known of it.</summary>
|
||
private static void Delete(string path)
|
||
{
|
||
File.Delete(path);
|
||
Headers.TryRemove(path, out _);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Whether a proxy whose long edge is <paramref name="longEdge"/> falls short of the policy's for a frame of
|
||
/// <paramref name="width"/> × <paramref name="height"/> (the policy's long edge, or the frame's own when smaller or
|
||
/// at full size): one made from an embedded preview. A frame of unknown size is never judged (a small original's
|
||
/// proxy would be refused again and again); a tenth of slack spares decoders that crop a few pixels.
|
||
/// </summary>
|
||
private static bool TooSmall(int longEdge, int width, int height, ProxyPolicy policy)
|
||
{
|
||
var own = System.Math.Max(width, height);
|
||
if (own <= 0) return false;
|
||
var expected = policy.LongEdge == ProxyPolicy.FullSize ? own : System.Math.Min(own, policy.LongEdge);
|
||
return longEdge * 10L < expected * 9L;
|
||
}
|
||
|
||
/// <summary>The frame's proxy as the frame to draw: a JPEG already oriented, under <c>proxies/</c>.</summary>
|
||
private static SidecarFrame ProxySource(SidecarFrame frame) => frame with
|
||
{
|
||
File = ArchiveLayout.ProxiesFolder + "/" + ArchiveLayout.ProxyFileName(frame.I),
|
||
Format = ImageFileFormat.Jpeg,
|
||
Orientation = 1,
|
||
PreviewOffset = null,
|
||
PreviewLength = null,
|
||
};
|
||
|
||
/// <summary>A colour or grey JPEG whose headers parse and whose scans decode (at a small DCT-scaled size).</summary>
|
||
private static bool Decodes(byte[] jpeg)
|
||
{
|
||
try
|
||
{
|
||
var info = JpegDecoder.ReadInfo(jpeg);
|
||
if (info.Process == JpegProcess.Lossless || info.Components is not (1 or 3) || info.Width < 1 || info.Height < 1) return false;
|
||
using var probe = JpegDecoder.DecodeToLinear(jpeg, new JpegDecodeOptions { MaxLongEdge = ProbeLongEdge });
|
||
return true;
|
||
}
|
||
catch (Exception ex) when (ex is InvalidDataException or NotSupportedException)
|
||
{
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// A frame no decoder can read is skipped and counted; it never stops the pass. With <paramref name="recordTier"/>,
|
||
/// what decoding found comes back with the image: unavailable when nothing decoded a file that is there (a read error
|
||
/// proves nothing), and the probed tier when a frame flagged undecodable decoded after all. Runs on the pass's
|
||
/// workers, so it shares nothing with the others.
|
||
/// </summary>
|
||
private (EncodedImage? Image, FileTier? Tier) Render(SequencePaths folder, SidecarFrame frame, string? companion, bool recordTier, Func<EncodedImage?> render)
|
||
{
|
||
var path = folder.File(frame.File);
|
||
EncodedImage? image;
|
||
try
|
||
{
|
||
image = render();
|
||
}
|
||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex);
|
||
return (null, null);
|
||
}
|
||
catch (Exception ex) when (ex is InvalidDataException or NotSupportedException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex);
|
||
image = null;
|
||
}
|
||
if (!recordTier) return (image, null);
|
||
if (image is null)
|
||
return (null, File.Exists(path) ? new FileTier(frame.Format, DecodeTier.Unavailable, null, "no decoder produced pixels", 0, 0) : null);
|
||
return (image, (frame.Flags & FrameFlags.Undecodable) != 0 ? Reprobe(path, frame, companion, image.Tier) : null);
|
||
}
|
||
|
||
/// <summary>The tier of a frame that decoded after it was flagged undecodable (the image's own tier when probing finds none).</summary>
|
||
private FileTier Reprobe(string path, SidecarFrame frame, string? companion, DecodeTier decoded)
|
||
{
|
||
try
|
||
{
|
||
if (_writer.Options.Decoder.Probe(path, frame.Format, companion) is { Tier: not DecodeTier.Unavailable } tier) return tier;
|
||
}
|
||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"{path} could not be probed", ex);
|
||
}
|
||
return new FileTier(frame.Format, decoded, null, null, 0, 0);
|
||
}
|
||
|
||
/// <summary>Commit/checkpoint rhythm: every <see cref="CommitFrames"/> rendered frames or <see cref="CommitInterval"/>.</summary>
|
||
private sealed class Cadence
|
||
{
|
||
private readonly Stopwatch _clock = Stopwatch.StartNew();
|
||
private int _frames;
|
||
|
||
public bool Due()
|
||
{
|
||
if (++_frames < CommitFrames && _clock.Elapsed < CommitInterval) return false;
|
||
_frames = 0;
|
||
_clock.Restart();
|
||
return true;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Sets <paramref name="flag"/> in the sidecar and the index when the pass covered every frame it read
|
||
/// (<paramref name="complete"/>) and the sidecar, as it is under the lease, still lists those frames
|
||
/// (<see cref="SameFrames"/>); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or
|
||
/// renumbered after the pass read <paramref name="read"/> have nothing of it, and a scan that cleared the flag for
|
||
/// them queues the next pass only while the flag stays clear. A <paramref name="pass"/> of the same kind already
|
||
/// queued for the sequence (an upload's, or a scan's) draws them when it runs, so the flag stays set meanwhile, as
|
||
/// between an upload and its pass: a camera uploading faster than a pass ends would otherwise keep it clear, and a
|
||
/// server would offer no thumbnail or proxy of the sequence at all until uploads paused. The index is written under
|
||
/// the same lease, so the flag never lands there after a scan cleared it in both.
|
||
/// </summary>
|
||
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, JobKind pass, SequenceStatus flag,
|
||
bool complete, CancellationToken cancellationToken)
|
||
{
|
||
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||
{
|
||
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
|
||
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
|
||
var built = complete && (SameFrames(manifest.Frames, read.Frames)
|
||
|| await _db.Jobs.IsQueuedAsync(pass, sequence.Id, cancellationToken).ConfigureAwait(false));
|
||
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||
var status = built ? manifest.Status | flag : manifest.Status & ~flag;
|
||
if (status != manifest.Status)
|
||
{
|
||
manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
|
||
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
|
||
}
|
||
await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken)
|
||
.ConfigureAwait(false);
|
||
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Whether <paramref name="current"/> lists the photos <paramref name="read"/> did, each at the same index with the
|
||
/// same file, size, capture time and content hash: thumbnails and proxies are kept by frame index, so photos added,
|
||
/// removed, renumbered or replaced since have none of what the pass made. A hash learnt since (a verify hashing a
|
||
/// photo the scan adopted) changes nothing about the photo, and its thumbnail stays its own (<see cref="ThumbPack.IsFrom"/>).
|
||
/// </summary>
|
||
/// <param name="names">Whether a file renamed since counts as a change (a pass's own batches: it does not, a proxy is
|
||
/// kept by frame index and a Reorganize renames files).</param>
|
||
private static bool SameFrames(IReadOnlyList<SidecarFrame> current, IReadOnlyList<SidecarFrame> read, bool names = true)
|
||
{
|
||
if (current.Count != read.Count) return false;
|
||
for (var i = 0; i < current.Count; i++)
|
||
{
|
||
var (now, then) = (current[i], read[i]);
|
||
if (now.Id != then.Id || now.I != then.I || (names && now.File != then.File) || now.Size != then.Size || now.CapturedAt != then.CapturedAt
|
||
|| (then.Sha256 is not null && !string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase))) return false;
|
||
}
|
||
return true;
|
||
}
|
||
}
|