using System.Collections.Concurrent;
using System.Diagnostics;
using Chronos.Core.Archiving;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.IO;
using Chronos.Core.Localization;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
namespace Chronos.Data.Archiving;
/// Frames processed by one thumbnail or proxy pass over a sequence.
/// Frames of a proxy pass whose originals this host cannot decode and whose proxy it waits for from a
/// client or render node that can (BRIEF §2.3): unlike ones, nothing is wrong with them.
public sealed record ImagePassResult(int Built, int Reused, int Undecodable, int Waiting = 0);
/// What became of a proxy sent by a client or render node ().
public enum ProxyStoreResult
{
Stored,
/// The archive's proxy policy makes no proxies.
Disabled,
/// Not a JPEG that decodes, or larger than .
Invalid,
/// Its Exif capture time is not the frame's: it was made from another photo.
WrongFrame,
/// Smaller than the archive's proxies of that frame: made from an embedded preview, not the original.
TooSmall,
/// The frame's sequence left the index meanwhile (deleted), or its folder (moved by another index): nothing was written.
Gone,
}
///
/// Thumbnails (thumbs.pack) and proxies (proxies/0001.jpg) of sequences (BRIEF §8.5). Both are
/// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after
/// frames were renumbered the stale ones are rebuilt and the rest reused. A frame whose original this host cannot
/// decode () gets its proxy from a client or render node that can
/// (, BRIEF §2.3) and its thumbnail from that proxy. A frame nothing decodes is flagged
/// undecodable (BRIEF §8.2 tier 3); one flagged so that decodes after all gets its tier probed again.
///
public sealed class ArchiveImages
{
/// Largest proxy a client or render node may send: a full-size proxy of a large sensor at q92 fits.
public const long MaxProxyBytes = 64L << 20;
private const int CommitFrames = 200;
private const int ProbeLongEdge = 64;
/// Proxy headers kept at most (): some 30 MB, the proxies of a few large sequences.
private const int MaxHeaders = 1 << 17;
private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10);
/// How long a proxy pass lets the thumbnail lease go between two batches: longer than a waiting writer sleeps between its tries.
private static readonly TimeSpan BatchGap = LeaseFile.LongestPoll * 1.5;
///
/// Age of a proxy file under which its header is not kept: a file system's clock may give a file written again at
/// once (with the same length) the same write time, at a granularity of up to two seconds.
///
private static readonly TimeSpan Settling = TimeSpan.FromSeconds(3);
private static readonly ImageDecoder ServerDecoders = DecoderFactory.CreateServer();
///
/// The capture time and size of proxies read lately, by path, while the file keeps its length and write time: the
/// server's manifests and tier records check every proxy of a sequence, so an unchanged one costs a stat.
///
private static readonly ConcurrentDictionary Headers = new(StringComparer.Ordinal);
private readonly ChronosDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
private readonly FrameImageRenderer _renderer;
private readonly FrameTiers _tiers;
internal ArchiveImages(ChronosDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
_renderer = new FrameImageRenderer(writer.Options.Decoder);
_tiers = new FrameTiers(db, writer, indexer);
}
///
/// Whether a host with takes the pixels of a original from its
/// proxy: no decoder reads the format, or only previews of a HEIF, which embeds thumbnails at best (BRIEF §2.3,
/// §8.1: the server keeps HEIF originals and relies on proxies made by a Windows client or render node). Never for a
/// JPEG (proxies are JPEGs: a host that reads them reads it) or a file no format was recognised in.
///
public static bool NeedsProxy(ImageDecoder decoder, ImageFileFormat format)
{
ArgumentNullException.ThrowIfNull(decoder);
if (format is ImageFileFormat.Unknown or ImageFileFormat.Jpeg) return false;
var tier = decoder.TierFor(format);
return tier == DecodeTier.Unavailable || (tier != DecodeTier.Native && ImageFormatInfo.Of(format).Family == ImageFamily.Heif);
}
/// with this archive engine's decoders.
public bool NeedsProxy(ImageFileFormat format) => NeedsProxy(_writer.Options.Decoder, format);
///
/// on a Chronos Server, whose decoders are managed only (BRIEF
/// §4.2): it asks clients and render nodes for the proxies of those originals.
///
public static bool ServerNeedsProxy(ImageFileFormat format) => NeedsProxy(ServerDecoders, format);
///
/// Whether the file at (a sequence's proxies/0001.jpg) is the proxy of the frame
/// captured at , of × : its Exif capture
/// time is the frame's to the microsecond (the index's precision), or the frame has none to compare with; and, for a
/// frame whose original this host takes from its proxy ( given: the archive's), it is not
/// smaller than the policy's (: one an earlier version made from an embedded preview). Proxies
/// are named by frame index, so one left by a frame renumbered since (frames inserted before it) is another photo's.
/// Renders, analyses, decode tiers and the server's manifests and downloads treat a proxy that fails this as missing,
/// and the proxy pass removes it ().
///
/// The archive's proxy policy for a frame whose original this host cannot decode (its proxy comes
/// from a client or render node); null for one this host decodes and makes the proxy of itself.
public static bool IsProxyOf(string proxy, DateTime? capturedAt, int width, int height, ProxyPolicy? policy) =>
Check(proxy, capturedAt, width, height, policy).Check is ProxyCheck.Current or ProxyCheck.Unverifiable;
///
/// A proxy of the original at for a server that cannot decode it ():
/// at (0 = full size) and , with 's capture
/// data (the server's record, which it checks) and in its Exif. Null when
/// cannot decode the original either.
///
/// The original cannot be read.
public static byte[]? MakeProxy(ImageDecoder decoder, string path, Frame frame, SidecarCamera? camera, int longEdge, int quality,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(frame);
if (NeedsProxy(decoder, frame.Format)) return null;
// The original's folder stands in for a sequence folder: the renderer reads frames relative to one.
var folder = new ArchivePaths(Path.GetDirectoryName(Path.GetFullPath(path))!).Sequence(string.Empty);
var source = new SidecarFrame
{
Id = frame.Id,
I = frame.Index,
File = Path.GetFileName(path),
Size = frame.Size,
CapturedAt = frame.CapturedAt,
UtcOffset = frame.UtcOffsetMinutes,
Exposure = frame.ExposureTime,
FNumber = frame.FNumber,
Iso = frame.Iso,
Focal = frame.FocalLength,
Bias = frame.ExposureBias,
Orientation = frame.Orientation,
Format = frame.Format,
};
var policy = new ProxyPolicy { LongEdge = longEdge, Quality = quality };
return new FrameImageRenderer(decoder).Proxy(folder, source, camera, policy, cancellationToken)?.Jpeg;
}
///
/// Files a proxy made elsewhere (a client or render node whose decoders read the original) as the frame's proxy,
/// replacing the one there: a JPEG of at most that decodes, whose Exif capture time is the
/// frame's (proxies carry it, so the proxy counts as current and cannot belong to another photo) and whose size is
/// the policy's (), written atomically under the sequence's lease into its folder as the index has
/// it then: a Reorganize moves the folder under that lease, and the folder it left is never created again (one another
/// index moved, which this index finds at its next scan, takes nothing, nor does that folder when it holds another
/// sequence now: ).
///
/// The archive is offline or read-only.
/// Another writer held the sequence for longer than the lease wait.
public async Task StoreProxyAsync(Archive archive, Sequence sequence, Frame frame, byte[] jpeg, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(jpeg);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
if (policy.LongEdge == ProxyPolicy.Disabled) return ProxyStoreResult.Disabled;
if (jpeg.Length is 0 or > (int)MaxProxyBytes || !Decodes(jpeg)) return ProxyStoreResult.Invalid;
var info = JpegDecoder.ReadInfo(jpeg);
if (TooSmall(System.Math.Max(info.Width, info.Height), frame.Width, frame.Height, policy)) return ProxyStoreResult.TooSmall;
if (frame.CapturedAt is { } captured)
{
using var stream = new MemoryStream(jpeg, writable: false);
if (!SameCapture(MetadataReader.Read(stream, "proxy.jpg").CapturedAt, captured)) return ProxyStoreResult.WrongFrame;
}
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
if (await _db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not { } row) return ProxyStoreResult.Gone;
var folder = paths.Sequence(row.Folder);
if (!SequenceFolders.Holds(folder, row.Id)) return ProxyStoreResult.Gone;
Directory.CreateDirectory(folder.Proxies);
var path = folder.Proxy(frame.Index);
AtomicFile.WriteAllBytes(path, jpeg);
Headers.TryRemove(path, out _);
}
return ProxyStoreResult.Stored;
}
/// Resume point inside the sequence (frames before it were done by an earlier run: the
/// sequence's built status counts them as they are now).
/// Called with the next frame index whenever the pack has been committed.
///
/// Several frames are drawn at once (); the pack takes them in frame order
/// (), so a checkpoint still means that every frame before it is in the pack.
///
public async Task BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame,
Func checkpoint, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var archiveManifest = ManifestStore.ReadArchive(paths.ManifestPath);
var policy = archiveManifest.ThumbnailPolicy;
SequenceManifest manifest;
int built = 0, reused = 0, undecodable = 0;
var complete = true;
var tiers = new Dictionary();
// Image passes exclude each other only (a proxy pass holds this lease a batch at a time); ingest keeps committing
// meanwhile. A Reorganize moves the folder under this lease too, so the folder is the one the index has once it is
// held: the pack never opens in a folder the sequence left, which opening it would create again.
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
{
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
manifest = ManifestStore.ReadSequence(folder.ManifestPath);
// Another sequence's folder now (another index moved this one away and that one in): its pack is never opened.
if (manifest.Id != sequence.Id) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
var companions = await GprCompanions.FindAsync(_db, archive, manifest.Frames, cancellationToken).ConfigureAwait(false);
using var pack = ThumbPackWriter.Open(folder.ThumbsPackPath);
for (var i = 0; i < startFrame && i < manifest.Frames.Count; i++)
{
var frame = manifest.Frames[i];
complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
}
var cadence = new Cadence();
await using (var work = new OrderedWork(ThumbnailParallelism(manifest.Frames, policy), drawn =>
{
if (drawn.Image is { } image)
{
pack.Append(drawn.Frame.I, image.Width, image.Height, drawn.Fingerprint, image.Jpeg);
built++;
}
else
{
undecodable++;
}
if (drawn.Tier is { } tier) tiers[drawn.Frame.Id] = tier;
if (!cadence.Due()) return ValueTask.CompletedTask;
pack.Commit();
return checkpoint(drawn.Index + 1, manifest.Frames.Count);
}, cancellationToken))
{
for (var i = startFrame; i < manifest.Frames.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var frame = manifest.Frames[i];
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
var entry = pack.Entries.GetValueOrDefault(frame.I);
var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height);
var current = !force && entry is not null && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
// A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded
// preview, whose smaller thumbnail the proxy's replaces.
var fromProxy = false;
if (NeedsProxy(frame.Format) && !(current && edge >= policy.LongEdge)
&& Check(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, archiveManifest.ProxyPolicy) is { Check: ProxyCheck.Current or ProxyCheck.Unverifiable } proxy)
fromProxy = !current || edge < System.Math.Min(policy.LongEdge, proxy.LongEdge);
if (current && !fromProxy)
{
reused++;
continue;
}
var companion = companions.GetValueOrDefault(frame.Id);
var index = i;
// A thumbnail drawn from the proxy says nothing about how this host decodes the original.
await work.AddAsync(fromProxy
? ct => Drawn.Of(index, frame, fingerprint, Render(folder, frame, null, recordTier: false,
() => _renderer.Thumbnail(folder, ProxySource(frame), policy, ct)))
: ct => Drawn.Of(index, frame, fingerprint, Render(folder, frame, companion, recordTier: true,
() => _renderer.Thumbnail(folder, frame, policy, ct, companion)))).ConfigureAwait(false);
}
await work.CompleteAsync().ConfigureAwait(false);
}
pack.Commit();
}
await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, manifest, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken)
.ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable);
}
///
///
/// is set only when every frame has its proxy: a server asks clients and
/// render nodes for none while it is set (BRIEF §2.3), so a pass resumed at checks the
/// proxies of the frames an earlier run did again (their headers only). The flag
/// of frames this host reads through their proxies follows whether they have one (). Several
/// originals are decoded at once (), and a checkpoint still means that every frame before
/// it has its proxy written (). The pass works in batches (), each
/// under the thumbnail lease with the sequence's folder and sidecar read again; a folder that is not where the index
/// has it (another index moved it, until this index's next scan), or that holds another sequence now, stops the pass
/// with an , nothing written there.
///
public async Task BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame,
Func checkpoint, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0);
SequenceManifest? manifest = null;
int built = 0, reused = 0, undecodable = 0, waiting = 0;
var complete = true;
var tiers = new Dictionary();
var cadence = new Cadence();
// Under the thumbnail lease, as the thumbnail pass: image passes exclude each other only, ingest keeps committing
// meanwhile, and a Reorganize waits for the batch, so the proxies go to the folder the index has once the lease is
// held, never into one the sequence left, which writing them would create again. Between two batches the lease is let
// go for longer than a waiting writer sleeps between its tries, so another process's Reorganize, sequence deletion or
// pack compaction waits for one batch, not for the whole pass.
for (var next = 0; ;)
{
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
{
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
if (!SequenceFolders.Holds(folder, sequence.Id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
var read = ManifestStore.ReadSequence(folder.ManifestPath);
// Photos added, removed or renumbered since the last batch: what the earlier batches made may not be theirs.
if (manifest is not null && !SameFrames(read.Frames, manifest.Frames, names: false)) complete = false;
manifest = read;
Directory.CreateDirectory(folder.Proxies);
var companions = await GprCompanions.FindAsync(_db, archive, read.Frames, cancellationToken).ConfigureAwait(false);
var batch = Stopwatch.StartNew();
await using (var work = new OrderedWork(ProxyParallelism(read.Frames, policy), made =>
{
switch (made.Outcome)
{
case ProxyOutcome.Built:
built++;
break;
case ProxyOutcome.Reused:
reused++;
break;
case ProxyOutcome.Waiting:
waiting++;
break;
case ProxyOutcome.Failed:
undecodable++;
complete = false;
break;
default:
undecodable++;
break;
}
if (made.Tier is { } tier) tiers[made.FrameId] = tier;
return cadence.Due() ? checkpoint(made.Index + 1, read.Frames.Count) : ValueTask.CompletedTask;
}, cancellationToken))
{
// At least one frame per batch, so the pass always moves on.
for (var first = next; next < read.Frames.Count && (next == first || batch.Elapsed < ProxyBatch); next++)
{
cancellationToken.ThrowIfCancellationRequested();
var frame = read.Frames[next];
var path = folder.Proxy(frame.I);
var done = next < startFrame;
if (NeedsProxy(frame.Format))
{
// Only a client or render node that decodes the original makes this proxy: a preview would make one the
// size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again, and so
// does one smaller than the policy's (made here from a preview by an earlier version).
var check = Check(path, frame.CapturedAt, frame.W, frame.H, policy).Check;
var usable = check is ProxyCheck.Current or ProxyCheck.Unverifiable;
if (check == ProxyCheck.Stale) Delete(path);
complete &= usable;
if (done) continue;
var outcome = usable ? ProxyOutcome.Reused
: (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0 ? ProxyOutcome.Unreadable
: ProxyOutcome.Waiting;
await work.AddAsync(new Made(next, frame.Id, outcome, null)).ConfigureAwait(false);
continue;
}
if (done)
{
// Made by the earlier run, unless the frame did not decode then.
complete &= ReadProxy(path, frame.CapturedAt).Check is ProxyCheck.Current or ProxyCheck.Unverifiable;
continue;
}
if (!force && ReadProxy(path, frame.CapturedAt).Check == ProxyCheck.Current)
{
reused++;
continue;
}
var index = next;
var companion = companions.GetValueOrDefault(frame.Id);
await work.AddAsync(ct =>
{
var (image, tier) = Render(folder, frame, companion, recordTier: true,
() => _renderer.Proxy(folder, frame, read.Camera, policy, ct, companion));
if (image is null) return new Made(index, frame.Id, ProxyOutcome.Failed, tier);
AtomicFile.WriteAllBytes(path, image.Jpeg);
Headers.TryRemove(path, out _);
return new Made(index, frame.Id, ProxyOutcome.Built, tier);
}).ConfigureAwait(false);
}
await work.CompleteAsync().ConfigureAwait(false);
}
if (next >= read.Frames.Count) break;
}
await Task.Delay(BatchGap, cancellationToken).ConfigureAwait(false);
}
await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, manifest, JobKind.Proxies, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable, waiting);
}
///
/// How long a proxy pass holds the thumbnail lease for one batch of frames: a quarter of the lease wait, so a writer of
/// another process waiting for the lease meanwhile gets it well before it gives up (the frames the batch started finish
/// first).
///
private TimeSpan ProxyBatch => _writer.Options.LeaseWait / 4;
/// Photos a thumbnail pass draws at once (): previews of the thumbnail's size serve it.
private int ThumbnailParallelism(IReadOnlyList frames, ThumbnailPolicy policy) =>
PassParallelism(_writer.Options.Parallelism(), _writer.Options.MemoryBudget(), frames, policy.LongEdge, preferPreview: true, _writer.Options.Decoder);
/// Originals a proxy pass decodes at once (): it develops RAWs, and decodes whole for a full-size policy.
private int ProxyParallelism(IReadOnlyList frames, ProxyPolicy policy) =>
PassParallelism(_writer.Options.Parallelism(), _writer.Options.MemoryBudget(), frames, System.Math.Max(policy.LongEdge, 0), preferPreview: false,
_writer.Options.Decoder);
///
/// Photos a pass decoding at (0: full size) works on at once: the whole parallelism while no
/// frame is developed whole (a JPEG at least twice the edge decodes scaled down; a frame this host reads through its
/// proxy or preview develops nothing here, nor one whose embedded preview reaches the edge when previews serve), else
/// for the largest original developed (a TIFF, a PNG, a RAW).
///
internal static int PassParallelism(int parallelism, long memoryBudget, IReadOnlyList frames, int edge, bool preferPreview, ImageDecoder decoder)
{
parallelism = Math.Max(1, parallelism);
long largest = 0;
foreach (var frame in frames)
{
if (NeedsProxy(decoder, frame.Format)) continue;
largest = System.Math.Max(largest, DecodeBudget.Developed(frame.Format, frame.W, frame.H, System.Math.Max(frame.PreviewW ?? 0, frame.PreviewH ?? 0),
edge, preferPreview));
}
return largest == 0 ? parallelism : DecodeBudget.PhotosAtOnce(parallelism, memoryBudget, largest);
}
/// A thumbnail drawn for the frame at of the pass (null: nothing decoded it) and what decoding found.
private sealed record Drawn(int Index, SidecarFrame Frame, byte[] Fingerprint, EncodedImage? Image, FileTier? Tier)
{
public static Drawn Of(int index, SidecarFrame frame, byte[] fingerprint, (EncodedImage? Image, FileTier? Tier) rendered) =>
new(index, frame, fingerprint, rendered.Image, rendered.Tier);
}
/// What became of the proxy of a frame of a proxy pass.
private enum ProxyOutcome
{
Built,
/// The frame this host cannot decode has its proxy from a client or render node.
Reused,
/// The frame this host cannot decode waits for its proxy from a client or render node.
Waiting,
/// The frame this host cannot decode is missing or damaged.
Unreadable,
/// No decoder read the original.
Failed,
}
/// The proxy outcome of the frame at of the pass, and what decoding found.
private sealed record Made(int Index, Guid FrameId, ProxyOutcome Outcome, FileTier? Tier);
/// Where a frame's proxy stands; only a capture time tells whether it is the frame's.
private enum ProxyCheck
{
Missing,
/// Its Exif capture time is the frame's.
Current,
/// The frame has no capture time to compare with.
Unverifiable,
/// Made from another frame (frames were renumbered since), or too small ().
Stale,
}
/// A proxy's Exif capture time and oriented long edge, read when the file had this length and write time.
private readonly record struct ProxyHeader(long Length, long WriteTicks, DateTime? CapturedAt, int LongEdge);
///
/// Where the proxy at stands for the frame captured at , of
/// × , and its long edge: also when
/// is given and the proxy is for it ().
///
private static (ProxyCheck Check, int LongEdge) Check(string proxy, DateTime? capturedAt, int width, int height, ProxyPolicy? policy)
{
var (check, longEdge) = ReadProxy(proxy, capturedAt);
if (check is ProxyCheck.Current or ProxyCheck.Unverifiable && policy is not null && TooSmall(longEdge, width, height, policy)) check = ProxyCheck.Stale;
return (check, longEdge);
}
/// Where the proxy at stands for the frame captured at , and its long edge.
private static (ProxyCheck Check, int LongEdge) ReadProxy(string proxy, DateTime? capturedAt)
{
if (ReadHeader(proxy) is not { } header) return (ProxyCheck.Missing, 0);
var check = capturedAt is not { } captured ? ProxyCheck.Unverifiable : SameCapture(header.CapturedAt, captured) ? ProxyCheck.Current : ProxyCheck.Stale;
return (check, header.LongEdge);
}
///
/// The header of the proxy at ; null when there is none or it cannot be read. Kept in
/// while the file's length and write time stay as they were when it was read (taken first, so
/// a file replaced meanwhile is read again next time), once the file is older than ; a full
/// cache starts over.
///
private static ProxyHeader? ReadHeader(string proxy)
{
try
{
var file = new FileInfo(proxy);
if (!file.Exists) return null;
var (length, written) = (file.Length, file.LastWriteTimeUtc);
if (Headers.TryGetValue(proxy, out var cached) && cached.Length == length && cached.WriteTicks == written.Ticks) return cached;
var metadata = MetadataReader.Read(proxy);
var (width, height) = metadata.OrientedSize;
var header = new ProxyHeader(length, written.Ticks, metadata.CapturedAt, System.Math.Max(width, height));
if (DateTime.UtcNow - written < Settling)
{
Headers.TryRemove(proxy, out _);
return header;
}
if (Headers.Count >= MaxHeaders) Headers.Clear();
Headers[proxy] = header;
return header;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
return null;
}
}
///
/// Whether a proxy's Exif capture time is the frame's to the microsecond: the index keeps
/// capture times to the microsecond, sidecars and Exif to the 100 ns tick, so a proxy made from either record of the
/// frame (here from the sidecar, by a client from the server's index) matches both.
///
private static bool SameCapture(DateTime? proxy, DateTime captured) =>
proxy is { } time && time.Ticks / TimeSpan.TicksPerMicrosecond == captured.Ticks / TimeSpan.TicksPerMicrosecond;
/// Removes the proxy at and what is known of it.
private static void Delete(string path)
{
File.Delete(path);
Headers.TryRemove(path, out _);
}
///
/// Whether a proxy whose long edge is falls short of the policy's for a frame of
/// × (the policy's long edge, or the frame's own when smaller or
/// at full size): one made from an embedded preview. A frame of unknown size is never judged (a small original's
/// proxy would be refused again and again); a tenth of slack spares decoders that crop a few pixels.
///
private static bool TooSmall(int longEdge, int width, int height, ProxyPolicy policy)
{
var own = System.Math.Max(width, height);
if (own <= 0) return false;
var expected = policy.LongEdge == ProxyPolicy.FullSize ? own : System.Math.Min(own, policy.LongEdge);
return longEdge * 10L < expected * 9L;
}
/// The frame's proxy as the frame to draw: a JPEG already oriented, under proxies/.
private static SidecarFrame ProxySource(SidecarFrame frame) => frame with
{
File = ArchiveLayout.ProxiesFolder + "/" + ArchiveLayout.ProxyFileName(frame.I),
Format = ImageFileFormat.Jpeg,
Orientation = 1,
PreviewOffset = null,
PreviewLength = null,
};
/// A colour or grey JPEG whose headers parse and whose scans decode (at a small DCT-scaled size).
private static bool Decodes(byte[] jpeg)
{
try
{
var info = JpegDecoder.ReadInfo(jpeg);
if (info.Process == JpegProcess.Lossless || info.Components is not (1 or 3) || info.Width < 1 || info.Height < 1) return false;
using var probe = JpegDecoder.DecodeToLinear(jpeg, new JpegDecodeOptions { MaxLongEdge = ProbeLongEdge });
return true;
}
catch (Exception ex) when (ex is InvalidDataException or NotSupportedException)
{
return false;
}
}
///
/// A frame no decoder can read is skipped and counted; it never stops the pass. With ,
/// what decoding found comes back with the image: unavailable when nothing decoded a file that is there (a read error
/// proves nothing), and the probed tier when a frame flagged undecodable decoded after all. Runs on the pass's
/// workers, so it shares nothing with the others.
///
private (EncodedImage? Image, FileTier? Tier) Render(SequencePaths folder, SidecarFrame frame, string? companion, bool recordTier, Func render)
{
var path = folder.File(frame.File);
EncodedImage? image;
try
{
image = render();
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex);
return (null, null);
}
catch (Exception ex) when (ex is InvalidDataException or NotSupportedException)
{
Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex);
image = null;
}
if (!recordTier) return (image, null);
if (image is null)
return (null, File.Exists(path) ? new FileTier(frame.Format, DecodeTier.Unavailable, null, "no decoder produced pixels", 0, 0) : null);
return (image, (frame.Flags & FrameFlags.Undecodable) != 0 ? Reprobe(path, frame, companion, image.Tier) : null);
}
/// The tier of a frame that decoded after it was flagged undecodable (the image's own tier when probing finds none).
private FileTier Reprobe(string path, SidecarFrame frame, string? companion, DecodeTier decoded)
{
try
{
if (_writer.Options.Decoder.Probe(path, frame.Format, companion) is { Tier: not DecodeTier.Unavailable } tier) return tier;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("archive", $"{path} could not be probed", ex);
}
return new FileTier(frame.Format, decoded, null, null, 0, 0);
}
/// Commit/checkpoint rhythm: every rendered frames or .
private sealed class Cadence
{
private readonly Stopwatch _clock = Stopwatch.StartNew();
private int _frames;
public bool Due()
{
if (++_frames < CommitFrames && _clock.Elapsed < CommitInterval) return false;
_frames = 0;
_clock.Restart();
return true;
}
}
///
/// Sets in the sidecar and the index when the pass covered every frame it read
/// () and the sidecar, as it is under the lease, still lists those frames
/// (); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or
/// renumbered after the pass read have nothing of it, and a scan that cleared the flag for
/// them queues the next pass only while the flag stays clear. A of the same kind already
/// queued for the sequence (an upload's, or a scan's) draws them when it runs, so the flag stays set meanwhile, as
/// between an upload and its pass: a camera uploading faster than a pass ends would otherwise keep it clear, and a
/// server would offer no thumbnail or proxy of the sequence at all until uploads paused. The index is written under
/// the same lease, so the flag never lands there after a scan cleared it in both.
///
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, JobKind pass, SequenceStatus flag,
bool complete, CancellationToken cancellationToken)
{
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var built = complete && (SameFrames(manifest.Frames, read.Frames)
|| await _db.Jobs.IsQueuedAsync(pass, sequence.Id, cancellationToken).ConfigureAwait(false));
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var status = built ? manifest.Status | flag : manifest.Status & ~flag;
if (status != manifest.Status)
{
manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
}
await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken)
.ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
}
///
/// Whether lists the photos did, each at the same index with the
/// same file, size, capture time and content hash: thumbnails and proxies are kept by frame index, so photos added,
/// removed, renumbered or replaced since have none of what the pass made. A hash learnt since (a verify hashing a
/// photo the scan adopted) changes nothing about the photo, and its thumbnail stays its own ().
///
/// Whether a file renamed since counts as a change (a pass's own batches: it does not, a proxy is
/// kept by frame index and a Reorganize renames files).
private static bool SameFrames(IReadOnlyList current, IReadOnlyList read, bool names = true)
{
if (current.Count != read.Count) return false;
for (var i = 0; i < current.Count; i++)
{
var (now, then) = (current[i], read[i]);
if (now.Id != then.Id || now.I != then.I || (names && now.File != then.File) || now.Size != then.Size || now.CapturedAt != then.CapturedAt
|| (then.Sha256 is not null && !string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase))) return false;
}
return true;
}
}