Merge branch 'fix/proxies-render-sync': proxies from clients and nodes, node manifests, sync and mass-rename fixes, server polling
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
45 files changed
+1833
-179
No files matched your search
@@ -17,7 +17,8 @@ public static class ArchiveFailures
|
||||
ArchiveOfflineException offline => Loc.L(offline.Availability == ArchiveAvailability.ReadOnly ? "Archive.Error.ReadOnly" : "Archive.Error.Offline",
|
||||
Formatting.LeftToRight(offline.Root)),
|
||||
// Already one localized sentence each.
|
||||
LeaseUnavailableException or NewerFormatException or StaleRenamePlanException or StaleDeletePlanException or SequenceInLibraryException => ex.Message,
|
||||
LeaseUnavailableException or NewerFormatException or StaleRenamePlanException or NestedRenameException or StaleDeletePlanException
|
||||
or SequenceInLibraryException => ex.Message,
|
||||
UnauthorizedAccessException => Loc.L("Archive.Failed.Access"),
|
||||
DatabaseException => Loc.L("Archive.Failed.Index"),
|
||||
ArgumentException => Loc.L("Archive.Failed.Gone"),
|
||||
|
||||
@@ -41,8 +41,8 @@ public static class MassRename
|
||||
: new RenameRow(s.SequenceId, s.FromTitle ?? string.Empty, s.ToTitle ?? string.Empty)),
|
||||
];
|
||||
|
||||
/// <summary>"12 of 36 sequences change".</summary>
|
||||
public static string Summary(RenamePlan plan, int setSize) => Loc.Plural("Archive.Rename.Summary", plan.Steps.Count, setSize);
|
||||
/// <summary>"12 of 36 sequences change"; sequences that only move with a folder they are in are listed, not counted.</summary>
|
||||
public static string Summary(RenamePlan plan, int setSize) => Loc.Plural("Archive.Rename.Summary", plan.Steps.Count(s => !s.Carried), setSize);
|
||||
}
|
||||
|
||||
/// <summary>Adding and removing tags on a set of sequences (BRIEF §12.5 Retag).</summary>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Naming;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
@@ -9,11 +10,20 @@ namespace Titano.Core.Archiving;
|
||||
public sealed record RenameSubject(Guid SequenceId, string Folder, string? Title, DateTime? FirstCapture, string? CameraMake, string? CameraModel, string? Lens);
|
||||
|
||||
/// <summary>One sequence's change: its folder, its title, or both.</summary>
|
||||
public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle)
|
||||
/// <param name="Carried">The folder moves only because a folder it is inside moves (a sequence nested in another).</param>
|
||||
public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle, bool Carried = false)
|
||||
{
|
||||
public bool MovesFolder => !string.Equals(FromFolder, ToFolder, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>A planned folder would lie inside a sequence's folder (nesting one sequence in another); nothing is planned.</summary>
|
||||
public sealed class NestedRenameException(string target, string folder)
|
||||
: ArgumentException(Loc.L("Archive.Error.NestedTarget", target, folder))
|
||||
{
|
||||
public string Target { get; } = target;
|
||||
public string Folder { get; } = folder;
|
||||
}
|
||||
|
||||
/// <summary>The exact changes a mass rename will make (its dry run); applying it makes exactly these.</summary>
|
||||
public sealed record RenamePlan(Guid Id, Guid ArchiveId, IReadOnlyList<RenameStep> Steps);
|
||||
|
||||
@@ -50,16 +60,21 @@ public static class RenamePlanner
|
||||
/// Expands <paramref name="titleTemplate"/> (where <c>{title}</c> is the current title) and then
|
||||
/// <paramref name="folderTemplate"/> (where <c>{title}</c> is the new title) for every subject. A target folder
|
||||
/// that is any current sequence folder, exists on disk or was given to an earlier subject gets " (2)", " (3)"….
|
||||
/// Null templates leave that part unchanged; subjects without a change produce no step.
|
||||
/// Null templates leave that part unchanged; subjects without a change produce no step. A folder that moves takes
|
||||
/// the sequences inside it along: each gets a <see cref="RenameStep.Carried"/> step after every move, so the index
|
||||
/// follows its files; a selected sequence inside a moving folder that moves elsewhere leaves first.
|
||||
/// </summary>
|
||||
/// <param name="occupied">Every sequence folder of the archive (renamed or not).</param>
|
||||
/// <param name="nested">The archive's other sequences inside a subject's folder, with their titles.</param>
|
||||
/// <exception cref="NestedRenameException">A target lies inside a sequence's folder, or inside another target.</exception>
|
||||
public static RenamePlan Plan(Guid archiveId, IReadOnlyList<RenameSubject> subjects, string? folderTemplate, string? titleTemplate,
|
||||
IEnumerable<string> occupied, Func<string, bool> existsOnDisk)
|
||||
IEnumerable<string> occupied, Func<string, bool> existsOnDisk, IReadOnlyList<RenameSubject>? nested = null)
|
||||
{
|
||||
var folder = folderTemplate is null ? null : PathTemplate.Parse(folderTemplate);
|
||||
var title = titleTemplate is null ? null : PathTemplate.Parse(titleTemplate);
|
||||
var claimed = new HashSet<string>(occupied, StringComparer.OrdinalIgnoreCase);
|
||||
var steps = new List<RenameStep>();
|
||||
var taken = occupied.ToList();
|
||||
var claimed = new HashSet<string>(taken, StringComparer.OrdinalIgnoreCase);
|
||||
var own = new List<RenameStep>(subjects.Count);
|
||||
foreach (var s in subjects)
|
||||
{
|
||||
var context = new NamingContext
|
||||
@@ -80,11 +95,39 @@ public static class RenamePlanner
|
||||
newFolder = expanded.Equals(s.Folder, StringComparison.OrdinalIgnoreCase) ? expanded : Unique(expanded, claimed, existsOnDisk);
|
||||
claimed.Add(newFolder);
|
||||
}
|
||||
if (newFolder != s.Folder || newTitle != s.Title) steps.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle));
|
||||
own.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle));
|
||||
}
|
||||
return new RenamePlan(Guid.CreateVersion7(), archiveId, steps);
|
||||
|
||||
var moving = own.Where(s => s.MovesFolder).ToList();
|
||||
foreach (var step in moving)
|
||||
{
|
||||
// Into a sequence's folder (its own included) or another target: one sequence would end up inside another.
|
||||
var host = taken.Concat(moving.Where(m => m != step).Select(m => m.ToFolder)).FirstOrDefault(f => IsInside(f, step.ToFolder));
|
||||
if (host is not null) throw new NestedRenameException(step.ToFolder, host);
|
||||
}
|
||||
var steps = new List<(RenameStep Step, int Order)>();
|
||||
foreach (var step in own.Concat((nested ?? []).Select(n => new RenameStep(n.SequenceId, n.Folder, n.Folder, n.Title, n.Title))))
|
||||
{
|
||||
// The nearest folder around it that moves carries it; a sequence that moves elsewhere leaves before its folder does.
|
||||
var carrier = step.MovesFolder ? null : moving.Where(m => IsInside(m.FromFolder, step.FromFolder)).MaxBy(m => m.FromFolder.Length);
|
||||
if (carrier is not null)
|
||||
steps.Add((step with { ToFolder = carrier.ToFolder + step.FromFolder[carrier.FromFolder.Length..], Carried = true }, 1));
|
||||
else if (step.MovesFolder)
|
||||
steps.Add((step, -moving.Count(m => m != step && IsInside(m.FromFolder, step.FromFolder))));
|
||||
else if (step.FromTitle != step.ToTitle)
|
||||
steps.Add((step, 0));
|
||||
}
|
||||
// Stable: subjects keep their order, sequences inside moving folders move out first, carried ones come last.
|
||||
return new RenamePlan(Guid.CreateVersion7(), archiveId, [.. steps.OrderBy(s => s.Order).Select(s => s.Step)]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether <paramref name="path"/> lies inside <paramref name="folder"/> (archive-relative, forward slashes); the
|
||||
/// archive root, a sequence only when a folder was adopted in place, counts as containing nothing.
|
||||
/// </summary>
|
||||
public static bool IsInside(string folder, string path) =>
|
||||
folder.Length > 0 && path.Length > folder.Length + 1 && path[folder.Length] == '/' && path.StartsWith(folder, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Moves the folder of a step; idempotent so a resumed plan can repeat it: an already moved folder is accepted,
|
||||
/// a case-only change goes through a temporary name (case-insensitive file systems).
|
||||
|
||||
@@ -279,7 +279,7 @@ public sealed class ThumbPackReader : IDisposable
|
||||
public void Dispose() => _stream?.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>Appends thumbnails to a pack; one writer per pack at a time (the sequence lease guarantees it).</summary>
|
||||
/// <summary>Appends thumbnails to a pack; one writer per pack at a time (the <c>thumbs-<sequence id></c> lease guarantees it).</summary>
|
||||
public sealed class ThumbPackWriter : IDisposable
|
||||
{
|
||||
private readonly FileStream _stream;
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
using System.Diagnostics;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
@@ -9,15 +12,40 @@ namespace Titano.Data.Archiving;
|
||||
/// <summary>Frames processed by one thumbnail or proxy pass over a sequence.</summary>
|
||||
public sealed record ImagePassResult(int Built, int Reused, int Undecodable);
|
||||
|
||||
/// <summary>What became of a proxy sent by a client or render node (<see cref="ArchiveImages.StoreProxy"/>).</summary>
|
||||
public enum ProxyStoreResult
|
||||
{
|
||||
Stored,
|
||||
|
||||
/// <summary>The archive's proxy policy makes no proxies.</summary>
|
||||
Disabled,
|
||||
|
||||
/// <summary>Not a JPEG that decodes, or larger than <see cref="ArchiveImages.MaxProxyBytes"/>.</summary>
|
||||
Invalid,
|
||||
|
||||
/// <summary>Its Exif capture time is not the frame's: it was made from another photo.</summary>
|
||||
WrongFrame,
|
||||
|
||||
/// <summary>Smaller than the archive's proxies of that frame: made from an embedded preview, not the original.</summary>
|
||||
TooSmall,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thumbnails (<c>thumbs.pack</c>) and proxies (<c>proxies/0001.jpg</c>) of sequences (BRIEF §8.5). Both are
|
||||
/// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after
|
||||
/// frames were renumbered the stale ones are rebuilt and the rest reused.
|
||||
/// frames were renumbered the stale ones are rebuilt and the rest reused. A frame whose original this host cannot
|
||||
/// decode (<see cref="NeedsProxy(ImageDecoder, ImageFileFormat)"/>) gets its proxy from a client or render node that can
|
||||
/// (<see cref="StoreProxy"/>, BRIEF §2.3) and its thumbnail from that proxy.
|
||||
/// </summary>
|
||||
public sealed class ArchiveImages
|
||||
{
|
||||
/// <summary>Largest proxy a client or render node may send: a full-size proxy of a large sensor at q92 fits.</summary>
|
||||
public const long MaxProxyBytes = 64L << 20;
|
||||
|
||||
private const int CommitFrames = 200;
|
||||
private const int ProbeLongEdge = 64;
|
||||
private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10);
|
||||
private static readonly ImageDecoder ServerDecoders = DecoderFactory.CreateServer();
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
@@ -32,6 +60,91 @@ public sealed class ArchiveImages
|
||||
_renderer = new FrameImageRenderer(writer.Options.Decoder);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a host with <paramref name="decoder"/> takes the pixels of a <paramref name="format"/> original from its
|
||||
/// proxy: no decoder reads the format, or only previews of a HEIF, which embeds thumbnails at best (BRIEF §2.3,
|
||||
/// §8.1: the server keeps HEIF originals and relies on proxies made by a Windows client or render node). Never for a
|
||||
/// JPEG (proxies are JPEGs: a host that reads them reads it) or a file no format was recognised in.
|
||||
/// </summary>
|
||||
public static bool NeedsProxy(ImageDecoder decoder, ImageFileFormat format)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(decoder);
|
||||
if (format is ImageFileFormat.Unknown or ImageFileFormat.Jpeg) return false;
|
||||
var tier = decoder.TierFor(format);
|
||||
return tier == DecodeTier.Unavailable || (tier != DecodeTier.Native && ImageFormatInfo.Of(format).Family == ImageFamily.Heif);
|
||||
}
|
||||
|
||||
/// <summary><see cref="NeedsProxy(ImageDecoder, ImageFileFormat)"/> with this archive engine's decoders.</summary>
|
||||
public bool NeedsProxy(ImageFileFormat format) => NeedsProxy(_writer.Options.Decoder, format);
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="NeedsProxy(ImageDecoder, ImageFileFormat)"/> on a Titano Server, whose decoders are managed only (BRIEF
|
||||
/// §4.2): it asks clients and render nodes for the proxies of those originals.
|
||||
/// </summary>
|
||||
public static bool ServerNeedsProxy(ImageFileFormat format) => NeedsProxy(ServerDecoders, format);
|
||||
|
||||
/// <summary>
|
||||
/// A proxy of the original at <paramref name="path"/> for a server that cannot decode it (<see cref="StoreProxy"/>):
|
||||
/// at <paramref name="longEdge"/> (0 = full size) and <paramref name="quality"/>, with <paramref name="frame"/>'s capture
|
||||
/// data (the server's record, which it checks) and <paramref name="camera"/> in its Exif. Null when
|
||||
/// <paramref name="decoder"/> cannot decode the original either.
|
||||
/// </summary>
|
||||
/// <exception cref="IOException">The original cannot be read.</exception>
|
||||
public static byte[]? MakeProxy(ImageDecoder decoder, string path, Frame frame, SidecarCamera? camera, int longEdge, int quality,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(frame);
|
||||
if (NeedsProxy(decoder, frame.Format)) return null;
|
||||
// The original's folder stands in for a sequence folder: the renderer reads frames relative to one.
|
||||
var folder = new ArchivePaths(Path.GetDirectoryName(Path.GetFullPath(path))!).Sequence(string.Empty);
|
||||
var source = new SidecarFrame
|
||||
{
|
||||
Id = frame.Id,
|
||||
I = frame.Index,
|
||||
File = Path.GetFileName(path),
|
||||
Size = frame.Size,
|
||||
CapturedAt = frame.CapturedAt,
|
||||
UtcOffset = frame.UtcOffsetMinutes,
|
||||
Exposure = frame.ExposureTime,
|
||||
FNumber = frame.FNumber,
|
||||
Iso = frame.Iso,
|
||||
Focal = frame.FocalLength,
|
||||
Bias = frame.ExposureBias,
|
||||
Orientation = frame.Orientation,
|
||||
Format = frame.Format,
|
||||
};
|
||||
var policy = new ProxyPolicy { LongEdge = longEdge, Quality = quality };
|
||||
return new FrameImageRenderer(decoder).Proxy(folder, source, camera, policy, cancellationToken)?.Jpeg;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Files a proxy made elsewhere (a client or render node whose decoders read the original) as the frame's proxy,
|
||||
/// replacing the one there: a JPEG of at most <see cref="MaxProxyBytes"/> that decodes, whose Exif capture time is the
|
||||
/// frame's (proxies carry it, so the proxy counts as current and cannot belong to another photo) and whose size is
|
||||
/// the policy's (<see cref="TooSmall"/>), written atomically.
|
||||
/// </summary>
|
||||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
|
||||
public ProxyStoreResult StoreProxy(Archive archive, Sequence sequence, Frame frame, byte[] jpeg)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(jpeg);
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
|
||||
if (policy.LongEdge == ProxyPolicy.Disabled) return ProxyStoreResult.Disabled;
|
||||
if (jpeg.Length is 0 or > (int)MaxProxyBytes || !Decodes(jpeg)) return ProxyStoreResult.Invalid;
|
||||
var info = JpegDecoder.ReadInfo(jpeg);
|
||||
if (TooSmall(System.Math.Max(info.Width, info.Height), frame.Width, frame.Height, policy)) return ProxyStoreResult.TooSmall;
|
||||
if (frame.CapturedAt is { } captured)
|
||||
{
|
||||
using var stream = new MemoryStream(jpeg, writable: false);
|
||||
if (MetadataReader.Read(stream, "proxy.jpg").CapturedAt != captured) return ProxyStoreResult.WrongFrame;
|
||||
}
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
AtomicFile.WriteAllBytes(folder.Proxy(frame.Index), jpeg);
|
||||
return ProxyStoreResult.Stored;
|
||||
}
|
||||
|
||||
/// <param name="startFrame">Resume point inside the sequence (frames before it were done by an earlier run).</param>
|
||||
/// <param name="checkpoint">Called with the next frame index whenever the pack has been committed.</param>
|
||||
public async Task<ImagePassResult> BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame,
|
||||
@@ -54,12 +167,22 @@ public sealed class ArchiveImages
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var frame = manifest.Frames[i];
|
||||
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
|
||||
if (!force && pack.Entries.TryGetValue(frame.I, out var entry) && entry.SourceHash.AsSpan().SequenceEqual(fingerprint))
|
||||
var entry = pack.Entries.GetValueOrDefault(frame.I);
|
||||
var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height);
|
||||
var current = !force && entry is not null && entry.SourceHash.AsSpan().SequenceEqual(fingerprint);
|
||||
// A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded
|
||||
// preview, whose smaller thumbnail the proxy's replaces.
|
||||
var fromProxy = false;
|
||||
if (NeedsProxy(frame.Format) && !(current && edge >= FrameImageRenderer.ThumbnailLongEdge)
|
||||
&& ReadProxy(folder.Proxy(frame.I), frame) is { Check: ProxyCheck.Current or ProxyCheck.Unverifiable } proxy)
|
||||
fromProxy = !current || edge < System.Math.Min(FrameImageRenderer.ThumbnailLongEdge, proxy.LongEdge);
|
||||
if (current && !fromProxy)
|
||||
{
|
||||
reused++;
|
||||
continue;
|
||||
}
|
||||
if (Render(() => _renderer.Thumbnail(folder, frame, policy, cancellationToken)) is { } image)
|
||||
var source = fromProxy ? ProxySource(frame) : frame;
|
||||
if (Render(() => _renderer.Thumbnail(folder, source, policy, cancellationToken)) is { } image)
|
||||
{
|
||||
pack.Append(frame.I, image.Width, image.Height, fingerprint, image.Jpeg);
|
||||
built++;
|
||||
@@ -98,12 +221,29 @@ public sealed class ArchiveImages
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var frame = manifest.Frames[i];
|
||||
var path = folder.Proxy(frame.I);
|
||||
if (!force && IsCurrent(path, frame))
|
||||
if (NeedsProxy(frame.Format))
|
||||
{
|
||||
// Only a client or render node that decodes the original makes this proxy: a preview would make one the
|
||||
// size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again, and so
|
||||
// does one smaller than the policy's (made here from a preview by an earlier version).
|
||||
var (check, longEdge) = ReadProxy(path, frame);
|
||||
if (check is ProxyCheck.Current or ProxyCheck.Unverifiable && TooSmall(longEdge, frame.W, frame.H, policy)) check = ProxyCheck.Stale;
|
||||
if (check is ProxyCheck.Current or ProxyCheck.Unverifiable)
|
||||
{
|
||||
reused++;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (check == ProxyCheck.Stale) File.Delete(path);
|
||||
undecodable++;
|
||||
}
|
||||
}
|
||||
else if (!force && ReadProxy(path, frame).Check == ProxyCheck.Current)
|
||||
{
|
||||
reused++;
|
||||
continue;
|
||||
}
|
||||
if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image)
|
||||
else if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image)
|
||||
{
|
||||
AtomicFile.WriteAllBytes(path, image.Jpeg);
|
||||
built++;
|
||||
@@ -118,15 +258,73 @@ public sealed class ArchiveImages
|
||||
return new ImagePassResult(built, reused, undecodable);
|
||||
}
|
||||
|
||||
/// <summary>A proxy is current when its Exif capture time is the frame's (frames renumbered since make it stale).</summary>
|
||||
private static bool IsCurrent(string proxy, SidecarFrame frame)
|
||||
/// <summary>Where a frame's proxy stands; only a capture time tells whether it is the frame's.</summary>
|
||||
private enum ProxyCheck
|
||||
{
|
||||
if (!File.Exists(proxy)) return false;
|
||||
Missing,
|
||||
|
||||
/// <summary>Its Exif capture time is the frame's.</summary>
|
||||
Current,
|
||||
|
||||
/// <summary>The frame has no capture time to compare with.</summary>
|
||||
Unverifiable,
|
||||
|
||||
/// <summary>Made from another frame (frames were renumbered since).</summary>
|
||||
Stale,
|
||||
}
|
||||
|
||||
/// <summary>Where the proxy at <paramref name="proxy"/> stands for <paramref name="frame"/>, and its long edge.</summary>
|
||||
private static (ProxyCheck Check, int LongEdge) ReadProxy(string proxy, SidecarFrame frame)
|
||||
{
|
||||
if (!File.Exists(proxy)) return (ProxyCheck.Missing, 0);
|
||||
try
|
||||
{
|
||||
return MetadataReader.Read(proxy).CapturedAt == frame.CapturedAt && frame.CapturedAt is not null;
|
||||
var metadata = MetadataReader.Read(proxy);
|
||||
var (width, height) = metadata.OrientedSize;
|
||||
var check = frame.CapturedAt is null ? ProxyCheck.Unverifiable : metadata.CapturedAt == frame.CapturedAt ? ProxyCheck.Current : ProxyCheck.Stale;
|
||||
return (check, System.Math.Max(width, height));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return (ProxyCheck.Missing, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a proxy whose long edge is <paramref name="longEdge"/> falls short of the policy's for a frame of
|
||||
/// <paramref name="width"/> × <paramref name="height"/> (the policy's long edge, or the frame's own when smaller or
|
||||
/// at full size): one made from an embedded preview. A frame of unknown size is never judged (a small original's
|
||||
/// proxy would be refused again and again); a tenth of slack spares decoders that crop a few pixels.
|
||||
/// </summary>
|
||||
private static bool TooSmall(int longEdge, int width, int height, ProxyPolicy policy)
|
||||
{
|
||||
var own = System.Math.Max(width, height);
|
||||
if (own <= 0) return false;
|
||||
var expected = policy.LongEdge == ProxyPolicy.FullSize ? own : System.Math.Min(own, policy.LongEdge);
|
||||
return longEdge * 10L < expected * 9L;
|
||||
}
|
||||
|
||||
/// <summary>The frame's proxy as the frame to draw: a JPEG already oriented, under <c>proxies/</c>.</summary>
|
||||
private static SidecarFrame ProxySource(SidecarFrame frame) => frame with
|
||||
{
|
||||
File = ArchiveLayout.ProxiesFolder + "/" + ArchiveLayout.ProxyFileName(frame.I),
|
||||
Format = ImageFileFormat.Jpeg,
|
||||
Orientation = 1,
|
||||
PreviewOffset = null,
|
||||
PreviewLength = null,
|
||||
};
|
||||
|
||||
/// <summary>A colour or grey JPEG whose headers parse and whose scans decode (at a small DCT-scaled size).</summary>
|
||||
private static bool Decodes(byte[] jpeg)
|
||||
{
|
||||
try
|
||||
{
|
||||
var info = JpegDecoder.ReadInfo(jpeg);
|
||||
if (info.Process == JpegProcess.Lossless || info.Components is not (1 or 3) || info.Width < 1 || info.Height < 1) return false;
|
||||
using var probe = JpegDecoder.DecodeToLinear(jpeg, new JpegDecodeOptions { MaxLongEdge = ProbeLongEdge });
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or NotSupportedException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -215,7 +215,11 @@ public sealed class ArchiveMaintenance
|
||||
if (length == 0 || garbage < length * CompactionRatio) continue;
|
||||
compacted.Add(location.Folder);
|
||||
if (dryRun) continue;
|
||||
// The thumbnail builder appends under the thumbs lease only: holding it too keeps the compacted pack from
|
||||
// replacing the file under a writer, whose later thumbnails would land in the orphaned one.
|
||||
await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false))
|
||||
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{location.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
|
||||
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
|
||||
ThumbPackWriter.Compact(sequence.ThumbsPackPath);
|
||||
}
|
||||
return compacted;
|
||||
|
||||
@@ -32,6 +32,8 @@ public sealed class MassRenamer
|
||||
|
||||
/// <param name="folderTemplate">New folder template (<c>{title}</c> = the new title), or null to keep folders.</param>
|
||||
/// <param name="titleTemplate">New title template (<c>{title}</c> = the current title), or null to keep titles.</param>
|
||||
/// <remarks>Sequences nested in a folder that moves are part of the plan (they move with it); see <see cref="RenamePlanner.Plan"/>.</remarks>
|
||||
/// <exception cref="NestedRenameException">A new folder would lie inside a sequence's folder.</exception>
|
||||
public async Task<RenamePlan> PlanAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, string? folderTemplate, string? titleTemplate,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -42,12 +44,23 @@ public sealed class MassRenamer
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } s || s.ArchiveId != archive.Id)
|
||||
throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(sequenceIds));
|
||||
subjects.Add(new RenameSubject(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens));
|
||||
subjects.Add(Subject(s));
|
||||
}
|
||||
var nested = new List<RenameSubject>();
|
||||
if (folderTemplate is not null)
|
||||
{
|
||||
var selected = sequenceIds.ToHashSet();
|
||||
foreach (var location in locations.Where(l => !selected.Contains(l.Id) && subjects.Any(s => RenamePlanner.IsInside(s.Folder, l.Folder))))
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is { } inner) nested.Add(Subject(inner));
|
||||
}
|
||||
}
|
||||
return RenamePlanner.Plan(archive.Id, subjects, folderTemplate, titleTemplate, locations.Select(l => l.Folder),
|
||||
folder => Directory.Exists(paths.Sequence(folder).Folder));
|
||||
folder => Directory.Exists(paths.Sequence(folder).Folder), nested);
|
||||
}
|
||||
|
||||
private static RenameSubject Subject(Sequence s) => new(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens);
|
||||
|
||||
/// <summary>Applies a plan; each completed step is journaled before the next starts.</summary>
|
||||
/// <exception cref="StaleRenamePlanException">A sequence is no longer where the plan expects it; nothing of that step was changed.</exception>
|
||||
public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
|
||||
@@ -80,55 +93,106 @@ public sealed class MassRenamer
|
||||
}
|
||||
}
|
||||
RenameJournal.Write(paths, journal);
|
||||
for (var i = 0; i < journal.Plan.Steps.Count; i++)
|
||||
var steps = journal.Plan.Steps;
|
||||
foreach (var step in steps)
|
||||
{
|
||||
var step = journal.Plan.Steps[i];
|
||||
if (completed.Contains(step.SequenceId)) continue;
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
await ApplyStepAsync(archive, paths, step, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (StaleRenamePlanException) when (resuming)
|
||||
{
|
||||
Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there");
|
||||
}
|
||||
// A folder that moves carries the sequences inside it: they are locked, moved and recorded with it.
|
||||
IReadOnlyList<RenameStep> carried = step.MovesFolder && !step.Carried
|
||||
? [.. steps.Where(c => c.Carried && !completed.Contains(c.SequenceId) && Carrier(steps, c) == step)]
|
||||
: [];
|
||||
await ApplyStepAsync(archive, paths, step, carried, resuming, cancellationToken).ConfigureAwait(false);
|
||||
completed.Add(step.SequenceId);
|
||||
foreach (var c in carried) completed.Add(c.SequenceId);
|
||||
RenameJournal.Write(paths, journal with { Completed = [.. completed] });
|
||||
progress?.Report(i + 1);
|
||||
progress?.Report(completed.Count);
|
||||
}
|
||||
RenameJournal.Delete(paths, journal.Plan.Id);
|
||||
}
|
||||
|
||||
/// <summary>Folder, then sidecar title, then index; each part is idempotent so a resumed step can repeat it.</summary>
|
||||
private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
|
||||
/// <summary>The step whose folder move carries <paramref name="carried"/>: the nearest moving folder around it (as planned).</summary>
|
||||
private static RenameStep? Carrier(IReadOnlyList<RenameStep> steps, RenameStep carried) =>
|
||||
steps.Where(m => !m.Carried && m.MovesFolder && RenamePlanner.IsInside(m.FromFolder, carried.FromFolder)).MaxBy(m => m.FromFolder.Length);
|
||||
|
||||
/// <summary>
|
||||
/// Applies a step, and the steps of the sequences its folder carries, under all their leases: sequence leases, then
|
||||
/// for a move the thumbnail leases, each by sequence id (the order trash and pack compaction take them in). No
|
||||
/// writer of a carried sequence (ingest, an editor save, a thumbnail pass) works in the folder while it moves or
|
||||
/// before its index row follows.
|
||||
/// </summary>
|
||||
/// <param name="resuming">A step whose sequence the index no longer has where the plan says is dropped instead of
|
||||
/// failing the rename: its sidecar still carries the id, so the next scan indexes it wherever it ended up.</param>
|
||||
private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, IReadOnlyList<RenameStep> carried, bool resuming,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using (await _writer.LockAsync(paths, step.SequenceId, cancellationToken).ConfigureAwait(false))
|
||||
var ids = carried.Select(c => c.SequenceId).Append(step.SequenceId).Order().ToList();
|
||||
var locks = new List<IAsyncDisposable>(ids.Count);
|
||||
var leases = new List<LeaseFile>(ids.Count);
|
||||
try
|
||||
{
|
||||
var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
if (row is null || (row.Folder != step.FromFolder && row.Folder != step.ToFolder)) throw new StaleRenamePlanException(step.SequenceId);
|
||||
RenamePlanner.MoveFolder(paths, step);
|
||||
var sequence = paths.Sequence(step.ToFolder);
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
||||
if (manifest.Title != step.ToTitle)
|
||||
foreach (var id in ids) locks.Add(await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false));
|
||||
if (step.MovesFolder)
|
||||
{
|
||||
manifest = manifest with
|
||||
foreach (var id in ids)
|
||||
{
|
||||
Title = step.ToTitle,
|
||||
Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
leases.Add(await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
|
||||
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
// Checked before anything moves; a dropped step drops the sequences it would have carried, which stay inside it.
|
||||
if (!await StandsAsync(step, resuming, cancellationToken).ConfigureAwait(false)) return;
|
||||
var follow = new List<RenameStep>(carried.Count);
|
||||
foreach (var c in carried)
|
||||
{
|
||||
var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (await StandsAsync(c, resuming, cancellationToken).ConfigureAwait(false)) follow.Add(c);
|
||||
}
|
||||
await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
RenamePlanner.MoveFolder(paths, step);
|
||||
await RecordAsync(archive, paths, step, cancellationToken).ConfigureAwait(false);
|
||||
// Moved with their carrier: only their sidecars and index rows follow.
|
||||
foreach (var c in follow) await RecordAsync(archive, paths, c, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
for (var i = leases.Count - 1; i >= 0; i--) leases[i].Dispose();
|
||||
for (var i = locks.Count - 1; i >= 0; i--) await locks[i].DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The index still has the step's sequence where the plan says (or where it already took it); otherwise stale.</summary>
|
||||
/// <exception cref="StaleRenamePlanException">Not resuming, and the sequence moved or vanished since the plan was made.</exception>
|
||||
private async Task<bool> StandsAsync(RenameStep step, bool resuming, CancellationToken cancellationToken)
|
||||
{
|
||||
var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
if (row is not null && (row.Folder == step.FromFolder || row.Folder == step.ToFolder)) return true;
|
||||
if (!resuming) throw new StaleRenamePlanException(step.SequenceId);
|
||||
Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there");
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it.</summary>
|
||||
private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = paths.Sequence(step.ToFolder);
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
||||
if (manifest.Title != step.ToTitle)
|
||||
{
|
||||
manifest = manifest with
|
||||
{
|
||||
Title = step.ToTitle,
|
||||
Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
}
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Client sync state (BRIEF §2.3, §14.5): the server's update time a linked copy's originals were compared at, kept
|
||||
-- while proxies the server asks for may be made on this PC; they change without the frame counters moving, so the
|
||||
-- copy is compared again when the server's sequence changed since. Mirrors sqlite/0006_sync_proxy_compare.sql.
|
||||
|
||||
ALTER TABLE sync_links ADD COLUMN diff_remote_updated_at timestamptz;
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Client sync state (BRIEF §2.3, §14.5): the server's update time a linked copy's originals were compared at, kept
|
||||
-- while proxies the server asks for may be made on this PC; they change without the frame counters moving, so the
|
||||
-- copy is compared again when the server's sequence changed since. Mirrors postgres/0006_sync_proxy_compare.sql.
|
||||
|
||||
ALTER TABLE sync_links ADD COLUMN diff_remote_updated_at TEXT;
|
||||
@@ -62,7 +62,7 @@ internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions opti
|
||||
var targets = all.Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).Select(t => (t.i, adapt(t.t))).ToList();
|
||||
if (targets.Count == 0) return;
|
||||
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, cancellation).ConfigureAwait(false);
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, options.Archive.Decoder, cancellation).ConfigureAwait(false);
|
||||
IReadOnlyList<TargetOutcome> outcomes;
|
||||
await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames.Count), options))
|
||||
{
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Imaging;
|
||||
@@ -8,6 +9,7 @@ using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Render;
|
||||
using Titano.Core.Video;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Protocol;
|
||||
using Titano.Protocol.Client;
|
||||
|
||||
@@ -47,8 +49,10 @@ public sealed record RenderNodeOptions
|
||||
/// <summary>
|
||||
/// "Offer this PC as render node" (BRIEF §14.7): registers this machine's single-file encoders with the server,
|
||||
/// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals
|
||||
/// hash-verified into a content-addressed cache, or proxies when the job asks; a dropped download resumes), renders the
|
||||
/// job's preset with the host's encoders and uploads the output, which the server files as a render of the sequence.
|
||||
/// hash-verified into a content-addressed cache, or proxies when the job asks or this PC cannot decode the original; a
|
||||
/// dropped download resumes; frames missing, corrupt or excluded on the server are left out), renders the job's preset
|
||||
/// with the host's encoders and uploads the output, which the server files as a render of the sequence. Then it sends
|
||||
/// the proxies the server asks for, made from the originals it fetched (BRIEF §2.3: the server cannot decode HEIF).
|
||||
/// Progress and render remarks go to the server; a job cancelled or reassigned there stops here; a failure is reported
|
||||
/// with its reason. Stopping abandons the job in flight without reporting it: the server requeues it when the node
|
||||
/// registers again or its heartbeats stop. Image sequences are not offered and batches are refused: the protocol uploads
|
||||
@@ -58,6 +62,12 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
{
|
||||
private const string LogCategory = "render-node";
|
||||
|
||||
/// <summary>
|
||||
/// Frames no node renders: absent on the server, not the archived file, or excluded. Undecodable is the server's
|
||||
/// verdict on its own decoders, so this PC's decoders try those frames.
|
||||
/// </summary>
|
||||
internal const FrameFlags NotFetched = FrameFlags.Missing | FrameFlags.Corrupt | FrameFlags.Excluded;
|
||||
|
||||
private readonly TitanoClient _client;
|
||||
private readonly ProcessingOptions _processing;
|
||||
private readonly RenderNodeOptions _options;
|
||||
@@ -127,7 +137,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(work);
|
||||
await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false);
|
||||
var input = await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false);
|
||||
// The job is done on the server; the proxies it asked for follow, on the node's own time.
|
||||
await SendProxiesAsync(claimed, input, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (job.IsCancellationRequested)
|
||||
{
|
||||
@@ -221,7 +233,8 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
|
||||
/// <summary>Renders the job and uploads its output; returns the frames it read.</summary>
|
||||
private async Task<SequenceFrames> RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
|
||||
{
|
||||
var export = ProcessingJobs.ReadExport(claimed.PresetJson);
|
||||
var resolved = ExportResolver.Resolve(export);
|
||||
@@ -232,9 +245,18 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export };
|
||||
|
||||
var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false));
|
||||
var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
var files = await DownloadAsync(claimed, work, reporter, cancellationToken).ConfigureAwait(false);
|
||||
var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id))]);
|
||||
// The server's Undecodable stands, as in its own render, except for a format only its decoders lack (HEIF) that
|
||||
// this PC's read: such a frame renders here from its original. One read from its proxy renders either way.
|
||||
var decoder = _processing.Archive.Decoder;
|
||||
var frames = (await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false))
|
||||
.ConvertAll(f => ArchiveImages.ServerNeedsProxy(f.Format) && !ArchiveImages.NeedsProxy(decoder, f.Format)
|
||||
? f with { Flags = f.Flags & ~FrameFlags.Undecodable }
|
||||
: f);
|
||||
var files = await DownloadAsync(claimed, frames, work, reporter, cancellationToken).ConfigureAwait(false);
|
||||
var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id).Path)])
|
||||
{
|
||||
Proxied = [.. frames.Select(f => files.GetValueOrDefault(f.Id).Proxy)],
|
||||
};
|
||||
|
||||
IReadOnlyList<TargetOutcome> outcomes;
|
||||
await using (var render = new SequenceRender(input, project, null, _processing))
|
||||
@@ -253,6 +275,56 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (outcomes[0].Error is { } error) throw new InvalidOperationException(error);
|
||||
return input;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends the proxies the server asks for (<see cref="ManifestFrame.WantsProxy"/>) that this PC can make from the
|
||||
/// originals it fetched, as the server's proxy policy says; a failure is logged and never fails the finished job.
|
||||
/// </summary>
|
||||
private async Task SendProxiesAsync(ClaimedJob claimed, SequenceFrames input, CancellationToken cancellationToken)
|
||||
{
|
||||
if (claimed.Proxies is not { } spec) return;
|
||||
var wanted = claimed.Frames.Where(f => f.WantsProxy).Select(f => f.Id).ToHashSet();
|
||||
if (wanted.Count == 0) return;
|
||||
var camera = new SidecarCamera { Make = input.Sequence.CameraMake, Model = input.Sequence.CameraModel };
|
||||
int sent = 0, unsent = 0;
|
||||
for (var i = 0; i < input.Count; i++)
|
||||
{
|
||||
var frame = input.Frames[i];
|
||||
if (!wanted.Remove(frame.Id)) continue;
|
||||
if (input.Proxied?[i] == true || input.Files[i] is not { } original)
|
||||
{
|
||||
unsent++;
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
var jpeg = await Task.Run(() => ArchiveImages.MakeProxy(_processing.Archive.Decoder, original, frame, camera, spec.LongEdge, spec.Quality,
|
||||
cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
if (jpeg is null)
|
||||
{
|
||||
unsent++;
|
||||
continue;
|
||||
}
|
||||
await _client.UploadProxyAsync(frame.Id, jpeg, cancellationToken).ConfigureAwait(false);
|
||||
sent++;
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status is HttpStatusCode.Conflict or HttpStatusCode.Forbidden)
|
||||
{
|
||||
// Proxies turned off on the server, or this account may no longer send them: none of the rest would go either.
|
||||
Log.Warning(LogCategory, $"The server refused the proxies of job {claimed.Job.Id}", ex);
|
||||
return;
|
||||
}
|
||||
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Best effort (a timed-out upload included): the job is done, and a proxy the server still lacks is asked for
|
||||
// again with the next job.
|
||||
Log.Warning(LogCategory, $"The proxy of {frame.FileName} could not be sent", ex);
|
||||
unsent++;
|
||||
}
|
||||
}
|
||||
Log.Info(LogCategory, $"Proxies of job {claimed.Job.Id}: {sent} sent, {unsent + wanted.Count} not");
|
||||
}
|
||||
|
||||
private async Task UploadAsync(Guid nodeId, Guid jobId, ExportTarget target, RenderResult result, CancellationToken cancellationToken)
|
||||
@@ -280,27 +352,35 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused
|
||||
/// across jobs, fetched once per content), or proxies into the job folder when the job asks for them and the server
|
||||
/// has them.
|
||||
/// The files of the job's frames (none for frames missing, corrupt or excluded on the server): originals verified
|
||||
/// against their SHA-256 into the content-addressed cache (reused across jobs, fetched once per content), originals the
|
||||
/// server has not hashed yet into the job folder, or proxies into the job folder when the job asks for them or this PC
|
||||
/// cannot decode the original, and the server has them.
|
||||
/// </summary>
|
||||
private async Task<Dictionary<Guid, string>> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
|
||||
private async Task<Dictionary<Guid, (string Path, bool Proxy)>> DownloadAsync(ClaimedJob claimed, IReadOnlyList<Frame> rows, string work,
|
||||
NodeReporter reporter, CancellationToken cancellationToken)
|
||||
{
|
||||
var files = new Dictionary<Guid, string>();
|
||||
var formats = rows.ToDictionary(f => f.Id, f => f.Format);
|
||||
var undecodable = rows.Where(f => (f.Flags & FrameFlags.Undecodable) != 0).Select(f => f.Id).ToHashSet();
|
||||
bool Proxy(ManifestFrame f) =>
|
||||
f.HasProxy && (claimed.UseProxies || (formats.TryGetValue(f.Id, out var format) && ArchiveImages.NeedsProxy(_processing.Archive.Decoder, format)));
|
||||
var files = new Dictionary<Guid, (string, bool)>();
|
||||
// An original that stays undecodable would only be skipped by the render.
|
||||
var fetched = claimed.Frames.Where(f => ((FrameFlags)f.Flags & NotFetched) == 0 && (Proxy(f) || !undecodable.Contains(f.Id))).ToList();
|
||||
var done = 0;
|
||||
var total = claimed.Frames.Count;
|
||||
var total = fetched.Count;
|
||||
await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false);
|
||||
var downloads = claimed.Frames.GroupBy(f => claimed.UseProxies && f.HasProxy ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
|
||||
var downloads = fetched.GroupBy(f => Proxy(f) || f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
|
||||
await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken },
|
||||
async (frames, ct) =>
|
||||
{
|
||||
var frame = frames.First();
|
||||
var path = claimed.UseProxies && frame.HasProxy
|
||||
var file = Proxy(frame)
|
||||
? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false)
|
||||
: await DownloadOriginalAsync(frame, ct).ConfigureAwait(false);
|
||||
: (await DownloadOriginalAsync(frame, work, ct).ConfigureAwait(false), false);
|
||||
lock (files)
|
||||
{
|
||||
foreach (var f in frames) files[f.Id] = path;
|
||||
foreach (var f in frames) files[f.Id] = file;
|
||||
}
|
||||
await reporter.ReportAsync(Interlocked.Add(ref done, frames.Count()), total, null).ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
@@ -312,25 +392,29 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
/// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server
|
||||
/// has none for the frame.
|
||||
/// </summary>
|
||||
private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||
private async Task<(string Path, bool Proxy)> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||
{
|
||||
var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
try
|
||||
{
|
||||
await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false);
|
||||
return path;
|
||||
return (path, true);
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
File.Delete(path);
|
||||
return await DownloadOriginalAsync(frame, cancellationToken).ConfigureAwait(false);
|
||||
return (await DownloadOriginalAsync(frame, work, cancellationToken).ConfigureAwait(false), false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256.</summary>
|
||||
private async Task<string> DownloadOriginalAsync(ManifestFrame frame, CancellationToken cancellationToken)
|
||||
/// <summary>
|
||||
/// A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256; an original the
|
||||
/// server has not hashed yet goes into the job folder and must have the manifest's size.
|
||||
/// </summary>
|
||||
private async Task<string> DownloadOriginalAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||
{
|
||||
if (frame.Sha256.Length == 0) return await DownloadUnhashedAsync(frame, work, cancellationToken).ConfigureAwait(false);
|
||||
var sha = frame.Sha256.ToLowerInvariant();
|
||||
if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
|
||||
var path = Path.Combine(_options.CacheDirectory, "originals", sha[..2], sha + Path.GetExtension(frame.FileName).ToLowerInvariant());
|
||||
@@ -355,6 +439,23 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An original without a SHA-256 on the server (adopted, not verified yet): nothing to check it against but its size,
|
||||
/// so it is neither cached nor shared with other frames; a copy of another size is fetched again once.
|
||||
/// </summary>
|
||||
private async Task<string> DownloadUnhashedAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||
{
|
||||
var path = Path.Combine(work, "originals", frame.Id.ToString("N") + Path.GetExtension(frame.FileName).ToLowerInvariant());
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
if (!File.Exists(path) || new FileInfo(path).Length < frame.Size) await FetchAsync(frame, "original", path, resume: true, cancellationToken).ConfigureAwait(false);
|
||||
if (new FileInfo(path).Length == frame.Size) return path;
|
||||
File.Delete(path);
|
||||
if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fetches a frame resource into <paramref name="path"/>. A dropped connection or a busy server is retried (up to
|
||||
/// <see cref="RenderNodeOptions.DownloadRetries"/> times, the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>);
|
||||
|
||||
@@ -36,7 +36,7 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, cancellationToken).ConfigureAwait(false);
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
|
||||
var metadata = frames.LightFrames();
|
||||
var light = LightAnalyzer.Analyze(metadata);
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
using Titano.Core.Analysis;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
@@ -20,22 +22,39 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList<Frame> Fr
|
||||
|
||||
public int Count => Frames.Count;
|
||||
|
||||
/// <summary>Every frame row of <paramref name="sequence"/> with its archive file (the proxy when asked and built).</summary>
|
||||
public static async Task<SequenceFrames> LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies,
|
||||
/// <summary>
|
||||
/// Per frame, whether its file is its proxy. <see cref="FrameFlags.Undecodable"/> is this host's verdict on the
|
||||
/// original, so it does not keep a frame read from its proxy out of a render. Null: none is.
|
||||
/// </summary>
|
||||
public IReadOnlyList<bool>? Proxied { get; init; }
|
||||
|
||||
/// <summary>The originals, for metadata (proxies carry no GPS); null when <see cref="Files"/> are the originals.</summary>
|
||||
public IReadOnlyList<string?>? Originals { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Every frame row of <paramref name="sequence"/> with its archive file: the proxy when asked, and whenever the
|
||||
/// original is one <paramref name="decoder"/> cannot decode (<see cref="ArchiveImages.NeedsProxy(ImageDecoder, ImageFileFormat)"/>:
|
||||
/// HEIF on the server, BRIEF §2.3, §8.5), as long as the proxy exists; else the original.
|
||||
/// </summary>
|
||||
public static async Task<SequenceFrames> LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies, ImageDecoder decoder,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var frames = await ListAsync(db, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var files = new string?[frames.Count];
|
||||
var originals = new string?[frames.Count];
|
||||
var proxied = new bool[frames.Count];
|
||||
for (var i = 0; i < files.Length; i++)
|
||||
{
|
||||
var frame = frames[i];
|
||||
if ((frame.Flags & FrameFlags.Missing) != 0) continue;
|
||||
var proxy = useProxies ? folder.Proxy(frame.Index) : null;
|
||||
files[i] = proxy is not null && File.Exists(proxy) ? proxy : paths.FromRelative(frame.RelativePath);
|
||||
originals[i] = paths.FromRelative(frame.RelativePath);
|
||||
var proxy = useProxies || ArchiveImages.NeedsProxy(decoder, frame.Format) ? folder.Proxy(frame.Index) : null;
|
||||
proxied[i] = proxy is not null && File.Exists(proxy);
|
||||
files[i] = proxied[i] ? proxy : originals[i];
|
||||
}
|
||||
return new SequenceFrames(sequence, frames, files);
|
||||
return new SequenceFrames(sequence, frames, files) { Proxied = proxied, Originals = originals };
|
||||
}
|
||||
|
||||
public static async Task<List<Frame>> ListAsync(TitanoDatabase db, Guid sequenceId, CancellationToken cancellationToken)
|
||||
@@ -80,15 +99,17 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList<Frame> Fr
|
||||
{
|
||||
for (var i = 0; i < Frames.Count; i++)
|
||||
{
|
||||
if (Files[i] is null || (Frames[i].Flags & NotRendered) != 0) yield return i;
|
||||
var skip = Proxied is { } proxied && proxied[i] ? NotRendered & ~FrameFlags.Undecodable : NotRendered;
|
||||
if (Files[i] is null || (Frames[i].Flags & skip) != 0) yield return i;
|
||||
}
|
||||
}
|
||||
|
||||
private (int Index, GpsPosition Gps)? FirstGpsFix()
|
||||
{
|
||||
for (var i = 0; i < Files.Count; i++)
|
||||
var files = Originals ?? Files;
|
||||
for (var i = 0; i < files.Count; i++)
|
||||
{
|
||||
if (Files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue;
|
||||
if (files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue;
|
||||
try
|
||||
{
|
||||
return MetadataReader.Read(file).Gps is { } gps ? (i, gps) : null;
|
||||
|
||||
@@ -11,7 +11,8 @@ internal sealed class SyncLinkStore(TitanoDatabase db)
|
||||
"server, remote_id, local_id, title, frame_count, total_bytes, render_count, remote_status, remote_settings_version, " +
|
||||
"remote_frames_version, remote_updated_at, base_local_settings, base_remote_settings, diff_local_frames, diff_remote_frames, " +
|
||||
"diff_local_renders, diff_remote_renders, missing_remote, missing_local, exports_to_get, exports_to_get_bytes, exports_to_send, " +
|
||||
"exports_to_send_bytes, conflict_local_settings, conflict_remote_settings, conflict_remote, proxies_cached, proxies_frames_version, error, synced_at";
|
||||
"exports_to_send_bytes, conflict_local_settings, conflict_remote_settings, conflict_remote, proxies_cached, proxies_frames_version, error, synced_at, " +
|
||||
"diff_remote_updated_at";
|
||||
|
||||
private static readonly string UpsertSql = SqlDialect.Upsert("sync_links", Columns.Split(", "), 2);
|
||||
|
||||
@@ -36,7 +37,7 @@ internal sealed class SyncLinkStore(TitanoDatabase db)
|
||||
l.DiffLocalRenders, l.DiffRemoteRenders, l.MissingRemote, l.MissingLocal, l.ExportsToGet, l.ExportsToGetBytes, l.ExportsToSend,
|
||||
l.ExportsToSendBytes, l.ConflictLocalSettings, l.ConflictRemoteSettings,
|
||||
l.ConflictRemote is { } remote ? JsonSerializer.Serialize(remote, SyncPayloadJson.Default.SyncPayload) : null,
|
||||
l.ProxiesCached, l.ProxiesFramesVersion, l.Error, l.SyncedAt,
|
||||
l.ProxiesCached, l.ProxiesFramesVersion, l.Error, l.SyncedAt, l.DiffRemoteUpdatedAt,
|
||||
], cancellationToken);
|
||||
|
||||
public Task DeleteAsync(string server, Guid remoteId, CancellationToken cancellationToken) =>
|
||||
@@ -103,5 +104,6 @@ internal sealed class SyncLinkStore(TitanoDatabase db)
|
||||
ProxiesFramesVersion = r.GetInt32OrNull(27),
|
||||
Error = r.GetStringOrNull(28),
|
||||
SyncedAt = r.GetDateTimeOffsetOrNull(29),
|
||||
DiffRemoteUpdatedAt = r.GetDateTimeOffsetOrNull(30),
|
||||
};
|
||||
}
|
||||
@@ -91,7 +91,18 @@ public sealed record SyncLink
|
||||
public int? DiffLocalRenders { get; init; }
|
||||
public int? DiffRemoteRenders { get; init; }
|
||||
|
||||
/// <summary>Local originals the server does not have.</summary>
|
||||
/// <summary>
|
||||
/// The server's update time the originals were compared at, kept while proxies it asks for may be made here (some
|
||||
/// were counted, or the copy holds originals the server cannot decode and this PC can); null otherwise. The proxies
|
||||
/// it asks for change without its frame counters moving (one arrives from a render node, its proxy policy changes),
|
||||
/// and every such change leaves a new update time, so the originals are compared again when it moved.
|
||||
/// </summary>
|
||||
public DateTimeOffset? DiffRemoteUpdatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Local originals the server does not have, and proxies it asks for that this PC can make (it cannot decode those
|
||||
/// originals): an upload sends both.
|
||||
/// </summary>
|
||||
public int MissingRemote { get; init; }
|
||||
|
||||
/// <summary>Server originals this PC does not have.</summary>
|
||||
|
||||
@@ -42,6 +42,13 @@ public interface ISyncRemote : IDisposable
|
||||
/// <summary>Streams a frame's original, proxy or thumb, resuming with Range from the destination's length.</summary>
|
||||
Task DownloadFrameAsync(Guid frameId, string kind, Stream destination, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Sends the proxy of a frame the server cannot decode (<see cref="ManifestFrame.WantsProxy"/>); a 422
|
||||
/// <see cref="TitanoApiException"/> means it was refused (not a JPEG of that frame), 409 that the server makes no
|
||||
/// proxies. A remote without the call (the screenshots' sample server) throws <see cref="NotSupportedException"/>.
|
||||
/// </summary>
|
||||
Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
|
||||
/// <summary>Streams a render's file, resuming with Range from the destination's length; returns the file name the server gives it.</summary>
|
||||
Task<string?> DownloadRenderAsync(Guid renderId, Stream destination, CancellationToken cancellationToken);
|
||||
|
||||
@@ -116,6 +123,8 @@ public sealed class TitanoSyncRemote(TitanoClient client, bool ownsClient) : ISy
|
||||
public Task DownloadFrameAsync(Guid frameId, string kind, Stream destination, CancellationToken cancellationToken) =>
|
||||
client.DownloadFrameAsync(frameId, kind, destination, null, cancellationToken);
|
||||
|
||||
public Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken cancellationToken) => client.UploadProxyAsync(frameId, jpeg, cancellationToken);
|
||||
|
||||
public Task<string?> DownloadRenderAsync(Guid renderId, Stream destination, CancellationToken cancellationToken) =>
|
||||
client.DownloadRenderAsync(renderId, destination, cancellationToken);
|
||||
|
||||
|
||||
@@ -2,8 +2,10 @@ using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.Json.Serialization;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Protocol;
|
||||
|
||||
namespace Titano.Data.Sync;
|
||||
@@ -130,6 +132,43 @@ public static class SyncRules
|
||||
return (hashed.Count - matched + Math.Max(0, unhashed - unmatchedRemote), Math.Max(0, unmatchedRemote - unhashed));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Frames whose proxy the server asks for (it cannot decode their originals, BRIEF §2.3) that this PC holds and its
|
||||
/// <paramref name="decoder"/> reads: an upload makes and sends them. A local original not hashed yet (a folder adopted
|
||||
/// and not verified) stands for a wanted frame of its size that no hash matched (by size, not name: the server names
|
||||
/// its copy by its own template); the upload hashes it before making the proxy.
|
||||
/// </summary>
|
||||
public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection<Frame> local, ImageDecoder decoder)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(manifest);
|
||||
var hashed = local.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal);
|
||||
var unhashed = local.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f)).GroupBy(f => f.Size).ToDictionary(g => g.Key, g => g.Count());
|
||||
var count = 0;
|
||||
foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length > 0))
|
||||
{
|
||||
var sha = frame.Sha256.ToLowerInvariant();
|
||||
if (hashed.Contains(sha))
|
||||
{
|
||||
if (hashed[sha].Any(Readable)) count++;
|
||||
}
|
||||
else if (unhashed.TryGetValue(frame.Size, out var left) && left > 0)
|
||||
{
|
||||
unhashed[frame.Size] = left - 1;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
|
||||
bool Readable(Frame frame) => !ArchiveImages.NeedsProxy(decoder, frame.Format);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether <paramref name="local"/> holds originals a server cannot decode (it asks for their proxies, BRIEF §2.3,
|
||||
/// §4.2: managed decoders only) that this PC's <paramref name="decoder"/> reads, so it can make those proxies.
|
||||
/// </summary>
|
||||
public static bool MakesProxiesFor(IEnumerable<Frame> local, ImageDecoder decoder) =>
|
||||
local.Select(f => f.Format).Distinct().Any(format => ArchiveImages.ServerNeedsProxy(format) && !ArchiveImages.NeedsProxy(decoder, format));
|
||||
|
||||
internal static string Blank(string? text) => text?.Trim() ?? string.Empty;
|
||||
|
||||
private static JsonNode? Parse(string json)
|
||||
|
||||
@@ -43,8 +43,9 @@ public sealed record SyncOptions
|
||||
/// conflict the user settles with <see cref="ResolveAsync"/>, against the versions the user was shown. Uploads and
|
||||
/// downloads run as jobs (<see cref="CreateJobHandlers"/>): uploads use the chunked, hash-verified, resumable protocol;
|
||||
/// downloaded originals are verified and committed through the archive ingest, so sidecars stay right; proxies go to a
|
||||
/// cache and open as a <see cref="RemoteSequence"/>; renders travel on request. Proxies and thumbnails never travel
|
||||
/// upward. A local copy and a server sequence holding the same originals are linked by content when no link joins
|
||||
/// cache and open as a <see cref="RemoteSequence"/>; renders travel on request. Thumbnails never travel upward, and
|
||||
/// proxies only for originals the server cannot decode (BRIEF §2.3: HEIF), made here and counted with the originals the
|
||||
/// server lacks. A local copy and a server sequence holding the same originals are linked by content when no link joins
|
||||
/// them. The state of each local sequence is written to its <c>sync_state</c> for the Library. A failure while
|
||||
/// comparing one sequence is recorded on its link and does not stop the others.
|
||||
/// </summary>
|
||||
@@ -58,7 +59,7 @@ public sealed class SyncService
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SyncLinkStore _links;
|
||||
private readonly ProjectStore _projects;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
|
||||
/// <summary>Unlinked pairs whose manifests were compared at these frame versions without a match (not fetched again).</summary>
|
||||
@@ -70,7 +71,7 @@ public sealed class SyncService
|
||||
Archives = archives;
|
||||
Options = options;
|
||||
_links = new SyncLinkStore(db);
|
||||
_projects = new ProjectStore(db, new SequenceWriter(options.Archive));
|
||||
_writer = new SequenceWriter(options.Archive);
|
||||
Proxies = new ProxyCache(options.ProxyCacheDirectory);
|
||||
}
|
||||
|
||||
@@ -211,8 +212,17 @@ public sealed class SyncService
|
||||
else
|
||||
{
|
||||
if (resolution == SyncResolution.SaveBoth) saved = await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false);
|
||||
var applied = await ApplyLocallyAsync(local, theirs, cancellationToken).ConfigureAwait(false);
|
||||
link = Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, seenRemote);
|
||||
if (await ApplyLocallyAsync(local, theirs, cancellationToken).ConfigureAwait(false) is { } applied)
|
||||
{
|
||||
link = Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, seenRemote);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Edited on this PC since it was read: the copy of a version the user did not choose to keep goes.
|
||||
changed = true;
|
||||
if (saved is not null) File.Delete(saved);
|
||||
saved = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!changed)
|
||||
@@ -359,7 +369,8 @@ public sealed class SyncService
|
||||
var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false);
|
||||
var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false);
|
||||
var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false);
|
||||
var applied = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false);
|
||||
// Edited on this PC meanwhile: nothing is agreed yet, so the next comparison settles both sides' content.
|
||||
if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false) is not { } applied) return;
|
||||
await _links.UpsertAsync(Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, manifest.SettingsVersion), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
@@ -479,13 +490,19 @@ public sealed class SyncService
|
||||
}
|
||||
SequenceManifest? manifest = null;
|
||||
if (link.DiffLocalFrames != local.FramesVersion || link.DiffRemoteFrames != link.RemoteFramesVersion
|
||||
|| link.DiffLocalRenders != local.RenderedCount || link.DiffRemoteRenders != link.RenderCount)
|
||||
|| link.DiffLocalRenders != local.RenderedCount || link.DiffRemoteRenders != link.RenderCount
|
||||
|| (link.DiffRemoteUpdatedAt is { } compared && !SameMoment(compared, link.RemoteUpdatedAt)))
|
||||
{
|
||||
manifest = await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
|
||||
var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false);
|
||||
// A frame flagged missing here counts as absent: downloading the originals restores it from the server.
|
||||
var (missingRemote, missingLocal) = SyncRules.CompareOriginals(
|
||||
[.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]);
|
||||
var present = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList();
|
||||
var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]);
|
||||
// Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3).
|
||||
var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder) : 0;
|
||||
// Those change without the server's frame counters moving: while some may be made here, the server's next
|
||||
// change of the sequence brings a new comparison.
|
||||
var watch = policy.CanEdit && (proxies > 0 || SyncRules.MakesProxiesFor(present, Options.Archive.Decoder));
|
||||
link = await CompareExportsAsync(link, local, manifest.Renders, cancellationToken).ConfigureAwait(false) with
|
||||
{
|
||||
DiffLocalFrames = local.FramesVersion,
|
||||
@@ -493,7 +510,8 @@ public sealed class SyncService
|
||||
RemoteFramesVersion = manifest.FramesVersion,
|
||||
DiffLocalRenders = local.RenderedCount,
|
||||
DiffRemoteRenders = manifest.Renders.Count,
|
||||
MissingRemote = missingRemote,
|
||||
DiffRemoteUpdatedAt = watch ? link.RemoteUpdatedAt : null,
|
||||
MissingRemote = missingRemote + proxies,
|
||||
MissingLocal = missingLocal,
|
||||
};
|
||||
}
|
||||
@@ -547,8 +565,13 @@ public sealed class SyncService
|
||||
return (policy.CanEdit ? await PushAsync(remote, link, local, link.BaseRemoteSettings, cancellationToken).ConfigureAwait(false) : link, local);
|
||||
manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
|
||||
if (manifest.SettingsVersion != dto.SettingsVersion) return (link, local);
|
||||
var pulled = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false);
|
||||
return (Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion), pulled);
|
||||
if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false) is { } pulled)
|
||||
return (Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion), pulled);
|
||||
// The local copy was read when the comparison began and an edit was saved on this PC since: both sides changed,
|
||||
// so the server's version must not replace it unseen (BRIEF §14.5).
|
||||
var fresh = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (fresh is null || fresh.SettingsVersion == local.SettingsVersion) return (link, local);
|
||||
return await SettleSettingsAsync(remote, link, dto, fresh, manifest, policy, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Sends this PC's payload unless the server moved past <paramref name="expected"/>, which shows up as a conflict next.</summary>
|
||||
@@ -621,20 +644,67 @@ public sealed class SyncService
|
||||
|
||||
private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified);
|
||||
|
||||
/// <summary>Writes a payload into the local sidecar, project and index (each write bumps the local settings version).</summary>
|
||||
private async Task<Sequence> ApplyLocallyAsync(Sequence local, SyncPayload payload, CancellationToken cancellationToken)
|
||||
/// <summary>The same instant as databases keep it (to the microsecond).</summary>
|
||||
private static bool SameMoment(DateTimeOffset a, DateTimeOffset b) => a.UtcTicks / 10 == b.UtcTicks / 10;
|
||||
|
||||
/// <summary>
|
||||
/// Writes a payload into the local index, project and sidecar in one step (settings version + 1) under the sequence's
|
||||
/// lease, which the editor's saves take too, and only while the local copy is still at the settings version
|
||||
/// <paramref name="local"/> was read at. Null when an edit was saved on this PC since: nothing is written then.
|
||||
/// </summary>
|
||||
private async Task<Sequence?> ApplyLocallyAsync(Sequence local, SyncPayload payload, CancellationToken cancellationToken)
|
||||
{
|
||||
var current = local;
|
||||
if (!payload.SameMetadata(SyncPayload.Of(local)))
|
||||
current = await Archives.SetSequenceInfoAsync(local.Id, payload.Title, payload.Tags, payload.Rating, payload.Notes, cancellationToken).ConfigureAwait(false);
|
||||
if (!SyncRules.SameSettings(payload.SettingsJson, current.SettingsJson))
|
||||
var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed.");
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(local.Folder);
|
||||
await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var archive = await _db.Archives.GetAsync(current.ArchiveId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidDataException($"Archive {current.ArchiveId} is not indexed.");
|
||||
var project = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson));
|
||||
await _projects.UpdateAsync(new ArchivePaths(archive.RootPath), current, _ => project, cancellationToken).ConfigureAwait(false);
|
||||
var row = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidDataException($"Sequence {local.Id} is not indexed.");
|
||||
if (row.SettingsVersion != local.SettingsVersion) return null;
|
||||
var metadata = !payload.SameMetadata(SyncPayload.Of(row));
|
||||
var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson);
|
||||
if (!metadata && !settings) return row;
|
||||
|
||||
string? project = null;
|
||||
Guid? hero = null;
|
||||
if (settings)
|
||||
{
|
||||
var edited = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson));
|
||||
project = edited.ToJson();
|
||||
hero = await HeroFrame.EditAsync(_db, row.Id, ProjectStore.Read(row, folder).HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
var edit = new SequenceEdit
|
||||
{
|
||||
Title = metadata ? payload.Title?.Trim() ?? string.Empty : null,
|
||||
Tags = metadata ? payload.Tags : null,
|
||||
Rating = metadata ? payload.Rating : null,
|
||||
Notes = metadata ? payload.Notes ?? string.Empty : null,
|
||||
SettingsJson = project,
|
||||
HeroFrameId = hero,
|
||||
};
|
||||
// The index first, at the version read under the lease: a writer that skips the lease still cannot be overwritten.
|
||||
var result = await _db.Sequences.EditAsync(row.Id, edit, row.SettingsVersion, cancellationToken).ConfigureAwait(false);
|
||||
if (result is not { Outcome: SequenceEditOutcome.Updated, Sequence: { } updated }) return null;
|
||||
if (project is not null) AtomicFile.WriteAllText(folder.ProjectPath, project);
|
||||
if (metadata)
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(folder, row.Id);
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
|
||||
{
|
||||
Title = updated.Title,
|
||||
Tags = updated.Tags,
|
||||
Rating = updated.Rating,
|
||||
Notes = updated.Notes,
|
||||
Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
await Archives.Indexer.RecordStampsAsync(archive, folder, row.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
return await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) ?? current;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -809,6 +879,7 @@ public sealed class SyncService
|
||||
DiffRemoteFrames = null,
|
||||
DiffLocalRenders = null,
|
||||
DiffRemoteRenders = null,
|
||||
DiffRemoteUpdatedAt = null,
|
||||
MissingRemote = 0,
|
||||
MissingLocal = 0,
|
||||
ExportsToGet = 0,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using Titano.Core.Archiving;
|
||||
@@ -16,7 +17,8 @@ namespace Titano.Data.Sync;
|
||||
/// <summary>
|
||||
/// The work of upload and download jobs. Uploads send every local original the server lacks through the chunked,
|
||||
/// resumable protocol after hashing it again (an original that no longer matches its archived hash is never sent),
|
||||
/// creating the server sequence first and sending its settings when the upload made it. Downloads fetch originals the
|
||||
/// creating the server sequence first and sending its settings when the upload made it, then the proxies the server asks
|
||||
/// for (originals it cannot decode, BRIEF §2.3), made here from this PC's originals. Downloads fetch originals the
|
||||
/// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 and commit
|
||||
/// them through the archive ingest, which names them and writes the sidecar; the first original creates the local
|
||||
/// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the
|
||||
@@ -49,12 +51,16 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count);
|
||||
await progress.ReportAsync().ConfigureAwait(false);
|
||||
var skipped = 0;
|
||||
// The SHA-256 of each original the index has none for (a folder adopted and not verified), as its upload computes
|
||||
// it: the proxies the server asks for are matched to this PC's originals by content. Every such original is
|
||||
// pending, so each one still here is hashed.
|
||||
var hashes = new ConcurrentDictionary<Guid, string>();
|
||||
|
||||
// Without a server sequence the originals go out one by one under this sequence's session key until the
|
||||
// server has filed one, which creates the sequence (an original it already holds elsewhere creates nothing).
|
||||
while (remoteId is null && pending.TryDequeue(out var first))
|
||||
{
|
||||
var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, ct)
|
||||
var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, hashes, ct)
|
||||
.ConfigureAwait(false);
|
||||
if (sent is null) skipped++;
|
||||
else if (sent.SequenceId != Guid.Empty)
|
||||
@@ -69,16 +75,100 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
|
||||
await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) =>
|
||||
{
|
||||
if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, token).ConfigureAwait(false) is null)
|
||||
if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, hashes, token).ConfigureAwait(false) is null)
|
||||
Interlocked.Increment(ref skipped);
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
await SendProxiesAsync(context, remote, target, local, paths, frames, hashes).ConfigureAwait(false);
|
||||
|
||||
if (created) await sync.PushAllAsync(remote, target, ct).ConfigureAwait(false);
|
||||
await sync.RecompareAsync(remote, target, ct).ConfigureAwait(false);
|
||||
if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The proxies the server asks for (<see cref="ManifestFrame.WantsProxy"/>) that this PC's decoders can make from its
|
||||
/// originals (by their archived SHA-256, or the one <paramref name="hashes"/> holds for an original the index has none
|
||||
/// for), made as the server's proxy policy says with the server's capture data, and sent. One the server refuses is
|
||||
/// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a
|
||||
/// server busy or out of reach fails the job like an original would.
|
||||
/// </summary>
|
||||
private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList<Frame> frames,
|
||||
IReadOnlyDictionary<Guid, string> hashes)
|
||||
{
|
||||
var ct = context.Cancellation;
|
||||
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
|
||||
var wanted = manifest.Frames.Where(f => f.WantsProxy).ToList();
|
||||
if (manifest.Proxies is not { } spec || wanted.Count == 0) return;
|
||||
var originals = new Dictionary<string, Frame>(StringComparer.Ordinal);
|
||||
foreach (var frame in frames)
|
||||
{
|
||||
if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame);
|
||||
}
|
||||
var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id);
|
||||
var camera = new SidecarCamera { Make = local.CameraMake, Model = local.CameraModel, Serial = local.CameraSerial };
|
||||
var decoder = sync.Options.Archive.Decoder;
|
||||
var progress = new TransferProgress(context, wanted.Count, 0);
|
||||
await progress.ReportAsync().ConfigureAwait(false);
|
||||
int sent = 0, unsent = 0;
|
||||
using var refused = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
try
|
||||
{
|
||||
await Parallel.ForEachAsync(wanted, Workers(sync.Options.Uploads, refused.Token), async (frame, token) =>
|
||||
{
|
||||
var jpeg = originals.TryGetValue(frame.Sha256.ToLowerInvariant(), out var original) && records.TryGetValue(frame.Id, out var record)
|
||||
? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName,
|
||||
token), token).ConfigureAwait(false)
|
||||
: null;
|
||||
try
|
||||
{
|
||||
if (jpeg is not null)
|
||||
{
|
||||
await remote.UploadProxyAsync(frame.Id, jpeg, token).ConfigureAwait(false);
|
||||
Interlocked.Increment(ref sent);
|
||||
}
|
||||
else
|
||||
{
|
||||
Interlocked.Increment(ref unsent);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is NotSupportedException or TitanoApiException { Status: HttpStatusCode.Conflict or HttpStatusCode.Forbidden })
|
||||
{
|
||||
// The server takes no proxies (or none from this account): nothing more to send.
|
||||
Core.Diagnostics.Log.Info("sync", $"Proxies of {local.DisplayTitle} not sent: {ex.Message}");
|
||||
await refused.CancelAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (TitanoApiException ex) when (!ex.IsTransient)
|
||||
{
|
||||
// This proxy only (not a JPEG of that frame, a frame gone meanwhile): the others still go.
|
||||
Core.Diagnostics.Log.Warning("sync", $"The proxy of {frame.FileName} was refused", ex);
|
||||
Interlocked.Increment(ref unsent);
|
||||
}
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (refused.IsCancellationRequested && !ct.IsCancellationRequested)
|
||||
{
|
||||
unsent = wanted.Count - sent;
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.ProxiesSent", sent, unsent));
|
||||
}
|
||||
|
||||
/// <summary>A proxy made here for the server; null (logged) when this PC cannot decode or read the original either.</summary>
|
||||
private static byte[]? MakeProxy(Core.Imaging.ImageDecoder decoder, string path, Frame frame, SidecarCamera camera, ProxySpec spec, string name,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
return ArchiveImages.MakeProxy(decoder, path, frame, camera, spec.LongEdge, spec.Quality, cancellationToken);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or NotSupportedException)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("sync", $"No proxy of {name} could be made", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DownloadOriginalsAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid? targetArchive)
|
||||
{
|
||||
var ct = context.Cancellation;
|
||||
@@ -364,10 +454,11 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
|
||||
/// <summary>
|
||||
/// Hashes the original again and sends it into <paramref name="sequenceId"/> (or the sequence of
|
||||
/// <paramref name="sessionKey"/>); null (logged) when it is gone or no longer the archived file.
|
||||
/// <paramref name="sessionKey"/>); null (logged) when it is gone or no longer the archived file. The hash of an
|
||||
/// original the index has none for goes into <paramref name="hashes"/>.
|
||||
/// </summary>
|
||||
private static async Task<CompleteUploadResponse?> UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path,
|
||||
Frame frame, CancellationToken cancellationToken)
|
||||
Frame frame, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
@@ -375,6 +466,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
return null;
|
||||
}
|
||||
var sha = await TitanoClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false);
|
||||
if (frame.Sha256 is null) hashes[frame.Id] = sha;
|
||||
if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
context.Log($"{frame.FileName}: changed since it was archived, not uploaded");
|
||||
@@ -397,7 +489,11 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two attempts).</summary>
|
||||
/// <summary>
|
||||
/// A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two
|
||||
/// attempts). A part as long as the file (a job stopped while hashing it) is only hashed again; a part the server can
|
||||
/// no longer resume (416) starts over.
|
||||
/// </summary>
|
||||
private static async Task<string> FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging, CancellationToken cancellationToken)
|
||||
{
|
||||
var sha = frame.Sha256.ToLowerInvariant();
|
||||
@@ -407,8 +503,19 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
var part = target + ".part";
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
|
||||
await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false);
|
||||
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
|
||||
await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1)
|
||||
{
|
||||
File.Delete(part);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
|
||||
{
|
||||
File.Move(part, target, overwrite: true);
|
||||
|
||||
@@ -142,6 +142,25 @@ public sealed class TitanoClient : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends the proxy of a frame whose original the server cannot decode (<see cref="ManifestFrame.WantsProxy"/>): a JPEG
|
||||
/// with the frame's capture time in its Exif, replacing the server's. A transient failure is retried.
|
||||
/// </summary>
|
||||
public Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(jpeg);
|
||||
return RetryAsync(async () =>
|
||||
{
|
||||
await Bandwidth.AcquireAsync(jpeg.Length, ct).ConfigureAwait(false);
|
||||
using var content = new ByteArrayContent(jpeg);
|
||||
content.Headers.ContentType = new MediaTypeHeaderValue("image/jpeg");
|
||||
using var response = await SendRawAsync(new HttpRequestMessage(HttpMethod.Put, $"api/v1/frames/{frameId}/proxy") { Content = content }, ct)
|
||||
.ConfigureAwait(false);
|
||||
await EnsureSuccessAsync(response, ct).ConfigureAwait(false);
|
||||
return true;
|
||||
}, ct);
|
||||
}
|
||||
|
||||
// ---------------- Uploads ----------------
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -37,7 +37,10 @@ public sealed record NodeEncoder(string Codec, string Name, bool Hardware, bool
|
||||
public sealed record RenderNodeDto(Guid Id, string ClientName, string Username, IReadOnlyList<NodeEncoder> Encoders, DateTimeOffset LastHeartbeat, string State);
|
||||
|
||||
/// <summary><c>POST /api/v1/nodes/{id}/claim</c>: the job and everything needed to run it, or 204 when none matches.</summary>
|
||||
public sealed record ClaimedJob(JobDto Job, Guid SequenceId, string SettingsJson, string PresetJson, bool UseProxies, IReadOnlyList<ManifestFrame> Frames);
|
||||
/// <param name="Frames">Every frame of the sequence; a node renders none flagged missing, corrupt or excluded.</param>
|
||||
/// <param name="Proxies">How to make the proxies the server asks for (<see cref="ManifestFrame.WantsProxy"/>); null when it makes none.</param>
|
||||
public sealed record ClaimedJob(JobDto Job, Guid SequenceId, string SettingsJson, string PresetJson, bool UseProxies, IReadOnlyList<ManifestFrame> Frames,
|
||||
ProxySpec? Proxies = null);
|
||||
|
||||
/// <summary><c>POST /api/v1/nodes/{id}/jobs/{jobId}/progress</c>.</summary>
|
||||
public sealed record NodeProgress(long Done, long Total, double? ItemsPerSecond, string? LogLine);
|
||||
|
||||
@@ -65,6 +65,7 @@ public sealed record FrameDto(
|
||||
long Flags);
|
||||
|
||||
/// <summary><c>GET /api/v1/sequences/{id}/manifest</c>: what a client needs to diff a sequence (BRIEF §14.5).</summary>
|
||||
/// <param name="Proxies">How to make the proxies the server asks for (<see cref="ManifestFrame.WantsProxy"/>); null when it makes none.</param>
|
||||
public sealed record SequenceManifest(
|
||||
Guid SequenceId,
|
||||
int FramesVersion,
|
||||
@@ -72,9 +73,21 @@ public sealed record SequenceManifest(
|
||||
string? SettingsJson,
|
||||
IReadOnlyList<string> Tags,
|
||||
IReadOnlyList<ManifestFrame> Frames,
|
||||
IReadOnlyList<RenderDto> Renders);
|
||||
IReadOnlyList<RenderDto> Renders,
|
||||
ProxySpec? Proxies = null);
|
||||
|
||||
public sealed record ManifestFrame(Guid Id, int Index, string FileName, long Size, string Sha256, bool HasProxy, bool HasThumb);
|
||||
/// <summary>One frame of a manifest: its content, whether the server has a proxy and thumbnail of it, and its flags.</summary>
|
||||
/// <param name="Sha256">Lower-case hex SHA-256; empty while the server has not hashed it (a folder adopted and not verified yet).</param>
|
||||
/// <param name="Flags">The frame's <c>FrameFlags</c> bits (missing, corrupt, excluded…).</param>
|
||||
/// <param name="WantsProxy">
|
||||
/// The server cannot decode this original and has no proxy of it (BRIEF §2.3, §8.1: HEIF): a client or render node that
|
||||
/// decodes it sends one with <c>PUT /api/v1/frames/{id}/proxy</c>, made as <see cref="SequenceManifest.Proxies"/> says.
|
||||
/// </param>
|
||||
public sealed record ManifestFrame(Guid Id, int Index, string FileName, long Size, string Sha256, bool HasProxy, bool HasThumb, long Flags = 0,
|
||||
bool WantsProxy = false);
|
||||
|
||||
/// <summary>How the server's proxies are made (BRIEF §8.5): long edge in pixels (0 = full size) and JPEG quality.</summary>
|
||||
public sealed record ProxySpec(int LongEdge, int Quality);
|
||||
|
||||
public sealed record RenderDto(
|
||||
Guid Id,
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
using System.Globalization;
|
||||
using Microsoft.AspNetCore.Http.Features;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Server.Jobs;
|
||||
using Titano.Server.Security;
|
||||
using Titano.Server.Storage;
|
||||
|
||||
@@ -8,7 +14,8 @@ namespace Titano.Server.Api;
|
||||
|
||||
/// <summary>
|
||||
/// Streamed archive files with Range support (BRIEF §14.3): originals, proxies, thumbnails and exports. Paths come
|
||||
/// from the database by id and are resolved under the archive root only.
|
||||
/// from the database by id and are resolved under the archive root only. Proxies of frames the server cannot decode
|
||||
/// arrive from clients and render nodes (<c>PUT /frames/{id}/proxy</c>, BRIEF §2.3).
|
||||
/// </summary>
|
||||
public static class FileEndpoints
|
||||
{
|
||||
@@ -21,6 +28,8 @@ public static class FileEndpoints
|
||||
api.MapGet("/frames/{id:guid}/original", OriginalAsync).Requires(Permissions.Edit | Permissions.RenderNode);
|
||||
api.MapGet("/frames/{id:guid}/proxy", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Proxy, db, a, l))
|
||||
.Requires(Permissions.Read | Permissions.RenderNode);
|
||||
// Made by clients that edit and by render nodes, which decode what the server cannot.
|
||||
api.MapPut("/frames/{id:guid}/proxy", StoreProxyAsync).Requires(Permissions.Edit | Permissions.RenderNode);
|
||||
api.MapGet("/frames/{id:guid}/thumb", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Thumbnail, db, a, l))
|
||||
.Requires(Permissions.Read);
|
||||
api.MapGet("/renders/{id:guid}/file", RenderAsync).Requires(Permissions.Read);
|
||||
@@ -48,6 +57,55 @@ public static class FileEndpoints
|
||||
?? ApiErrors.NotFound(context, "asset_missing");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Files the body (a JPEG made from the original, its Exif capture time the frame's, as large as the archive's proxies,
|
||||
/// at most <see cref="ArchiveImages.MaxProxyBytes"/>) as the proxy of a frame the server cannot decode
|
||||
/// (<see cref="ServerProxies"/>); the sequence's thumbnails and proxy state follow as background jobs. 204; 413
|
||||
/// <c>too_large</c>; 422 <c>proxy_not_wanted</c> (the server decodes that original itself, or it is missing or
|
||||
/// damaged), <c>invalid_proxy</c>, <c>proxy_mismatch</c> or <c>proxy_too_small</c>, which concern that proxy only; 409
|
||||
/// <c>proxies_disabled</c>, which concerns them all; 503 <c>archive_offline</c>.
|
||||
/// </summary>
|
||||
private static async Task<IResult> StoreProxyAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive,
|
||||
[FromServices] ArchiveService? archives, SequenceJobs follow)
|
||||
{
|
||||
var ct = context.RequestAborted;
|
||||
if (await db.Frames.GetAsync(id, ct) is not { } frame || await db.Sequences.GetAsync(frame.SequenceId, ct) is not { } sequence)
|
||||
return ApiErrors.NotFound(context, "frame");
|
||||
// A proxy replaces the server's own of an original it decodes, which renders and remote views would then show.
|
||||
if (!ServerProxies.CannotDecode(frame.Format) || (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0)
|
||||
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_not_wanted");
|
||||
if (context.Request.ContentLength is > ArchiveImages.MaxProxyBytes) return ApiErrors.Result(context, StatusCodes.Status413PayloadTooLarge, "too_large");
|
||||
// Proxies are larger than the JSON bodies Kestrel is limited to; reading past this answers 413.
|
||||
if (context.Features.Get<IHttpMaxRequestBodySizeFeature>() is { IsReadOnly: false } limit) limit.MaxRequestBodySize = ArchiveImages.MaxProxyBytes;
|
||||
using var body = new MemoryStream();
|
||||
await context.Request.Body.CopyToAsync(body, ct);
|
||||
|
||||
var images = (archives ?? new ArchiveService(db, new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = ArchiveModule.Owner })).Images;
|
||||
ProxyStoreResult result;
|
||||
try
|
||||
{
|
||||
result = images.StoreProxy(archive.Current, sequence, frame, body.ToArray());
|
||||
}
|
||||
catch (ArchiveOfflineException)
|
||||
{
|
||||
return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline");
|
||||
}
|
||||
switch (result)
|
||||
{
|
||||
case ProxyStoreResult.Disabled:
|
||||
return ApiErrors.Conflict(context, "proxies_disabled");
|
||||
case ProxyStoreResult.Invalid:
|
||||
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "invalid_proxy");
|
||||
case ProxyStoreResult.WrongFrame:
|
||||
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_mismatch");
|
||||
case ProxyStoreResult.TooSmall:
|
||||
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_too_small");
|
||||
}
|
||||
// The proxy is filed: its follow-up is queued even when the sender is gone by now.
|
||||
await follow.ProxyArrivedAsync(sequence, CancellationToken.None);
|
||||
return Results.NoContent();
|
||||
}
|
||||
|
||||
private static async Task<IResult> RenderAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive)
|
||||
{
|
||||
if (await db.Renders.GetAsync(id, context.RequestAborted) is not { } render) return ApiErrors.NotFound(context, "render");
|
||||
|
||||
@@ -213,29 +213,24 @@ public static class SequenceEndpoints
|
||||
_ => ApiErrors.Conflict(context, "folder_missing"),
|
||||
};
|
||||
|
||||
/// <summary>Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5); ETag over the version counters.</summary>
|
||||
private static async Task<IResult> ManifestAsync(HttpContext context, Guid id, TitanoDatabase db)
|
||||
/// <summary>
|
||||
/// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5), with the proxies the server asks
|
||||
/// for (<see cref="ServerProxies"/>); ETag over the version counters and the number of proxies asked for.
|
||||
/// </summary>
|
||||
private static async Task<IResult> ManifestAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive)
|
||||
{
|
||||
var ct = context.RequestAborted;
|
||||
if (await db.Sequences.GetAsync(id, ct) is not { } sequence) return ApiErrors.NotFound(context, "sequence");
|
||||
var etag = string.Create(CultureInfo.InvariantCulture, $"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}\"");
|
||||
var spec = ServerProxies.Spec(archive.Current);
|
||||
var frames = await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, ct);
|
||||
var etag = string.Create(CultureInfo.InvariantCulture,
|
||||
$"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}-{frames.Count(f => f.WantsProxy)}\"");
|
||||
if (context.Request.Headers.IfNoneMatch == etag) return Results.StatusCode(StatusCodes.Status304NotModified);
|
||||
|
||||
var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
|
||||
var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt);
|
||||
var frames = new List<ManifestFrame>(sequence.FrameCount);
|
||||
var after = -1;
|
||||
while (true)
|
||||
{
|
||||
var batch = await db.Frames.ListBySequenceAsync(id, after, 1000, ct);
|
||||
frames.AddRange(batch.Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb)));
|
||||
if (batch.Count < 1000) break;
|
||||
after = batch[^1].Index;
|
||||
}
|
||||
var renders = await db.Renders.ListBySequenceAsync(id, ct);
|
||||
context.Response.Headers.ETag = etag;
|
||||
return Results.Json(new SequenceManifest(id, sequence.FramesVersion, sequence.SettingsVersion, sequence.SettingsJson, sequence.Tags, frames,
|
||||
[.. renders.Select(Dto.Render)]), ProtocolJson.Default.SequenceManifest);
|
||||
[.. renders.Select(Dto.Render)], spec), ProtocolJson.Default.SequenceManifest);
|
||||
}
|
||||
|
||||
private static async Task<IResult> FramesAsync(HttpContext context, Guid id, string? cursor, int? limit, TitanoDatabase db)
|
||||
|
||||
@@ -16,11 +16,14 @@ namespace Titano.Server.Jobs;
|
||||
/// <summary>
|
||||
/// Render nodes (BRIEF §14.7): clients with the render-node right register their encoders, heartbeat, claim queued
|
||||
/// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of
|
||||
/// the sequence. A node that stops heart-beating loses its jobs to the queue (<see cref="RenderNodeMonitor"/>).
|
||||
/// the sequence. A claimed job lists every frame with its flags (a node renders none missing, corrupt or excluded) and
|
||||
/// the proxies the server asks for (<see cref="ServerProxies"/>), which the node sends from the originals it fetched.
|
||||
/// A node that stops heart-beating loses its jobs to the queue (<see cref="RenderNodeMonitor"/>).
|
||||
/// </summary>
|
||||
public sealed class RenderNodeService(
|
||||
TitanoDatabase db,
|
||||
ServerOptions options,
|
||||
ServerArchive archive,
|
||||
UploadService uploads,
|
||||
UploadStore store,
|
||||
RenderStore renders,
|
||||
@@ -84,8 +87,9 @@ public sealed class RenderNodeService(
|
||||
continue;
|
||||
}
|
||||
await TouchAsync(node, cancellationToken);
|
||||
var spec = ServerProxies.Spec(archive.Current);
|
||||
return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson,
|
||||
parameters.UseProxies, await ManifestFramesAsync(sequence, cancellationToken));
|
||||
parameters.UseProxies, await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, cancellationToken), spec);
|
||||
}
|
||||
await TouchAsync(node, cancellationToken);
|
||||
return (ClaimedJob?)null;
|
||||
@@ -234,22 +238,6 @@ public sealed class RenderNodeService(
|
||||
if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state));
|
||||
}
|
||||
|
||||
private async Task<List<ManifestFrame>> ManifestFramesAsync(Sequence sequence, CancellationToken cancellationToken)
|
||||
{
|
||||
var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
|
||||
var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt);
|
||||
var frames = new List<ManifestFrame>(sequence.FrameCount);
|
||||
var after = -1;
|
||||
while (true)
|
||||
{
|
||||
var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, 1000, cancellationToken);
|
||||
frames.AddRange(batch.Where(f => !f.Flags.HasFlag(FrameFlags.Excluded))
|
||||
.Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb)));
|
||||
if (batch.Count < 1000) return frames;
|
||||
after = batch[^1].Index;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The node has an encoder for the job's codec that handles its resolution: the encoder's own limit, else the node's
|
||||
/// (0 = no announced limit).
|
||||
|
||||
@@ -4,21 +4,27 @@ using Titano.Data;
|
||||
namespace Titano.Server.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// Follow-up work after frames arrive (BRIEF §14.4): thumbnails, proxies and light analysis of the sequence, queued at
|
||||
/// background priority once per sequence while one is still waiting, and only for kinds this server can run.
|
||||
/// Follow-up work after frames arrive (BRIEF §14.4): thumbnails, proxies and light analysis of the sequence; after a
|
||||
/// proxy sent by a client or render node arrives, the thumbnails and the sequence's proxy state. Queued at background
|
||||
/// priority once per sequence while one is still waiting, and only for kinds this server can run.
|
||||
/// </summary>
|
||||
public sealed class SequenceJobs(TitanoDatabase db, JobEngineHost jobs)
|
||||
{
|
||||
private static readonly JobKind[] AfterNewFrames = [JobKind.Thumbs, JobKind.Proxies, JobKind.LightAnalyse];
|
||||
private static readonly JobKind[] AfterProxy = [JobKind.Thumbs, JobKind.Proxies];
|
||||
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
|
||||
public async Task FramesAddedAsync(Sequence sequence, CancellationToken cancellationToken)
|
||||
public Task FramesAddedAsync(Sequence sequence, CancellationToken cancellationToken) => QueueAsync(sequence, AfterNewFrames, cancellationToken);
|
||||
|
||||
public Task ProxyArrivedAsync(Sequence sequence, CancellationToken cancellationToken) => QueueAsync(sequence, AfterProxy, cancellationToken);
|
||||
|
||||
private async Task QueueAsync(Sequence sequence, JobKind[] kinds, CancellationToken cancellationToken)
|
||||
{
|
||||
await _gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
foreach (var kind in AfterNewFrames)
|
||||
foreach (var kind in kinds)
|
||||
{
|
||||
if (!jobs.CanRun(kind) || await db.Jobs.IsQueuedAsync(kind, sequence.Id, cancellationToken)) continue;
|
||||
await jobs.Engine.EnqueueAsync(kind, [sequence.Id], sequence.DisplayTitle, priority: JobPriority.Background, cancellationToken: cancellationToken);
|
||||
|
||||
@@ -7,8 +7,9 @@ namespace Titano.Server.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and
|
||||
/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance). Decoding is
|
||||
/// managed only (BRIEF §4.2). The daily backup of an SQLite index runs with it (<see cref="DatabaseBackupService"/>).
|
||||
/// thumbnails, its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance) and the periodic
|
||||
/// incremental scan (<see cref="ArchivePolling"/>). Decoding is managed only (BRIEF §4.2). The daily backup of an
|
||||
/// SQLite index runs with it (<see cref="DatabaseBackupService"/>).
|
||||
/// </summary>
|
||||
public static class ArchiveModule
|
||||
{
|
||||
@@ -23,5 +24,7 @@ public static class ArchiveModule
|
||||
services.AddSingleton<IJobHandlerProvider>(sp => sp.GetRequiredService<ArchiveService>());
|
||||
services.AddSingleton<IFrameAssetLocator, ArchiveAssetLocator>();
|
||||
services.AddHostedService<DatabaseBackupService>();
|
||||
services.AddSingleton<ArchivePolling>();
|
||||
services.AddHostedService(sp => sp.GetRequiredService<ArchivePolling>());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Server.Jobs;
|
||||
|
||||
namespace Titano.Server.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Polls the archive (BRIEF §7: by modification time and size, no file-system watchers): an incremental scan at start
|
||||
/// and every <see cref="Interval"/> while the archive is online, so photos already in an adopted folder and sequences
|
||||
/// another writer adds (a desktop sharing the archive, files copied in by hand) are indexed without an administrator's
|
||||
/// rescan. A scan or rebuild of the archive that is waiting, running or paused is not queued again.
|
||||
/// </summary>
|
||||
public sealed class ArchivePolling(TitanoDatabase db, ServerArchive archive, JobEngineHost jobs, TimeProvider time, ILogger<ArchivePolling> logger)
|
||||
: BackgroundService
|
||||
{
|
||||
/// <summary>The desktop's rhythm (it polls its archive the same way).</summary>
|
||||
public static readonly TimeSpan Interval = TimeSpan.FromMinutes(10);
|
||||
|
||||
private static readonly JobState[] Pending = [JobState.Queued, JobState.Running, JobState.Paused];
|
||||
|
||||
/// <summary>Queues an incremental scan of an online archive unless one is pending; returns it, or null when none was queued.</summary>
|
||||
public async Task<Job?> PollAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (!jobs.CanRun(JobKind.Scan)) return null;
|
||||
var current = archive.Current;
|
||||
if (ArchiveService.GetAvailability(current) != ArchiveAvailability.Online) return null;
|
||||
foreach (var kind in new[] { JobKind.Scan, JobKind.Rebuild })
|
||||
{
|
||||
foreach (var state in Pending)
|
||||
{
|
||||
if ((await db.Jobs.ListAsync(state, kind, 50, null, cancellationToken)).Items.Any(j => j.TargetIds.Contains(current.Id))) return null;
|
||||
}
|
||||
}
|
||||
var scan = ArchiveJobs.Scan(current);
|
||||
// Behind what users asked for (BRIEF §14.6).
|
||||
return await jobs.Engine.EnqueueAsync(scan.Kind, scan.Targets, scan.Label, scan.ParametersJson, JobPriority.Background,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
await PollAsync(stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "The archive could not be polled");
|
||||
}
|
||||
try
|
||||
{
|
||||
await Task.Delay(Interval, time, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Protocol;
|
||||
|
||||
namespace Titano.Server.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Proxies the server asks for (BRIEF §2.3, §8.5): it decodes with managed decoders only, so the frames whose originals
|
||||
/// it cannot decode (HEIF) get their proxies from a Windows client or render node, made as the archive's proxy policy
|
||||
/// says. Manifests (sync and render nodes) list every frame with its flags and whether its proxy is wanted.
|
||||
/// </summary>
|
||||
public static class ServerProxies
|
||||
{
|
||||
private const int Page = 1000;
|
||||
|
||||
/// <summary>The server takes this format's pixels from proxies made elsewhere (its decoders are managed only, BRIEF §4.2).</summary>
|
||||
public static bool CannotDecode(ImageFileFormat format) => ArchiveImages.ServerNeedsProxy(format);
|
||||
|
||||
/// <summary>How proxies of the archive are made; null when it makes none (or its manifest cannot be read now).</summary>
|
||||
public static ProxySpec? Spec(Archive archive)
|
||||
{
|
||||
try
|
||||
{
|
||||
var policy = ManifestStore.ReadArchive(new ArchivePaths(archive.RootPath).ManifestPath).ProxyPolicy;
|
||||
return policy.LongEdge == ProxyPolicy.Disabled ? null : new ProxySpec(policy.LongEdge, policy.Quality);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Every frame of <paramref name="sequence"/> in index order, as a manifest lists it: a proxy is there when the server
|
||||
/// built them all or one was sent for the frame; it is wanted for a frame the server cannot decode that has none
|
||||
/// (and is not missing), unless <paramref name="spec"/> is null.
|
||||
/// </summary>
|
||||
public static async Task<List<ManifestFrame>> FramesAsync(TitanoDatabase db, Archive archive, Sequence sequence, ProxySpec? spec,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var proxies = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
|
||||
var thumbs = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt);
|
||||
var folder = new ArchivePaths(archive.RootPath).Sequence(sequence.Folder);
|
||||
var frames = new List<ManifestFrame>(sequence.FrameCount);
|
||||
var after = -1;
|
||||
while (true)
|
||||
{
|
||||
var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, Page, cancellationToken);
|
||||
foreach (var f in batch)
|
||||
{
|
||||
var sent = !proxies && CannotDecode(f.Format) && File.Exists(folder.Proxy(f.Index));
|
||||
var wanted = spec is not null && !proxies && !sent && CannotDecode(f.Format) && (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0;
|
||||
frames.Add(new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, proxies || sent, thumbs, (long)f.Flags, wanted));
|
||||
}
|
||||
if (batch.Count < Page) return frames;
|
||||
after = batch[^1].Index;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user