Merge branch 'fix/proxies-render-sync': proxies from clients and nodes, node manifests, sync and mass-rename fixes, server polling
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
45 files changed
+1833
-179
No files matched your search
@@ -1,6 +1,12 @@
|
||||
using System.Globalization;
|
||||
using Microsoft.AspNetCore.Http.Features;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Server.Jobs;
|
||||
using Titano.Server.Security;
|
||||
using Titano.Server.Storage;
|
||||
|
||||
@@ -8,7 +14,8 @@ namespace Titano.Server.Api;
|
||||
|
||||
/// <summary>
|
||||
/// Streamed archive files with Range support (BRIEF §14.3): originals, proxies, thumbnails and exports. Paths come
|
||||
/// from the database by id and are resolved under the archive root only.
|
||||
/// from the database by id and are resolved under the archive root only. Proxies of frames the server cannot decode
|
||||
/// arrive from clients and render nodes (<c>PUT /frames/{id}/proxy</c>, BRIEF §2.3).
|
||||
/// </summary>
|
||||
public static class FileEndpoints
|
||||
{
|
||||
@@ -21,6 +28,8 @@ public static class FileEndpoints
|
||||
api.MapGet("/frames/{id:guid}/original", OriginalAsync).Requires(Permissions.Edit | Permissions.RenderNode);
|
||||
api.MapGet("/frames/{id:guid}/proxy", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Proxy, db, a, l))
|
||||
.Requires(Permissions.Read | Permissions.RenderNode);
|
||||
// Made by clients that edit and by render nodes, which decode what the server cannot.
|
||||
api.MapPut("/frames/{id:guid}/proxy", StoreProxyAsync).Requires(Permissions.Edit | Permissions.RenderNode);
|
||||
api.MapGet("/frames/{id:guid}/thumb", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Thumbnail, db, a, l))
|
||||
.Requires(Permissions.Read);
|
||||
api.MapGet("/renders/{id:guid}/file", RenderAsync).Requires(Permissions.Read);
|
||||
@@ -48,6 +57,55 @@ public static class FileEndpoints
|
||||
?? ApiErrors.NotFound(context, "asset_missing");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Files the body (a JPEG made from the original, its Exif capture time the frame's, as large as the archive's proxies,
|
||||
/// at most <see cref="ArchiveImages.MaxProxyBytes"/>) as the proxy of a frame the server cannot decode
|
||||
/// (<see cref="ServerProxies"/>); the sequence's thumbnails and proxy state follow as background jobs. 204; 413
|
||||
/// <c>too_large</c>; 422 <c>proxy_not_wanted</c> (the server decodes that original itself, or it is missing or
|
||||
/// damaged), <c>invalid_proxy</c>, <c>proxy_mismatch</c> or <c>proxy_too_small</c>, which concern that proxy only; 409
|
||||
/// <c>proxies_disabled</c>, which concerns them all; 503 <c>archive_offline</c>.
|
||||
/// </summary>
|
||||
private static async Task<IResult> StoreProxyAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive,
|
||||
[FromServices] ArchiveService? archives, SequenceJobs follow)
|
||||
{
|
||||
var ct = context.RequestAborted;
|
||||
if (await db.Frames.GetAsync(id, ct) is not { } frame || await db.Sequences.GetAsync(frame.SequenceId, ct) is not { } sequence)
|
||||
return ApiErrors.NotFound(context, "frame");
|
||||
// A proxy replaces the server's own of an original it decodes, which renders and remote views would then show.
|
||||
if (!ServerProxies.CannotDecode(frame.Format) || (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0)
|
||||
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_not_wanted");
|
||||
if (context.Request.ContentLength is > ArchiveImages.MaxProxyBytes) return ApiErrors.Result(context, StatusCodes.Status413PayloadTooLarge, "too_large");
|
||||
// Proxies are larger than the JSON bodies Kestrel is limited to; reading past this answers 413.
|
||||
if (context.Features.Get<IHttpMaxRequestBodySizeFeature>() is { IsReadOnly: false } limit) limit.MaxRequestBodySize = ArchiveImages.MaxProxyBytes;
|
||||
using var body = new MemoryStream();
|
||||
await context.Request.Body.CopyToAsync(body, ct);
|
||||
|
||||
var images = (archives ?? new ArchiveService(db, new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = ArchiveModule.Owner })).Images;
|
||||
ProxyStoreResult result;
|
||||
try
|
||||
{
|
||||
result = images.StoreProxy(archive.Current, sequence, frame, body.ToArray());
|
||||
}
|
||||
catch (ArchiveOfflineException)
|
||||
{
|
||||
return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline");
|
||||
}
|
||||
switch (result)
|
||||
{
|
||||
case ProxyStoreResult.Disabled:
|
||||
return ApiErrors.Conflict(context, "proxies_disabled");
|
||||
case ProxyStoreResult.Invalid:
|
||||
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "invalid_proxy");
|
||||
case ProxyStoreResult.WrongFrame:
|
||||
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_mismatch");
|
||||
case ProxyStoreResult.TooSmall:
|
||||
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_too_small");
|
||||
}
|
||||
// The proxy is filed: its follow-up is queued even when the sender is gone by now.
|
||||
await follow.ProxyArrivedAsync(sequence, CancellationToken.None);
|
||||
return Results.NoContent();
|
||||
}
|
||||
|
||||
private static async Task<IResult> RenderAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive)
|
||||
{
|
||||
if (await db.Renders.GetAsync(id, context.RequestAborted) is not { } render) return ApiErrors.NotFound(context, "render");
|
||||
|
||||
@@ -213,29 +213,24 @@ public static class SequenceEndpoints
|
||||
_ => ApiErrors.Conflict(context, "folder_missing"),
|
||||
};
|
||||
|
||||
/// <summary>Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5); ETag over the version counters.</summary>
|
||||
private static async Task<IResult> ManifestAsync(HttpContext context, Guid id, TitanoDatabase db)
|
||||
/// <summary>
|
||||
/// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5), with the proxies the server asks
|
||||
/// for (<see cref="ServerProxies"/>); ETag over the version counters and the number of proxies asked for.
|
||||
/// </summary>
|
||||
private static async Task<IResult> ManifestAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive)
|
||||
{
|
||||
var ct = context.RequestAborted;
|
||||
if (await db.Sequences.GetAsync(id, ct) is not { } sequence) return ApiErrors.NotFound(context, "sequence");
|
||||
var etag = string.Create(CultureInfo.InvariantCulture, $"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}\"");
|
||||
var spec = ServerProxies.Spec(archive.Current);
|
||||
var frames = await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, ct);
|
||||
var etag = string.Create(CultureInfo.InvariantCulture,
|
||||
$"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}-{frames.Count(f => f.WantsProxy)}\"");
|
||||
if (context.Request.Headers.IfNoneMatch == etag) return Results.StatusCode(StatusCodes.Status304NotModified);
|
||||
|
||||
var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
|
||||
var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt);
|
||||
var frames = new List<ManifestFrame>(sequence.FrameCount);
|
||||
var after = -1;
|
||||
while (true)
|
||||
{
|
||||
var batch = await db.Frames.ListBySequenceAsync(id, after, 1000, ct);
|
||||
frames.AddRange(batch.Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb)));
|
||||
if (batch.Count < 1000) break;
|
||||
after = batch[^1].Index;
|
||||
}
|
||||
var renders = await db.Renders.ListBySequenceAsync(id, ct);
|
||||
context.Response.Headers.ETag = etag;
|
||||
return Results.Json(new SequenceManifest(id, sequence.FramesVersion, sequence.SettingsVersion, sequence.SettingsJson, sequence.Tags, frames,
|
||||
[.. renders.Select(Dto.Render)]), ProtocolJson.Default.SequenceManifest);
|
||||
[.. renders.Select(Dto.Render)], spec), ProtocolJson.Default.SequenceManifest);
|
||||
}
|
||||
|
||||
private static async Task<IResult> FramesAsync(HttpContext context, Guid id, string? cursor, int? limit, TitanoDatabase db)
|
||||
|
||||
@@ -16,11 +16,14 @@ namespace Titano.Server.Jobs;
|
||||
/// <summary>
|
||||
/// Render nodes (BRIEF §14.7): clients with the render-node right register their encoders, heartbeat, claim queued
|
||||
/// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of
|
||||
/// the sequence. A node that stops heart-beating loses its jobs to the queue (<see cref="RenderNodeMonitor"/>).
|
||||
/// the sequence. A claimed job lists every frame with its flags (a node renders none missing, corrupt or excluded) and
|
||||
/// the proxies the server asks for (<see cref="ServerProxies"/>), which the node sends from the originals it fetched.
|
||||
/// A node that stops heart-beating loses its jobs to the queue (<see cref="RenderNodeMonitor"/>).
|
||||
/// </summary>
|
||||
public sealed class RenderNodeService(
|
||||
TitanoDatabase db,
|
||||
ServerOptions options,
|
||||
ServerArchive archive,
|
||||
UploadService uploads,
|
||||
UploadStore store,
|
||||
RenderStore renders,
|
||||
@@ -84,8 +87,9 @@ public sealed class RenderNodeService(
|
||||
continue;
|
||||
}
|
||||
await TouchAsync(node, cancellationToken);
|
||||
var spec = ServerProxies.Spec(archive.Current);
|
||||
return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson,
|
||||
parameters.UseProxies, await ManifestFramesAsync(sequence, cancellationToken));
|
||||
parameters.UseProxies, await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, cancellationToken), spec);
|
||||
}
|
||||
await TouchAsync(node, cancellationToken);
|
||||
return (ClaimedJob?)null;
|
||||
@@ -234,22 +238,6 @@ public sealed class RenderNodeService(
|
||||
if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state));
|
||||
}
|
||||
|
||||
private async Task<List<ManifestFrame>> ManifestFramesAsync(Sequence sequence, CancellationToken cancellationToken)
|
||||
{
|
||||
var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
|
||||
var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt);
|
||||
var frames = new List<ManifestFrame>(sequence.FrameCount);
|
||||
var after = -1;
|
||||
while (true)
|
||||
{
|
||||
var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, 1000, cancellationToken);
|
||||
frames.AddRange(batch.Where(f => !f.Flags.HasFlag(FrameFlags.Excluded))
|
||||
.Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb)));
|
||||
if (batch.Count < 1000) return frames;
|
||||
after = batch[^1].Index;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The node has an encoder for the job's codec that handles its resolution: the encoder's own limit, else the node's
|
||||
/// (0 = no announced limit).
|
||||
|
||||
@@ -4,21 +4,27 @@ using Titano.Data;
|
||||
namespace Titano.Server.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// Follow-up work after frames arrive (BRIEF §14.4): thumbnails, proxies and light analysis of the sequence, queued at
|
||||
/// background priority once per sequence while one is still waiting, and only for kinds this server can run.
|
||||
/// Follow-up work after frames arrive (BRIEF §14.4): thumbnails, proxies and light analysis of the sequence; after a
|
||||
/// proxy sent by a client or render node arrives, the thumbnails and the sequence's proxy state. Queued at background
|
||||
/// priority once per sequence while one is still waiting, and only for kinds this server can run.
|
||||
/// </summary>
|
||||
public sealed class SequenceJobs(TitanoDatabase db, JobEngineHost jobs)
|
||||
{
|
||||
private static readonly JobKind[] AfterNewFrames = [JobKind.Thumbs, JobKind.Proxies, JobKind.LightAnalyse];
|
||||
private static readonly JobKind[] AfterProxy = [JobKind.Thumbs, JobKind.Proxies];
|
||||
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
|
||||
public async Task FramesAddedAsync(Sequence sequence, CancellationToken cancellationToken)
|
||||
public Task FramesAddedAsync(Sequence sequence, CancellationToken cancellationToken) => QueueAsync(sequence, AfterNewFrames, cancellationToken);
|
||||
|
||||
public Task ProxyArrivedAsync(Sequence sequence, CancellationToken cancellationToken) => QueueAsync(sequence, AfterProxy, cancellationToken);
|
||||
|
||||
private async Task QueueAsync(Sequence sequence, JobKind[] kinds, CancellationToken cancellationToken)
|
||||
{
|
||||
await _gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
foreach (var kind in AfterNewFrames)
|
||||
foreach (var kind in kinds)
|
||||
{
|
||||
if (!jobs.CanRun(kind) || await db.Jobs.IsQueuedAsync(kind, sequence.Id, cancellationToken)) continue;
|
||||
await jobs.Engine.EnqueueAsync(kind, [sequence.Id], sequence.DisplayTitle, priority: JobPriority.Background, cancellationToken: cancellationToken);
|
||||
|
||||
@@ -7,8 +7,9 @@ namespace Titano.Server.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and
|
||||
/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance). Decoding is
|
||||
/// managed only (BRIEF §4.2). The daily backup of an SQLite index runs with it (<see cref="DatabaseBackupService"/>).
|
||||
/// thumbnails, its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance) and the periodic
|
||||
/// incremental scan (<see cref="ArchivePolling"/>). Decoding is managed only (BRIEF §4.2). The daily backup of an
|
||||
/// SQLite index runs with it (<see cref="DatabaseBackupService"/>).
|
||||
/// </summary>
|
||||
public static class ArchiveModule
|
||||
{
|
||||
@@ -23,5 +24,7 @@ public static class ArchiveModule
|
||||
services.AddSingleton<IJobHandlerProvider>(sp => sp.GetRequiredService<ArchiveService>());
|
||||
services.AddSingleton<IFrameAssetLocator, ArchiveAssetLocator>();
|
||||
services.AddHostedService<DatabaseBackupService>();
|
||||
services.AddSingleton<ArchivePolling>();
|
||||
services.AddHostedService(sp => sp.GetRequiredService<ArchivePolling>());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Server.Jobs;
|
||||
|
||||
namespace Titano.Server.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Polls the archive (BRIEF §7: by modification time and size, no file-system watchers): an incremental scan at start
|
||||
/// and every <see cref="Interval"/> while the archive is online, so photos already in an adopted folder and sequences
|
||||
/// another writer adds (a desktop sharing the archive, files copied in by hand) are indexed without an administrator's
|
||||
/// rescan. A scan or rebuild of the archive that is waiting, running or paused is not queued again.
|
||||
/// </summary>
|
||||
public sealed class ArchivePolling(TitanoDatabase db, ServerArchive archive, JobEngineHost jobs, TimeProvider time, ILogger<ArchivePolling> logger)
|
||||
: BackgroundService
|
||||
{
|
||||
/// <summary>The desktop's rhythm (it polls its archive the same way).</summary>
|
||||
public static readonly TimeSpan Interval = TimeSpan.FromMinutes(10);
|
||||
|
||||
private static readonly JobState[] Pending = [JobState.Queued, JobState.Running, JobState.Paused];
|
||||
|
||||
/// <summary>Queues an incremental scan of an online archive unless one is pending; returns it, or null when none was queued.</summary>
|
||||
public async Task<Job?> PollAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (!jobs.CanRun(JobKind.Scan)) return null;
|
||||
var current = archive.Current;
|
||||
if (ArchiveService.GetAvailability(current) != ArchiveAvailability.Online) return null;
|
||||
foreach (var kind in new[] { JobKind.Scan, JobKind.Rebuild })
|
||||
{
|
||||
foreach (var state in Pending)
|
||||
{
|
||||
if ((await db.Jobs.ListAsync(state, kind, 50, null, cancellationToken)).Items.Any(j => j.TargetIds.Contains(current.Id))) return null;
|
||||
}
|
||||
}
|
||||
var scan = ArchiveJobs.Scan(current);
|
||||
// Behind what users asked for (BRIEF §14.6).
|
||||
return await jobs.Engine.EnqueueAsync(scan.Kind, scan.Targets, scan.Label, scan.ParametersJson, JobPriority.Background,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
await PollAsync(stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "The archive could not be polled");
|
||||
}
|
||||
try
|
||||
{
|
||||
await Task.Delay(Interval, time, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Protocol;
|
||||
|
||||
namespace Titano.Server.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Proxies the server asks for (BRIEF §2.3, §8.5): it decodes with managed decoders only, so the frames whose originals
|
||||
/// it cannot decode (HEIF) get their proxies from a Windows client or render node, made as the archive's proxy policy
|
||||
/// says. Manifests (sync and render nodes) list every frame with its flags and whether its proxy is wanted.
|
||||
/// </summary>
|
||||
public static class ServerProxies
|
||||
{
|
||||
private const int Page = 1000;
|
||||
|
||||
/// <summary>The server takes this format's pixels from proxies made elsewhere (its decoders are managed only, BRIEF §4.2).</summary>
|
||||
public static bool CannotDecode(ImageFileFormat format) => ArchiveImages.ServerNeedsProxy(format);
|
||||
|
||||
/// <summary>How proxies of the archive are made; null when it makes none (or its manifest cannot be read now).</summary>
|
||||
public static ProxySpec? Spec(Archive archive)
|
||||
{
|
||||
try
|
||||
{
|
||||
var policy = ManifestStore.ReadArchive(new ArchivePaths(archive.RootPath).ManifestPath).ProxyPolicy;
|
||||
return policy.LongEdge == ProxyPolicy.Disabled ? null : new ProxySpec(policy.LongEdge, policy.Quality);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Every frame of <paramref name="sequence"/> in index order, as a manifest lists it: a proxy is there when the server
|
||||
/// built them all or one was sent for the frame; it is wanted for a frame the server cannot decode that has none
|
||||
/// (and is not missing), unless <paramref name="spec"/> is null.
|
||||
/// </summary>
|
||||
public static async Task<List<ManifestFrame>> FramesAsync(TitanoDatabase db, Archive archive, Sequence sequence, ProxySpec? spec,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var proxies = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
|
||||
var thumbs = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt);
|
||||
var folder = new ArchivePaths(archive.RootPath).Sequence(sequence.Folder);
|
||||
var frames = new List<ManifestFrame>(sequence.FrameCount);
|
||||
var after = -1;
|
||||
while (true)
|
||||
{
|
||||
var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, Page, cancellationToken);
|
||||
foreach (var f in batch)
|
||||
{
|
||||
var sent = !proxies && CannotDecode(f.Format) && File.Exists(folder.Proxy(f.Index));
|
||||
var wanted = spec is not null && !proxies && !sent && CannotDecode(f.Format) && (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0;
|
||||
frames.Add(new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, proxies || sent, thumbs, (long)f.Flags, wanted));
|
||||
}
|
||||
if (batch.Count < Page) return frames;
|
||||
after = batch[^1].Index;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user