Merge branch 'fix/proxies-render-sync': proxies from clients and nodes, node manifests, sync and mass-rename fixes, server polling

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 11:12:25 +00:00
commit ccbd5132ee
45 files changed
+1833 -179

No files matched your search

+59 -1
View File
@@ -1,6 +1,12 @@
using System.Globalization;
using Microsoft.AspNetCore.Http.Features;
using Microsoft.AspNetCore.Mvc;
using Titano.Core.Archiving;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Model;
using Titano.Data;
using Titano.Data.Archiving;
using Titano.Server.Jobs;
using Titano.Server.Security;
using Titano.Server.Storage;
@@ -8,7 +14,8 @@ namespace Titano.Server.Api;
/// <summary>
/// Streamed archive files with Range support (BRIEF §14.3): originals, proxies, thumbnails and exports. Paths come
/// from the database by id and are resolved under the archive root only.
/// from the database by id and are resolved under the archive root only. Proxies of frames the server cannot decode
/// arrive from clients and render nodes (<c>PUT /frames/{id}/proxy</c>, BRIEF §2.3).
/// </summary>
public static class FileEndpoints
{
@@ -21,6 +28,8 @@ public static class FileEndpoints
api.MapGet("/frames/{id:guid}/original", OriginalAsync).Requires(Permissions.Edit | Permissions.RenderNode);
api.MapGet("/frames/{id:guid}/proxy", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Proxy, db, a, l))
.Requires(Permissions.Read | Permissions.RenderNode);
// Made by clients that edit and by render nodes, which decode what the server cannot.
api.MapPut("/frames/{id:guid}/proxy", StoreProxyAsync).Requires(Permissions.Edit | Permissions.RenderNode);
api.MapGet("/frames/{id:guid}/thumb", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Thumbnail, db, a, l))
.Requires(Permissions.Read);
api.MapGet("/renders/{id:guid}/file", RenderAsync).Requires(Permissions.Read);
@@ -48,6 +57,55 @@ public static class FileEndpoints
?? ApiErrors.NotFound(context, "asset_missing");
}
/// <summary>
/// Files the body (a JPEG made from the original, its Exif capture time the frame's, as large as the archive's proxies,
/// at most <see cref="ArchiveImages.MaxProxyBytes"/>) as the proxy of a frame the server cannot decode
/// (<see cref="ServerProxies"/>); the sequence's thumbnails and proxy state follow as background jobs. 204; 413
/// <c>too_large</c>; 422 <c>proxy_not_wanted</c> (the server decodes that original itself, or it is missing or
/// damaged), <c>invalid_proxy</c>, <c>proxy_mismatch</c> or <c>proxy_too_small</c>, which concern that proxy only; 409
/// <c>proxies_disabled</c>, which concerns them all; 503 <c>archive_offline</c>.
/// </summary>
private static async Task<IResult> StoreProxyAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive,
[FromServices] ArchiveService? archives, SequenceJobs follow)
{
var ct = context.RequestAborted;
if (await db.Frames.GetAsync(id, ct) is not { } frame || await db.Sequences.GetAsync(frame.SequenceId, ct) is not { } sequence)
return ApiErrors.NotFound(context, "frame");
// A proxy replaces the server's own of an original it decodes, which renders and remote views would then show.
if (!ServerProxies.CannotDecode(frame.Format) || (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0)
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_not_wanted");
if (context.Request.ContentLength is > ArchiveImages.MaxProxyBytes) return ApiErrors.Result(context, StatusCodes.Status413PayloadTooLarge, "too_large");
// Proxies are larger than the JSON bodies Kestrel is limited to; reading past this answers 413.
if (context.Features.Get<IHttpMaxRequestBodySizeFeature>() is { IsReadOnly: false } limit) limit.MaxRequestBodySize = ArchiveImages.MaxProxyBytes;
using var body = new MemoryStream();
await context.Request.Body.CopyToAsync(body, ct);
var images = (archives ?? new ArchiveService(db, new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = ArchiveModule.Owner })).Images;
ProxyStoreResult result;
try
{
result = images.StoreProxy(archive.Current, sequence, frame, body.ToArray());
}
catch (ArchiveOfflineException)
{
return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline");
}
switch (result)
{
case ProxyStoreResult.Disabled:
return ApiErrors.Conflict(context, "proxies_disabled");
case ProxyStoreResult.Invalid:
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "invalid_proxy");
case ProxyStoreResult.WrongFrame:
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_mismatch");
case ProxyStoreResult.TooSmall:
return ApiErrors.Result(context, StatusCodes.Status422UnprocessableEntity, "proxy_too_small");
}
// The proxy is filed: its follow-up is queued even when the sender is gone by now.
await follow.ProxyArrivedAsync(sequence, CancellationToken.None);
return Results.NoContent();
}
private static async Task<IResult> RenderAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive)
{
if (await db.Renders.GetAsync(id, context.RequestAborted) is not { } render) return ApiErrors.NotFound(context, "render");
+10 -15
View File
@@ -213,29 +213,24 @@ public static class SequenceEndpoints
_ => ApiErrors.Conflict(context, "folder_missing"),
};
/// <summary>Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5); ETag over the version counters.</summary>
private static async Task<IResult> ManifestAsync(HttpContext context, Guid id, TitanoDatabase db)
/// <summary>
/// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5), with the proxies the server asks
/// for (<see cref="ServerProxies"/>); ETag over the version counters and the number of proxies asked for.
/// </summary>
private static async Task<IResult> ManifestAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive)
{
var ct = context.RequestAborted;
if (await db.Sequences.GetAsync(id, ct) is not { } sequence) return ApiErrors.NotFound(context, "sequence");
var etag = string.Create(CultureInfo.InvariantCulture, $"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}\"");
var spec = ServerProxies.Spec(archive.Current);
var frames = await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, ct);
var etag = string.Create(CultureInfo.InvariantCulture,
$"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}-{frames.Count(f => f.WantsProxy)}\"");
if (context.Request.Headers.IfNoneMatch == etag) return Results.StatusCode(StatusCodes.Status304NotModified);
var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt);
var frames = new List<ManifestFrame>(sequence.FrameCount);
var after = -1;
while (true)
{
var batch = await db.Frames.ListBySequenceAsync(id, after, 1000, ct);
frames.AddRange(batch.Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb)));
if (batch.Count < 1000) break;
after = batch[^1].Index;
}
var renders = await db.Renders.ListBySequenceAsync(id, ct);
context.Response.Headers.ETag = etag;
return Results.Json(new SequenceManifest(id, sequence.FramesVersion, sequence.SettingsVersion, sequence.SettingsJson, sequence.Tags, frames,
[.. renders.Select(Dto.Render)]), ProtocolJson.Default.SequenceManifest);
[.. renders.Select(Dto.Render)], spec), ProtocolJson.Default.SequenceManifest);
}
private static async Task<IResult> FramesAsync(HttpContext context, Guid id, string? cursor, int? limit, TitanoDatabase db)
+6 -18
View File
@@ -16,11 +16,14 @@ namespace Titano.Server.Jobs;
/// <summary>
/// Render nodes (BRIEF §14.7): clients with the render-node right register their encoders, heartbeat, claim queued
/// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of
/// the sequence. A node that stops heart-beating loses its jobs to the queue (<see cref="RenderNodeMonitor"/>).
/// the sequence. A claimed job lists every frame with its flags (a node renders none missing, corrupt or excluded) and
/// the proxies the server asks for (<see cref="ServerProxies"/>), which the node sends from the originals it fetched.
/// A node that stops heart-beating loses its jobs to the queue (<see cref="RenderNodeMonitor"/>).
/// </summary>
public sealed class RenderNodeService(
TitanoDatabase db,
ServerOptions options,
ServerArchive archive,
UploadService uploads,
UploadStore store,
RenderStore renders,
@@ -84,8 +87,9 @@ public sealed class RenderNodeService(
continue;
}
await TouchAsync(node, cancellationToken);
var spec = ServerProxies.Spec(archive.Current);
return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson,
parameters.UseProxies, await ManifestFramesAsync(sequence, cancellationToken));
parameters.UseProxies, await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, cancellationToken), spec);
}
await TouchAsync(node, cancellationToken);
return (ClaimedJob?)null;
@@ -234,22 +238,6 @@ public sealed class RenderNodeService(
if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state));
}
private async Task<List<ManifestFrame>> ManifestFramesAsync(Sequence sequence, CancellationToken cancellationToken)
{
var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt);
var frames = new List<ManifestFrame>(sequence.FrameCount);
var after = -1;
while (true)
{
var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, 1000, cancellationToken);
frames.AddRange(batch.Where(f => !f.Flags.HasFlag(FrameFlags.Excluded))
.Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb)));
if (batch.Count < 1000) return frames;
after = batch[^1].Index;
}
}
/// <summary>
/// The node has an encoder for the job's codec that handles its resolution: the encoder's own limit, else the node's
/// (0 = no announced limit).
+10 -4
View File
@@ -4,21 +4,27 @@ using Titano.Data;
namespace Titano.Server.Jobs;
/// <summary>
/// Follow-up work after frames arrive (BRIEF §14.4): thumbnails, proxies and light analysis of the sequence, queued at
/// background priority once per sequence while one is still waiting, and only for kinds this server can run.
/// Follow-up work after frames arrive (BRIEF §14.4): thumbnails, proxies and light analysis of the sequence; after a
/// proxy sent by a client or render node arrives, the thumbnails and the sequence's proxy state. Queued at background
/// priority once per sequence while one is still waiting, and only for kinds this server can run.
/// </summary>
public sealed class SequenceJobs(TitanoDatabase db, JobEngineHost jobs)
{
private static readonly JobKind[] AfterNewFrames = [JobKind.Thumbs, JobKind.Proxies, JobKind.LightAnalyse];
private static readonly JobKind[] AfterProxy = [JobKind.Thumbs, JobKind.Proxies];
private readonly SemaphoreSlim _gate = new(1, 1);
public async Task FramesAddedAsync(Sequence sequence, CancellationToken cancellationToken)
public Task FramesAddedAsync(Sequence sequence, CancellationToken cancellationToken) => QueueAsync(sequence, AfterNewFrames, cancellationToken);
public Task ProxyArrivedAsync(Sequence sequence, CancellationToken cancellationToken) => QueueAsync(sequence, AfterProxy, cancellationToken);
private async Task QueueAsync(Sequence sequence, JobKind[] kinds, CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken);
try
{
foreach (var kind in AfterNewFrames)
foreach (var kind in kinds)
{
if (!jobs.CanRun(kind) || await db.Jobs.IsQueuedAsync(kind, sequence.Id, cancellationToken)) continue;
await jobs.Engine.EnqueueAsync(kind, [sequence.Id], sequence.DisplayTitle, priority: JobPriority.Background, cancellationToken: cancellationToken);
+5 -2
View File
@@ -7,8 +7,9 @@ namespace Titano.Server.Storage;
/// <summary>
/// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and
/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance). Decoding is
/// managed only (BRIEF §4.2). The daily backup of an SQLite index runs with it (<see cref="DatabaseBackupService"/>).
/// thumbnails, its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance) and the periodic
/// incremental scan (<see cref="ArchivePolling"/>). Decoding is managed only (BRIEF §4.2). The daily backup of an
/// SQLite index runs with it (<see cref="DatabaseBackupService"/>).
/// </summary>
public static class ArchiveModule
{
@@ -23,5 +24,7 @@ public static class ArchiveModule
services.AddSingleton<IJobHandlerProvider>(sp => sp.GetRequiredService<ArchiveService>());
services.AddSingleton<IFrameAssetLocator, ArchiveAssetLocator>();
services.AddHostedService<DatabaseBackupService>();
services.AddSingleton<ArchivePolling>();
services.AddHostedService(sp => sp.GetRequiredService<ArchivePolling>());
}
}
@@ -0,0 +1,68 @@
using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data;
using Titano.Data.Archiving;
using Titano.Server.Jobs;
namespace Titano.Server.Storage;
/// <summary>
/// Polls the archive (BRIEF §7: by modification time and size, no file-system watchers): an incremental scan at start
/// and every <see cref="Interval"/> while the archive is online, so photos already in an adopted folder and sequences
/// another writer adds (a desktop sharing the archive, files copied in by hand) are indexed without an administrator's
/// rescan. A scan or rebuild of the archive that is waiting, running or paused is not queued again.
/// </summary>
public sealed class ArchivePolling(TitanoDatabase db, ServerArchive archive, JobEngineHost jobs, TimeProvider time, ILogger<ArchivePolling> logger)
: BackgroundService
{
/// <summary>The desktop's rhythm (it polls its archive the same way).</summary>
public static readonly TimeSpan Interval = TimeSpan.FromMinutes(10);
private static readonly JobState[] Pending = [JobState.Queued, JobState.Running, JobState.Paused];
/// <summary>Queues an incremental scan of an online archive unless one is pending; returns it, or null when none was queued.</summary>
public async Task<Job?> PollAsync(CancellationToken cancellationToken)
{
if (!jobs.CanRun(JobKind.Scan)) return null;
var current = archive.Current;
if (ArchiveService.GetAvailability(current) != ArchiveAvailability.Online) return null;
foreach (var kind in new[] { JobKind.Scan, JobKind.Rebuild })
{
foreach (var state in Pending)
{
if ((await db.Jobs.ListAsync(state, kind, 50, null, cancellationToken)).Items.Any(j => j.TargetIds.Contains(current.Id))) return null;
}
}
var scan = ArchiveJobs.Scan(current);
// Behind what users asked for (BRIEF §14.6).
return await jobs.Engine.EnqueueAsync(scan.Kind, scan.Targets, scan.Label, scan.ParametersJson, JobPriority.Background,
cancellationToken: cancellationToken);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (true)
{
try
{
await PollAsync(stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
logger.LogWarning(ex, "The archive could not be polled");
}
try
{
await Task.Delay(Interval, time, stoppingToken);
}
catch (OperationCanceledException)
{
return;
}
}
}
}
@@ -0,0 +1,62 @@
using Titano.Core.Archiving;
using Titano.Core.Imaging;
using Titano.Core.Model;
using Titano.Data;
using Titano.Data.Archiving;
using Titano.Protocol;
namespace Titano.Server.Storage;
/// <summary>
/// Proxies the server asks for (BRIEF §2.3, §8.5): it decodes with managed decoders only, so the frames whose originals
/// it cannot decode (HEIF) get their proxies from a Windows client or render node, made as the archive's proxy policy
/// says. Manifests (sync and render nodes) list every frame with its flags and whether its proxy is wanted.
/// </summary>
public static class ServerProxies
{
private const int Page = 1000;
/// <summary>The server takes this format's pixels from proxies made elsewhere (its decoders are managed only, BRIEF §4.2).</summary>
public static bool CannotDecode(ImageFileFormat format) => ArchiveImages.ServerNeedsProxy(format);
/// <summary>How proxies of the archive are made; null when it makes none (or its manifest cannot be read now).</summary>
public static ProxySpec? Spec(Archive archive)
{
try
{
var policy = ManifestStore.ReadArchive(new ArchivePaths(archive.RootPath).ManifestPath).ProxyPolicy;
return policy.LongEdge == ProxyPolicy.Disabled ? null : new ProxySpec(policy.LongEdge, policy.Quality);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
{
return null;
}
}
/// <summary>
/// Every frame of <paramref name="sequence"/> in index order, as a manifest lists it: a proxy is there when the server
/// built them all or one was sent for the frame; it is wanted for a frame the server cannot decode that has none
/// (and is not missing), unless <paramref name="spec"/> is null.
/// </summary>
public static async Task<List<ManifestFrame>> FramesAsync(TitanoDatabase db, Archive archive, Sequence sequence, ProxySpec? spec,
CancellationToken cancellationToken)
{
var proxies = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt);
var thumbs = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt);
var folder = new ArchivePaths(archive.RootPath).Sequence(sequence.Folder);
var frames = new List<ManifestFrame>(sequence.FrameCount);
var after = -1;
while (true)
{
var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, Page, cancellationToken);
foreach (var f in batch)
{
var sent = !proxies && CannotDecode(f.Format) && File.Exists(folder.Proxy(f.Index));
var wanted = spec is not null && !proxies && !sent && CannotDecode(f.Format) && (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0;
frames.Add(new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, proxies || sent, thumbs, (long)f.Flags, wanted));
}
if (batch.Count < Page) return frames;
after = batch[^1].Index;
}
}
}