fix(server): scan the archive at start and every ten minutes

The server indexed only what arrived through its own uploads, so photos
already in an adopted folder and sequences another writer added stayed
unknown until an administrator asked for a rescan. ArchivePolling, registered
by the archive module, queues an incremental scan of the online archive at
start and every ten minutes at background priority, unless a scan or rebuild
of it is waiting, running or paused (ENG-13).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 09:58:45 +00:00
1 parent c87357b02c
commit 078cac75e4
3 files changed
+136 -4

No files matched your search

+4 -2
View File
@@ -7,8 +7,8 @@ namespace Titano.Server.Storage;
/// <summary>
/// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and
/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance). Decoding is
/// managed only (BRIEF §4.2).
/// thumbnails, its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance) and the periodic
/// incremental scan (<see cref="ArchivePolling"/>). Decoding is managed only (BRIEF §4.2).
/// </summary>
public static class ArchiveModule
{
@@ -22,5 +22,7 @@ public static class ArchiveModule
services.AddSingleton<IArchiveIngest>(sp => sp.GetRequiredService<ArchiveService>().Ingest);
services.AddSingleton<IJobHandlerProvider>(sp => sp.GetRequiredService<ArchiveService>());
services.AddSingleton<IFrameAssetLocator, ArchiveAssetLocator>();
services.AddSingleton<ArchivePolling>();
services.AddHostedService(sp => sp.GetRequiredService<ArchivePolling>());
}
}
@@ -0,0 +1,68 @@
using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data;
using Titano.Data.Archiving;
using Titano.Server.Jobs;
namespace Titano.Server.Storage;
/// <summary>
/// Polls the archive (BRIEF §7: by modification time and size, no file-system watchers): an incremental scan at start
/// and every <see cref="Interval"/> while the archive is online, so photos already in an adopted folder and sequences
/// another writer adds (a desktop sharing the archive, files copied in by hand) are indexed without an administrator's
/// rescan. A scan or rebuild of the archive that is waiting, running or paused is not queued again.
/// </summary>
public sealed class ArchivePolling(TitanoDatabase db, ServerArchive archive, JobEngineHost jobs, TimeProvider time, ILogger<ArchivePolling> logger)
: BackgroundService
{
/// <summary>The desktop's rhythm (it polls its archive the same way).</summary>
public static readonly TimeSpan Interval = TimeSpan.FromMinutes(10);
private static readonly JobState[] Pending = [JobState.Queued, JobState.Running, JobState.Paused];
/// <summary>Queues an incremental scan of an online archive unless one is pending; returns it, or null when none was queued.</summary>
public async Task<Job?> PollAsync(CancellationToken cancellationToken)
{
if (!jobs.CanRun(JobKind.Scan)) return null;
var current = archive.Current;
if (ArchiveService.GetAvailability(current) != ArchiveAvailability.Online) return null;
foreach (var kind in new[] { JobKind.Scan, JobKind.Rebuild })
{
foreach (var state in Pending)
{
if ((await db.Jobs.ListAsync(state, kind, 50, null, cancellationToken)).Items.Any(j => j.TargetIds.Contains(current.Id))) return null;
}
}
var scan = ArchiveJobs.Scan(current);
// Behind what users asked for (BRIEF §14.6).
return await jobs.Engine.EnqueueAsync(scan.Kind, scan.Targets, scan.Label, scan.ParametersJson, JobPriority.Background,
cancellationToken: cancellationToken);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (true)
{
try
{
await PollAsync(stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
logger.LogWarning(ex, "The archive could not be polled");
}
try
{
await Task.Delay(Interval, time, stoppingToken);
}
catch (OperationCanceledException)
{
return;
}
}
}
}
@@ -1,6 +1,9 @@
using System.Net;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Model;
using Titano.Server.Jobs;
using Titano.Server.Storage;
using Titano.Testing;
@@ -42,11 +45,70 @@ public sealed class ArchiveModuleTests(TestContext context)
}
}
private static byte[] Photo()
[Test]
public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPolling()
{
var photo = Photo();
// An adopted folder that already holds photos.
await using var server = await TestServer.StartAsync(context, options =>
{
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo);
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService<ServerArchive>();
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
// Another writer adds a folder: the next poll queues a scan, but not while one is pending.
Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden"));
File.WriteAllBytes(Path.Combine(server.Options.ArchivePath, "garden", "IMG_0002.JPG"), Photo(seed: 3));
var polling = server.Services.GetRequiredService<ArchivePolling>();
var paused = new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.Scan,
TargetIds = [archive.Id],
State = JobState.Paused,
Created = DateTimeOffset.UtcNow,
};
await server.Database.Jobs.EnqueueAsync(paused, context.Cancellation);
Assert.Null(await polling.PollAsync(context.Cancellation));
await server.Services.GetRequiredService<JobEngineHost>().Engine.CancelAsync(paused.Id, context.Cancellation);
var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation));
Assert.Equal(JobPriority.Background, scan.Priority);
await UntilIndexedAsync(server, archive.Id, "garden");
}
private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (!(await server.Database.Sequences.ListLocationsAsync(archiveId)).Any(l => l.Folder == folder))
{
Assert.True(DateTime.UtcNow < until, $"{folder} was not indexed");
await Task.Delay(50);
}
}
/// <summary>Waits for the scan queued at start to finish: its sequences are indexed before it ends.</summary>
private static async Task UntilScannedAsync(TestServer server)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
foreach (var state in new[] { JobState.Queued, JobState.Running })
{
while ((await server.Database.Jobs.ListAsync(state, JobKind.Scan)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, "the scan at start did not finish");
await Task.Delay(50);
}
}
}
private static byte[] Photo(int seed = 0)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)(i * 7 % 251);
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
}