diff --git a/src/Titano.Server/Storage/ArchiveModule.cs b/src/Titano.Server/Storage/ArchiveModule.cs index 3002c9d4..29df251d 100644 --- a/src/Titano.Server/Storage/ArchiveModule.cs +++ b/src/Titano.Server/Storage/ArchiveModule.cs @@ -7,8 +7,8 @@ namespace Titano.Server.Storage; /// /// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and -/// thumbnails, and its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance). Decoding is -/// managed only (BRIEF §4.2). +/// thumbnails, its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance) and the periodic +/// incremental scan (). Decoding is managed only (BRIEF §4.2). /// public static class ArchiveModule { @@ -22,5 +22,7 @@ public static class ArchiveModule services.AddSingleton(sp => sp.GetRequiredService().Ingest); services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); } } diff --git a/src/Titano.Server/Storage/ArchivePolling.cs b/src/Titano.Server/Storage/ArchivePolling.cs new file mode 100644 index 00000000..fe5bb2e4 --- /dev/null +++ b/src/Titano.Server/Storage/ArchivePolling.cs @@ -0,0 +1,68 @@ +using Titano.Core.Archiving; +using Titano.Core.Model; +using Titano.Data; +using Titano.Data.Archiving; +using Titano.Server.Jobs; + +namespace Titano.Server.Storage; + +/// +/// Polls the archive (BRIEF §7: by modification time and size, no file-system watchers): an incremental scan at start +/// and every while the archive is online, so photos already in an adopted folder and sequences +/// another writer adds (a desktop sharing the archive, files copied in by hand) are indexed without an administrator's +/// rescan. A scan or rebuild of the archive that is waiting, running or paused is not queued again. +/// +public sealed class ArchivePolling(TitanoDatabase db, ServerArchive archive, JobEngineHost jobs, TimeProvider time, ILogger logger) + : BackgroundService +{ + /// The desktop's rhythm (it polls its archive the same way). + public static readonly TimeSpan Interval = TimeSpan.FromMinutes(10); + + private static readonly JobState[] Pending = [JobState.Queued, JobState.Running, JobState.Paused]; + + /// Queues an incremental scan of an online archive unless one is pending; returns it, or null when none was queued. + public async Task PollAsync(CancellationToken cancellationToken) + { + if (!jobs.CanRun(JobKind.Scan)) return null; + var current = archive.Current; + if (ArchiveService.GetAvailability(current) != ArchiveAvailability.Online) return null; + foreach (var kind in new[] { JobKind.Scan, JobKind.Rebuild }) + { + foreach (var state in Pending) + { + if ((await db.Jobs.ListAsync(state, kind, 50, null, cancellationToken)).Items.Any(j => j.TargetIds.Contains(current.Id))) return null; + } + } + var scan = ArchiveJobs.Scan(current); + // Behind what users asked for (BRIEF §14.6). + return await jobs.Engine.EnqueueAsync(scan.Kind, scan.Targets, scan.Label, scan.ParametersJson, JobPriority.Background, + cancellationToken: cancellationToken); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (true) + { + try + { + await PollAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + logger.LogWarning(ex, "The archive could not be polled"); + } + try + { + await Task.Delay(Interval, time, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + } + } +} diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs index 172540f8..c2ba6b44 100644 --- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs @@ -1,6 +1,9 @@ using System.Net; +using Microsoft.Extensions.DependencyInjection; using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; +using Titano.Core.Model; +using Titano.Server.Jobs; using Titano.Server.Storage; using Titano.Testing; @@ -42,11 +45,70 @@ public sealed class ArchiveModuleTests(TestContext context) } } - private static byte[] Photo() + [Test] + public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPolling() + { + var photo = Photo(); + // An adopted folder that already holds photos. + await using var server = await TestServer.StartAsync(context, options => + { + Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof")); + File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo); + }, ArchiveModule.Register); + var archive = server.Services.GetRequiredService(); + await UntilIndexedAsync(server, archive.Id, "roof"); + await UntilScannedAsync(server); + + // Another writer adds a folder: the next poll queues a scan, but not while one is pending. + Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden")); + File.WriteAllBytes(Path.Combine(server.Options.ArchivePath, "garden", "IMG_0002.JPG"), Photo(seed: 3)); + var polling = server.Services.GetRequiredService(); + var paused = new Job + { + Id = Guid.CreateVersion7(), + Kind = JobKind.Scan, + TargetIds = [archive.Id], + State = JobState.Paused, + Created = DateTimeOffset.UtcNow, + }; + await server.Database.Jobs.EnqueueAsync(paused, context.Cancellation); + Assert.Null(await polling.PollAsync(context.Cancellation)); + await server.Services.GetRequiredService().Engine.CancelAsync(paused.Id, context.Cancellation); + + var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation)); + Assert.Equal(JobPriority.Background, scan.Priority); + await UntilIndexedAsync(server, archive.Id, "garden"); + } + + private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder) + { + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!(await server.Database.Sequences.ListLocationsAsync(archiveId)).Any(l => l.Folder == folder)) + { + Assert.True(DateTime.UtcNow < until, $"{folder} was not indexed"); + await Task.Delay(50); + } + } + + /// Waits for the scan queued at start to finish: its sequences are indexed before it ends. + private static async Task UntilScannedAsync(TestServer server) + { + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + foreach (var state in new[] { JobState.Queued, JobState.Running }) + { + while ((await server.Database.Jobs.ListAsync(state, JobKind.Scan)).Items.Count > 0) + { + Assert.True(DateTime.UtcNow < until, "the scan at start did not finish"); + await Task.Delay(50); + } + } + } + + private static byte[] Photo(int seed = 0) { const int width = 96, height = 64; var pixels = new byte[width * height * 3]; - for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)(i * 7 % 251); + for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed) % 251); return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 }); } }