fix(archive): scans leave sequences waiting for a folder alone and build server images

A sequence created through POST /api/v1/sequences keeps an empty folder until its
first frame. Scans no longer adopt loose photos at the archive root under its id,
and a rebuild, the orphans report and fix, verify and the scan follow-ups no longer
treat it as the root sequence, as missing or as a sequence to analyse. All of them
use ArchiveIngest.AwaitsFolder.

On the server (ArchiveServiceOptions.ImagesAfterScan), a scan that indexes sequences
without thumbnails or proxies queues background Thumbs and Proxies passes for them,
as an upload does. A scan that finds photos added or removed clears ThumbsBuilt and
ProxiesBuilt together with AnalyzedLight.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 15:58:37 +00:00
1 parent 32a0e9bbc1
commit fba05c2840
10 files changed
+290 -29

No files matched your search

+42 -14
View File
@@ -20,7 +20,9 @@ public enum ScanMode
public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed);
/// <param name="Failed">Folders that could not be indexed, with the reason; they are retried by the next scan.</param>
public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed);
/// <param name="Sequences">The sequences this run indexed (<paramref name="Indexed"/> of them), in folder order.</param>
public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed,
IReadOnlyList<Guid> Sequences);
/// <summary>
/// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts
@@ -49,11 +51,15 @@ public sealed class ArchiveIndexer
if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken);
var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var locations = new Locations(await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false));
var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
// A sequence waiting for its first frame has no folder yet: the archive root, adopted or re-adopted, never takes its id.
var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false);
var locations = new Locations(rows.Where(r => !awaiting.Contains(r.Id)));
var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null;
using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null;
int indexed = 0, adopted = 0;
int adopted = 0;
var indexed = new List<Guid>();
var failed = new List<(string, string)>();
for (var i = 0; i < folders.Count; i++)
{
@@ -67,14 +73,14 @@ public sealed class ArchiveIndexer
try
{
var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos);
var bytes = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
if (snapshotWriter is not null)
{
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value,
bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
}
indexed++;
indexed.Add(id);
if (folder.Sidecar is null) adopted++;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
@@ -88,7 +94,7 @@ public sealed class ArchiveIndexer
var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0;
if (failed.Count == 0) snapshotWriter?.Commit();
return new ScanSummary(folders.Count, indexed, adopted, removed, failed);
return new ScanSummary(folders.Count, indexed.Count, adopted, removed, failed, indexed);
}
/// <summary>
@@ -156,8 +162,8 @@ public sealed class ArchiveIndexer
}
}
/// <summary>Indexes one folder; returns the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
private async Task<byte[]?> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
/// <summary>Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
Locations locations, CancellationToken cancellationToken)
{
var sequence = paths.Sequence(folder.RelativeFolder);
@@ -189,10 +195,11 @@ public sealed class ArchiveIndexer
}
if (reconcile || File.Exists(sequence.IngestJournalPath))
{
// Photos added, gone or back: the light analysis no longer describes the frames, so the scan's follow-up runs it again.
// Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, so the scan's
// follow-ups make them again (the images on a server, ArchiveServiceOptions.ImagesAfterScan).
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id,
m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled
? reconciled with { Status = reconciled.Status & ~SequenceStatus.AnalyzedLight }
? reconciled with { Status = reconciled.Status & ~(SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt) }
: null,
cancellationToken).ConfigureAwait(false);
bytes = null;
@@ -208,7 +215,23 @@ public sealed class ArchiveIndexer
await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
}
locations.Set(manifest.Id, folder.RelativeFolder);
return bytes;
return (manifest.Id, bytes);
}
/// <summary>
/// The sequences among <paramref name="locations"/> created without photos that still wait for their first frame
/// (<see cref="ArchiveIngest.AwaitsFolder"/>). Their empty folder is not the archive root: scans do not adopt the root
/// under their id, a rebuild keeps them although they have no sidecar yet, and orphan checks do not count them missing.
/// </summary>
internal async Task<HashSet<Guid>> AwaitingFolderAsync(ArchivePaths paths, IEnumerable<SequenceLocation> locations, CancellationToken cancellationToken)
{
var awaiting = new HashSet<Guid>();
foreach (var location in locations.Where(l => l.Folder.Length == 0))
{
if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is { } sequence && ArchiveIngest.AwaitsFolder(sequence, paths))
awaiting.Add(sequence.Id);
}
return awaiting;
}
/// <summary>The index holds a project for this sequence and other frames than the sidecar: its positions may move.</summary>
@@ -261,16 +284,21 @@ public sealed class ArchiveIndexer
UpdatedAt = DateTimeOffset.UtcNow,
};
/// <summary>Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched).</summary>
/// <summary>
/// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence
/// waiting for its first frame has no sidecar yet and stays.
/// </summary>
private async Task<int> RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList<SequenceFolder> folders, Locations locations,
CancellationToken cancellationToken)
{
ArchiveVolume.EnsureWritable(paths);
var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal);
var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false);
var removed = 0;
foreach (var location in await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
foreach (var location in rows)
{
if (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id) continue;
if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue;
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
}
@@ -71,7 +71,10 @@ public sealed class ArchiveMaintenance
{
var paths = new ArchivePaths(archive.RootPath);
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
// A sequence waiting for its first frame has no folder yet: none of it is missing, and the archive root is not its folder.
var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false);
var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList();
var sequences = new List<SequenceOrphans>();
var missing = new List<SequenceLocation>();
foreach (var location in locations)
@@ -91,7 +94,10 @@ public sealed class ArchiveMaintenance
return new OrphanReport(sequences, missing, unindexed);
}
/// <summary>Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept.</summary>
/// <summary>
/// Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept, and
/// so are sequences waiting for their first frame (<see cref="ArchiveIngest.AwaitsFolder"/>), which have none yet.
/// </summary>
public async Task<int> RemoveMissingSequencesAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
@@ -100,7 +106,7 @@ public sealed class ArchiveMaintenance
foreach (var id in sequenceIds)
{
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue;
if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath)) continue;
if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath) || ArchiveIngest.AwaitsFolder(sequence, paths)) continue;
if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++;
await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false);
}
+2 -2
View File
@@ -54,8 +54,8 @@ public sealed class ArchiveService : IJobHandlerProvider
/// <summary>Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the <see cref="JobEngine"/>.</summary>
public IReadOnlyList<IJobHandler> CreateJobHandlers() =>
[
new ScanJobHandler(_db, Indexer, JobKind.Scan),
new ScanJobHandler(_db, Indexer, JobKind.Rebuild),
new ScanJobHandler(_db, Indexer, JobKind.Scan, _writer.Options.ImagesAfterScan),
new ScanJobHandler(_db, Indexer, JobKind.Rebuild, _writer.Options.ImagesAfterScan),
new VerifyJobHandler(_db, Verifier),
new ImportJobHandler(_db, Imports),
new ImageJobHandler(_db, Images, JobKind.Thumbs),
+5 -3
View File
@@ -44,7 +44,7 @@ public sealed class ArchiveVerifier
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var targets = await TargetsAsync(archive, sequenceIds, cancellationToken).ConfigureAwait(false);
var targets = await TargetsAsync(archive, paths, sequenceIds, cancellationToken).ConfigureAwait(false);
var bytesTotal = targets.Sum(t => t.TotalBytes);
var bytesDone = targets.Take(resume.Sequence).Sum(t => t.TotalBytes);
long checkedFrames = 0, ok = 0, hashed = 0;
@@ -82,13 +82,15 @@ public sealed class ArchiveVerifier
return new VerifyReport(targets.Count, checkedFrames, ok, hashed, problems);
}
private async Task<List<Sequence>> TargetsAsync(Archive archive, IReadOnlyList<Guid>? sequenceIds, CancellationToken cancellationToken)
/// <summary>The sequences to check; one waiting for its first frame (<see cref="ArchiveIngest.AwaitsFolder"/>) has no sidecar to check yet.</summary>
private async Task<List<Sequence>> TargetsAsync(Archive archive, ArchivePaths paths, IReadOnlyList<Guid>? sequenceIds, CancellationToken cancellationToken)
{
var ids = sequenceIds ?? (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id).ToList();
var targets = new List<Sequence>(ids.Count);
foreach (var id in ids)
{
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id) targets.Add(s);
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id && !ArchiveIngest.AwaitsFolder(s, paths))
targets.Add(s);
}
return targets;
}
@@ -1,7 +1,9 @@
using Titano.Core.Archiving;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
using Titano.Data.Processing;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving.Jobs;
@@ -18,9 +20,12 @@ internal static class JobArchive
/// <see cref="JobKind.Scan"/> (incremental, or full with <see cref="ScanParameters.Full"/>) and
/// <see cref="JobKind.Rebuild"/>. Checkpoint: <c>after:&lt;folder&gt;</c>, the last folder indexed. When anything was
/// indexed, the sequences without a current light analysis (adopted folders, new sidecars, photos added to a folder) are
/// queued for it, and for a background deep analysis too when <see cref="ScanParameters.DeepAnalysis"/> asks.
/// queued for it, and for a background deep analysis too when <see cref="ScanParameters.DeepAnalysis"/> asks. On the
/// server (<paramref name="imagesAfterScan"/>, <see cref="ArchiveServiceOptions.ImagesAfterScan"/>) the sequences it indexed
/// without thumbnails or proxies are queued for them too, as after an upload. Sequences waiting for their first frame
/// (<see cref="ArchiveIngest.AwaitsFolder"/>) have nothing to analyse or draw yet and are left out.
/// </summary>
internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler
internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind, bool imagesAfterScan = false) : IJobHandler
{
private const string Prefix = "after:";
@@ -42,9 +47,47 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer,
// A resumed scan may have indexed its folders before the interruption.
if (summary.Indexed > 0 || resume is not null)
{
var unanalysed = await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false);
var awaiting = await indexer.AwaitingFolderAsync(new ArchivePaths(archive.RootPath),
await db.Sequences.ListLocationsAsync(archive.Id, context.Cancellation).ConfigureAwait(false), context.Cancellation).ConfigureAwait(false);
var unanalysed = (await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false))
.Where(id => !awaiting.Contains(id)).ToList();
await AnalysisFollowUp.QueueAsync(db, context, unanalysed).ConfigureAwait(false);
if (kind == JobKind.Scan && parameters.DeepAnalysis) await AnalysisFollowUp.QueueDeepAsync(db, context, unanalysed).ConfigureAwait(false);
if (imagesAfterScan)
{
var resumed = resume is not null;
await QueueImagesAsync(context, archive, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, summary.Sequences, resumed, awaiting).ConfigureAwait(false);
// An archive whose policy makes no proxies would get a pass that does nothing.
if (archive.ProxyLongEdge != ProxyPolicy.Disabled)
await QueueImagesAsync(context, archive, JobKind.Proxies, SequenceStatus.ProxiesBuilt, summary.Sequences, resumed, awaiting).ConfigureAwait(false);
}
}
}
/// <summary>
/// Queues one background <paramref name="pass"/> (thumbnails or proxies) of the sequences this run indexed whose
/// <paramref name="built"/> flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the
/// flag then). After a resumed run, which may have indexed sequences before the interruption, every sequence of the
/// archive without the flag but those in <paramref name="awaiting"/>. Sequences already waiting in a queued pass of that
/// kind are left out.
/// </summary>
private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList<Guid> indexed,
bool resumed, IReadOnlySet<Guid> awaiting)
{
var missing = new HashSet<Guid>();
var query = new SequenceQuery { ArchiveId = archive.Id, WithoutStatus = built, Sort = SequenceSort.Oldest, Limit = 1000 };
do
{
var page = await db.Sequences.ListAsync(query, context.Cancellation).ConfigureAwait(false);
missing.UnionWith(page.Items.Select(s => s.Id));
query = query with { Cursor = page.NextCursor };
}
while (query.Cursor is not null);
missing.ExceptWith(awaiting);
var waiting = (await db.Jobs.ListQueuedAsync([pass], 1000, context.Cancellation).ConfigureAwait(false)).SelectMany(j => j.TargetIds).ToHashSet();
var targets = (resumed ? missing : indexed.Where(missing.Contains)).Where(id => !waiting.Contains(id)).Distinct().ToList();
if (targets.Count == 0) return;
var request = pass == JobKind.Thumbs ? ArchiveJobs.Thumbnails(targets, context.Job.TargetLabel) : ArchiveJobs.Proxies(targets, context.Job.TargetLabel);
await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false);
}
}
@@ -17,6 +17,13 @@ public sealed record ArchiveServiceOptions
public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2);
public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration;
/// <summary>
/// Scans queue thumbnails and proxies of the sequences they index without them, as an upload does (the server, whose
/// clients load both from it: BRIEF §4.3, §14.4). The desktop decodes its thumbnails on demand and makes proxies when
/// asked.
/// </summary>
public bool ImagesAfterScan { get; init; }
}
/// <summary>
+4 -3
View File
@@ -8,8 +8,9 @@ namespace Titano.Server.Storage;
/// <summary>
/// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and
/// thumbnails, its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance) and the periodic
/// incremental scan (<see cref="ArchivePolling"/>). Decoding is managed only (BRIEF §4.2). The daily backup of an
/// SQLite index runs with it (<see cref="DatabaseBackupService"/>).
/// incremental scan (<see cref="ArchivePolling"/>); a scan queues thumbnails and proxies of what it indexes, as an upload
/// does. Decoding is managed only (BRIEF §4.2). The daily backup of an SQLite index runs with it
/// (<see cref="DatabaseBackupService"/>).
/// </summary>
public static class ArchiveModule
{
@@ -19,7 +20,7 @@ public static class ArchiveModule
public static void Register(IServiceCollection services)
{
services.AddSingleton(sp => new ArchiveService(sp.GetRequiredService<TitanoDatabase>(),
new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = Owner }));
new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = Owner, ImagesAfterScan = true }));
services.AddSingleton<IArchiveIngest>(sp => sp.GetRequiredService<ArchiveService>().Ingest);
services.AddSingleton<IJobHandlerProvider>(sp => sp.GetRequiredService<ArchiveService>());
services.AddSingleton<IFrameAssetLocator, ArchiveAssetLocator>();
@@ -17,6 +17,24 @@ public sealed class ArchiveIngestTests(TestContext context)
Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") =>
new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null);
/// <summary>A sequence as <c>POST /api/v1/sequences</c> creates it: a row without a folder or frames.</summary>
private static async Task<Sequence> CreateWithoutPhotosAsync(ArchiveFixture fx, string title)
{
var created = new Sequence
{
Id = Guid.CreateVersion7(),
ArchiveId = fx.Archive.Id,
Title = title,
Folder = string.Empty,
Rating = 4,
Origin = SequenceOrigin.Server,
OwnerUserId = Guid.CreateVersion7(),
UpdatedAt = DateTimeOffset.UtcNow,
};
await fx.Db.Sequences.UpsertAsync(created, fx.Ct);
return created;
}
[Test]
public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates()
{
@@ -242,6 +260,80 @@ public sealed class ArchiveIngestTests(TestContext context)
Assert.FileExists(staged.Path);
}
[Test]
public async Task ASequenceWaitingForItsFirstFrameNeverLendsItsIdToTheArchiveRoot()
{
await using var fx = await CreateAsync(context);
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
// Loose photos at the root, which a scan adopts in place.
Photos.WriteSeries(fx.Archive.RootPath, Start.AddDays(-1), 2);
await fx.ScanAsync();
var root = ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath);
Assert.NotEqual(waiting.Id, root.Id);
var rootRow = (await fx.Db.Sequences.GetAsync(root.Id))!;
Assert.Equal((string.Empty, 2), (rootRow.Folder, rootRow.FrameCount));
var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
Assert.Equal((string.Empty, 0, "Harbour sunset"), (row.Folder, row.FrameCount, row.Title));
// Its first frame still gets a folder of its own, and the root keeps its photos.
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), sequenceId: waiting.Id), fx.Ct);
Assert.Equal(waiting.Id, first.SequenceId);
var placed = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
Assert.Equal(("2026/2026-09-28 Canon EOS R5", 1), (placed.Folder, placed.FrameCount));
Assert.Equal(root.Id, ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath).Id);
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(root.Id))!.FrameCount);
}
[Test]
public async Task ARootSequenceWhoseSidecarIsGoneIsAdoptedAgainUnderItsOwnId()
{
await using var fx = await CreateAsync(context);
Photos.WriteSeries(fx.Archive.RootPath, Start, 2);
await fx.ScanAsync();
var adopted = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault());
// Created later, so its id sorts after the root sequence's.
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
File.Delete(fx.Paths.Sequence(string.Empty).ManifestPath);
await fx.ScanAsync();
Assert.Equal(adopted.Id, ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath).Id);
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(adopted.Id))!.FrameCount);
var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
Assert.Equal((string.Empty, 0), (row.Folder, row.FrameCount));
}
[Test]
public async Task RebuildOrphansAndVerifyLeaveASequenceWaitingForItsFirstFrame()
{
await using var fx = await CreateAsync(context);
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2);
await fx.ScanAsync();
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
// It has no sidecar yet, which is no reason to drop it.
var rebuilt = await fx.ScanAsync(ScanMode.Rebuild);
Assert.Equal(0, rebuilt.Removed);
var orphans = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
Assert.Empty(orphans.MissingSequences);
Assert.Empty(orphans.UnindexedFolders);
Assert.Equal(0, await fx.Service.RemoveMissingSequencesAsync(fx.Archive.Id, [waiting.Id], fx.Ct));
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
Assert.Equal((1, 2L), (report.Sequences, report.Checked));
var kept = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id));
Assert.Equal(("Harbour sunset", 4), (kept.Title, kept.Rating));
// The archive root is not its folder: loose photos there are a folder the index does not know yet.
Photos.WriteSeries(fx.Archive.RootPath, Start.AddDays(-1), 1, firstSeed: 50);
Assert.SequenceEqual([string.Empty], (await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).UnindexedFolders);
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct);
Assert.Equal(waiting.Id, first.SequenceId);
Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder);
}
[Test]
public async Task FramesInsertedBeforeOthersTakeTheProjectPositionsAlong()
{
@@ -3,6 +3,7 @@ using Titano.Core.Import;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
using Titano.Data.Repositories;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
@@ -40,6 +41,9 @@ public sealed class ArchiveJobTests(TestContext context)
var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(rebuild.Id, JobState.Done);
Assert.Equal(2, (await LocationsAsync(fx)).Count);
// The desktop decodes its thumbnails on demand and makes proxies when asked.
Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
}
[Test]
@@ -88,6 +92,56 @@ public sealed class ArchiveJobTests(TestContext context)
Assert.Contains("made 3, reused 3", done.LogTail);
}
[Test]
public async Task AServerScanQueuesThumbnailsAndProxiesOfTheSequencesItIndexesWithoutThem()
{
await using var fx = await CreateAsync(context);
Populate(fx);
// The server's archive engine (ArchiveModule); the desktop's scans queue no image passes.
var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true });
await using var engine = new JobEngine(fx.Db, service.CreateJobHandlers(),
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync(fx.Ct);
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(scan.Id, JobState.Done);
var locations = await LocationsAsync(fx);
var first = new Dictionary<JobKind, Guid>();
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single();
Assert.SequenceEqual(locations.Select(l => l.Id), pass.TargetIds);
Assert.Equal(JobPriority.Background, pass.Priority);
await fx.WaitForAsync(pass.Id, JobState.Done);
first[kind] = pass.Id;
}
foreach (var location in locations) Assert.True(Built((await fx.Db.Sequences.GetAsync(location.Id))!.Status), location.Folder);
// Every folder indexed again, but none without its images: nothing is queued.
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
await fx.WaitForAsync(full.Id, JobState.Done);
Assert.Count(1, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
Assert.Count(1, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
// A photo added to a folder: its sequence lacks the images of that frame until the passes queued for it alone ran.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(5), 1, firstSeed: 90, firstNumber: 7);
var added = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(added.Id, JobState.Done);
var a = locations.Single(l => l.Folder == "a");
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first[kind]);
Assert.SequenceEqual([a.Id], pass.TargetIds);
await fx.WaitForAsync(pass.Id, JobState.Done);
}
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built again");
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
Assert.Count(7, pack.Entries);
Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies));
static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt);
}
[Test]
public async Task AThumbnailPassFlagsFramesNothingDecodesAndClearsThemOnceTheyDecode()
{
@@ -68,7 +68,7 @@ public sealed class ArchiveModuleTests(TestContext context)
}
[Test]
public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPolling()
public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPollingWithThumbnailsAndProxies()
{
var photo = Photo();
// An adopted folder that already holds photos.
@@ -78,8 +78,11 @@ public sealed class ArchiveModuleTests(TestContext context)
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo);
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService<ServerArchive>();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
// The scan queues the thumbnails and proxies clients load from the server, as an upload does.
await UntilImagesAsync(server, editor, archive.Id, "roof");
// Another writer adds a folder: the next poll queues a scan, but not while one is pending.
Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden"));
@@ -100,6 +103,7 @@ public sealed class ArchiveModuleTests(TestContext context)
var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation));
Assert.Equal(JobPriority.Background, scan.Priority);
await UntilIndexedAsync(server, archive.Id, "garden");
await UntilImagesAsync(server, editor, archive.Id, "garden");
}
private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder)
@@ -112,6 +116,30 @@ public sealed class ArchiveModuleTests(TestContext context)
}
}
/// <summary>Waits until the first frame of the sequence in <paramref name="folder"/> has its thumbnail and proxy served.</summary>
private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder)
{
var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder);
var frame = (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).First();
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
foreach (var asset in new[] { "thumb", "proxy" })
{
while (true)
{
using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
if (response.StatusCode == HttpStatusCode.OK)
{
Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {folder}");
break;
}
Assert.True(DateTime.UtcNow < until, $"no {asset} of {folder}: {(int)response.StatusCode}");
await Task.Delay(100);
}
}
var built = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status;
Assert.True(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt), $"{folder}: {built}");
}
/// <summary>Waits for the scan queued at start to finish: its sequences are indexed before it ends.</summary>
private static async Task UntilScannedAsync(TestServer server)
{