fix(analysis): follow-ups skip queued sequences; photos added by a scan are analysed again
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
6 files changed
+75
-10
No files matched your search
@@ -160,7 +160,11 @@ public sealed class ArchiveIndexer
|
||||
if (reconcile || File.Exists(sequence.IngestJournalPath))
|
||||
{
|
||||
var before = manifest;
|
||||
manifest = await _writer.UpdateAsync(paths, manifest.Id, folder.RelativeFolder, m => SequenceAdopter.Reconcile(sequence, m, cancellationToken),
|
||||
// Photos added, gone or back: the light analysis no longer describes the frames, so the scan's follow-up runs it again.
|
||||
manifest = await _writer.UpdateAsync(paths, manifest.Id, folder.RelativeFolder,
|
||||
m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled
|
||||
? reconciled with { Status = reconciled.Status & ~SequenceStatus.AnalyzedLight }
|
||||
: null,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
if (!ReferenceEquals(before, manifest)) bytes = null;
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Scan),
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Rebuild),
|
||||
new VerifyJobHandler(_db, Verifier),
|
||||
new ImportJobHandler(Imports),
|
||||
new ImportJobHandler(_db, Imports),
|
||||
new ImageJobHandler(_db, Images, JobKind.Thumbs),
|
||||
new ImageJobHandler(_db, Images, JobKind.Proxies),
|
||||
new MaintenanceJobHandler(_db, Maintenance),
|
||||
|
||||
@@ -13,7 +13,7 @@ namespace Titano.Data.Archiving.Jobs;
|
||||
/// number of plan items done. A removed card fails the job at a batch boundary; resuming it after re-insertion
|
||||
/// continues there. The sequences the import went into are then queued for light analysis.
|
||||
/// </summary>
|
||||
internal sealed class ImportJobHandler(ImportService imports) : IJobHandler
|
||||
internal sealed class ImportJobHandler(TitanoDatabase db, ImportService imports) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => JobKind.Import;
|
||||
|
||||
@@ -30,6 +30,6 @@ internal sealed class ImportJobHandler(ImportService imports) : IJobHandler
|
||||
.ConfigureAwait(false);
|
||||
}, p => context.ReportAsync(p.BytesDone, p.BytesTotal, p.Current), context.Cancellation).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Archive.Log.ImportDone", final.Copied, final.Skipped, final.Failed));
|
||||
await AnalysisFollowUp.QueueAsync(context, await imports.SequencesOfAsync(parameters, context.Cancellation).ConfigureAwait(false)).ConfigureAwait(false);
|
||||
await AnalysisFollowUp.QueueAsync(db, context, await imports.SequencesOfAsync(parameters, context.Cancellation).ConfigureAwait(false)).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,8 @@ internal static class JobArchive
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Scan"/> (incremental, or full with <see cref="ScanParameters.Full"/>) and
|
||||
/// <see cref="JobKind.Rebuild"/>. Checkpoint: <c>after:<folder></c>, the last folder indexed. When anything was
|
||||
/// indexed, the sequences never light-analysed (adopted folders, new sidecars) are queued for it.
|
||||
/// indexed, the sequences without a current light analysis (adopted folders, new sidecars, photos added to a folder) are
|
||||
/// queued for it.
|
||||
/// </summary>
|
||||
internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler
|
||||
{
|
||||
@@ -41,7 +42,7 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer,
|
||||
context.Log(Loc.L("Archive.Log.ScanDone", summary.Folders, summary.Indexed, summary.Adopted, summary.Removed));
|
||||
// A resumed scan may have indexed its folders before the interruption.
|
||||
if (summary.Indexed > 0 || resume is not null)
|
||||
await AnalysisFollowUp.QueueAsync(context, await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false))
|
||||
await AnalysisFollowUp.QueueAsync(db, context, await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false))
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -7,19 +7,24 @@ namespace Titano.Data.Processing;
|
||||
/// <summary>
|
||||
/// Light analysis queued after work that brings sequences or frames into the index (card import, adoption, scan), so the
|
||||
/// advisor's Auto values and recommended preset exist without any user action (BRIEF §1.1, §4.3). Nothing is queued
|
||||
/// when the engine cannot run light analysis.
|
||||
/// when the engine cannot run light analysis, and a sequence already waiting in a queued light analysis is not queued
|
||||
/// again (that job runs after the changes that led here).
|
||||
/// </summary>
|
||||
internal static class AnalysisFollowUp
|
||||
{
|
||||
/// <summary>Queues one light analysis of <paramref name="sequenceIds"/> (labelled like the job that found them).</summary>
|
||||
public static async Task QueueAsync(JobContext context, IReadOnlyCollection<Guid> sequenceIds)
|
||||
public static async Task QueueAsync(TitanoDatabase db, JobContext context, IReadOnlyCollection<Guid> sequenceIds)
|
||||
{
|
||||
if (sequenceIds.Count == 0) return;
|
||||
var request = ProcessingJobs.LightAnalysis([.. sequenceIds], context.Job.TargetLabel);
|
||||
var waiting = (await db.Jobs.ListQueuedAsync([JobKind.LightAnalyse], 1000, context.Cancellation).ConfigureAwait(false))
|
||||
.SelectMany(j => j.TargetIds).ToHashSet();
|
||||
var targets = sequenceIds.Where(id => !waiting.Contains(id)).Distinct().ToList();
|
||||
if (targets.Count == 0) return;
|
||||
var request = ProcessingJobs.LightAnalysis(targets, context.Job.TargetLabel);
|
||||
await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Sequences of <paramref name="archiveId"/> that were never light-analysed (new or adopted ones).</summary>
|
||||
/// <summary>Sequences of <paramref name="archiveId"/> without a current light analysis (new, adopted, or with frames a scan added).</summary>
|
||||
public static async Task<List<Guid>> UnanalysedAsync(TitanoDatabase db, Guid archiveId, CancellationToken cancellationToken)
|
||||
{
|
||||
var ids = new List<Guid>();
|
||||
|
||||
@@ -60,6 +60,61 @@ public sealed class AnalysisFollowUpTests(TestContext context)
|
||||
Assert.True((await fx.Db.Sequences.GetAsync(sequenceId.Value, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedLight), "analysed");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task PhotosAddedToAnAnalysedFolderAreAnalysedAgain()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof");
|
||||
await fx.StartEngineAsync(withArchive: true);
|
||||
var first = await fx.EnqueueAsync(Titano.Data.Processing.ProcessingJobs.LightAnalysis([sequence.Id]));
|
||||
await fx.WaitForAsync(first.Id, JobState.Done);
|
||||
|
||||
// The camera kept shooting into the folder: the next scan indexes the new photos and analyses the sequence again.
|
||||
fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 18 }), "Roof", first: 12);
|
||||
var scan = await fx.EnqueueAsync(ArchiveJobs.Scan(fx.Archive.Archive));
|
||||
await fx.WaitForAsync(scan.Id, JobState.Done);
|
||||
var again = (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first.Id);
|
||||
await fx.WaitForAsync(again.Id, JobState.Done);
|
||||
|
||||
var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal(18, analysed.FrameCount);
|
||||
Assert.True(analysed.Status.HasFlag(SequenceStatus.AnalyzedLight), "analysed again");
|
||||
Assert.True(ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ASequenceWaitingForLightAnalysisIsNotQueuedAgain()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof");
|
||||
// Queued before the scan, which runs first (user priority) and finds the sequence not analysed yet.
|
||||
var waiting = await QueueAsync(fx, Titano.Data.Processing.ProcessingJobs.LightAnalysis([sequence.Id]));
|
||||
var scan = await QueueAsync(fx, ArchiveJobs.Scan(fx.Archive.Archive, full: true));
|
||||
await fx.StartEngineAsync(withArchive: true);
|
||||
|
||||
await fx.WaitForAsync(scan.Id, JobState.Done);
|
||||
await fx.WaitForAsync(waiting.Id, JobState.Done);
|
||||
Assert.Equal(waiting.Id, (await SingleLightJobAsync(fx))?.Id);
|
||||
}
|
||||
|
||||
/// <summary>Queues a job before the engine starts.</summary>
|
||||
private static async Task<Job> QueueAsync(ProcessingFixture fx, Titano.Data.Archiving.Jobs.ArchiveJobRequest request)
|
||||
{
|
||||
var job = new Job
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Kind = request.Kind,
|
||||
TargetIds = request.Targets,
|
||||
TargetLabel = request.Label,
|
||||
ParametersJson = request.ParametersJson,
|
||||
Priority = request.Priority,
|
||||
State = JobState.Queued,
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
};
|
||||
await fx.Db.Jobs.EnqueueAsync(job, fx.Ct);
|
||||
return job;
|
||||
}
|
||||
|
||||
private static async Task<Job?> SingleLightJobAsync(ProcessingFixture fx) =>
|
||||
(await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.SingleOrDefault();
|
||||
}
|
||||
Reference in new issue
Block a user