diff --git a/src/Titano.App/Sections/ArchiveViewModel.cs b/src/Titano.App/Sections/ArchiveViewModel.cs
index 30ab9f78..89253226 100644
--- a/src/Titano.App/Sections/ArchiveViewModel.cs
+++ b/src/Titano.App/Sections/ArchiveViewModel.cs
@@ -1001,7 +1001,9 @@ public sealed class ArchiveViewModel : SectionViewModel
switch (row.Kind)
{
case OrphanKind.UnlistedFiles or OrphanKind.MissingFiles when row.SequenceId is { } id:
- await Task.Run(() => archives.ReconcileAsync(id));
+ // The photos it indexes are analysed in the background, as after a scan.
+ var (followUps, deep) = (Services.Engine, Services.Settings.DeepAnalysisAfterScan);
+ await Task.Run(() => archives.ReconcileAsync(id, followUps, deep));
break;
case OrphanKind.MissingSequence when row.SequenceId is { } id:
var confirmed = Services.Dialogs.Confirm(new ConfirmRequest
diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs
index 67cf0b84..d0619248 100644
--- a/src/Titano.Data/Archiving/ArchiveIndexer.cs
+++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs
@@ -105,9 +105,10 @@ public sealed class ArchiveIndexer
///
/// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames
/// inserted before indexed ones take the project's frame positions along (). The frames
- /// version never goes back: a sidecar read under the lease that is behind the locked row (an older copy put back by
- /// hand or by a file-sync tool) is written again one version past the row, so editors and clients holding the row's
- /// version see the change.
+ /// version never goes back, nor names other frames than it did: a sidecar read under the lease that is behind the
+ /// locked row (an older copy put back by hand or by a file-sync tool), or at the row's version over other frames
+ /// (another index of the archive bumped a copy put back to it, ), is written again one
+ /// version past the row, so editors and clients holding the row's version see the change.
///
/// The folder as the scan listed it before reading without the sequence
/// lease; null when the caller holds the lease and read the sidecar under it.
@@ -131,7 +132,11 @@ public sealed class ArchiveIndexer
if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar
|| (existing is not null && existing.FramesVersion != manifest.Versions.Frames)))
return null;
- if (existing is not null && existing.FramesVersion > manifest.Versions.Frames)
+ // Under the lease, the row's own version may also name other frames: another index of the archive bumped a copy
+ // put back to it (OtherFramesAsync). A caller without the lease compared them before its read (FramesMovedAsync).
+ if (existing is not null && (existing.FramesVersion > manifest.Versions.Frames
+ || (listed is null && existing.FramesVersion == manifest.Versions.Frames
+ && await OtherFramesAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false))))
{
manifest = manifest with
{
@@ -220,16 +225,39 @@ public sealed class ArchiveIndexer
(await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos
!= FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence));
- /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).
- public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
+ ///
+ /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action), as a scan's reconcile
+ /// does (): what covered the frames is cleared in the sidecar and the row.
+ ///
+ /// Whether photos were added, gone or back: the light analysis, thumbnails and proxies are then to be made again
+ /// (). The folder's stamps are recorded, so no scan comes back to it and
+ /// queues them.
+ public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
- return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken)
- .ConfigureAwait(false);
+ var changed = false;
+ await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m =>
+ {
+ var reconciled = Reconcile(folder, m, cancellationToken);
+ changed = reconciled is not null;
+ return reconciled;
+ }, cancellationToken).ConfigureAwait(false);
+ return changed;
}
+ ///
+ /// The sidecar brought in line with the folder's photos (); null when nothing
+ /// changed. Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the
+ /// sidecar then tells every host, so the follow-ups make them again (the images on a server,
+ /// ).
+ ///
+ private static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
+ SequenceAdopter.Reconcile(sequence, manifest, cancellationToken) is { } reconciled
+ ? reconciled with { Status = reconciled.Status & ~CoveringStatus }
+ : null;
+
///
/// Applies to the sidecar (null writes nothing) and indexes the result, both under the sequence
/// lease. Frames inserted before indexed ones move the project's positions () without a
@@ -297,14 +325,9 @@ public sealed class ArchiveIndexer
}
if (reconcile || File.Exists(sequence.IngestJournalPath))
{
- // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the
- // sidecar then tells every host, so the scan's follow-ups make them again (the images on a server,
- // ArchiveServiceOptions.ImagesAfterScan).
- manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id,
- m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled
- ? reconciled with { Status = reconciled.Status & ~CoveringStatus }
- : null,
- cancellationToken).ConfigureAwait(false);
+ // Photos added, gone or back clear what covered the frames, and the scan's follow-ups make it again.
+ manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => Reconcile(sequence, m, cancellationToken), cancellationToken)
+ .ConfigureAwait(false);
bytes = null;
}
else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false))
@@ -369,7 +392,23 @@ public sealed class ArchiveIndexer
///
private async Task FramesMovedAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
- && indexed.FramesVersion != manifest.Versions.Frames;
+ && await OtherFramesAsync(_db, indexed, manifest, cancellationToken).ConfigureAwait(false);
+
+ ///
+ /// Whether the index row holds other frames than : another frames
+ /// version, or the same version over other frames (ids, or their order). Each index of a shared archive bumps a sidecar
+ /// put back behind its own row one version past that row (), and another index may hold that
+ /// very version for the frames as they were before: that index still sees the change, and bumps past it in turn.
+ ///
+ private static async Task OtherFramesAsync(TitanoRepositories db, Sequence indexed, SequenceManifest manifest, CancellationToken cancellationToken)
+ {
+ if (indexed.FramesVersion != manifest.Versions.Frames) return true;
+ var rows = await db.Frames.ListPositionsAsync(indexed.Id, cancellationToken).ConfigureAwait(false);
+ if (rows.Count != manifest.Frames.Count) return true;
+ var listed = new Dictionary(manifest.Frames.Count);
+ foreach (var frame in manifest.Frames) listed[frame.I] = frame.Id;
+ return rows.Any(row => !listed.TryGetValue(row.Index, out var id) || id != row.FrameId);
+ }
/// A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.
private async Task AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs
index b76809a2..94969be5 100644
--- a/src/Titano.Data/Archiving/ArchiveService.cs
+++ b/src/Titano.Data/Archiving/ArchiveService.cs
@@ -6,6 +6,7 @@ using Titano.Core.Model;
using Titano.Core.Naming;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
+using Titano.Data.Processing;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
@@ -284,11 +285,35 @@ public sealed class ArchiveService : IJobHandlerProvider
public async Task FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
- /// Orphans fix: index the unlisted photos of a sequence and flag its missing files.
- public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default)
+ ///
+ /// Orphans fix: index the unlisted photos of a sequence and flag its missing files. Photos added, gone or back clear
+ /// what covered the frames, as a scan's reconcile does (), and what a scan's
+ /// follow-ups would make again is queued on (null: nothing): the light analysis, a
+ /// background deep analysis when asks (as
+ /// does for a scan) and, on a server (), the thumbnails and proxies.
+ /// A kind the engine does not run, or one a queued job of that kind already covers for the sequence, is left out.
+ ///
+ public async Task ReconcileAsync(Guid sequenceId, JobEngine? followUps, bool deepAnalysis, CancellationToken cancellationToken = default)
{
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
- await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
+ if (!await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false) || followUps is null) return;
+ List requests = [ProcessingJobs.LightAnalysis([sequenceId], sequence.DisplayTitle)];
+ // BRIEF §10.2: background, low priority.
+ if (deepAnalysis) requests.Add(ProcessingJobs.DeepAnalysis([sequenceId], sequence.DisplayTitle) with { Priority = JobPriority.Background });
+ if (_writer.Options.ImagesAfterScan)
+ {
+ requests.Add(ArchiveJobs.Thumbnails([sequenceId], sequence.DisplayTitle));
+ // An archive whose policy makes no proxies would get a pass that does nothing.
+ if (archive.ProxyLongEdge != ProxyPolicy.Disabled) requests.Add(ArchiveJobs.Proxies([sequenceId], sequence.DisplayTitle));
+ }
+ foreach (var request in requests)
+ {
+ if (!followUps.Runs(request.Kind)) continue;
+ // A queued job runs after the fix, on the frames as they are then, whatever else it targets (a scan's follow-ups).
+ var queued = await _db.Jobs.ListQueuedAsync(request.Kind, 1000, cancellationToken).ConfigureAwait(false);
+ if (queued.Any(j => j.TargetIds.Contains(sequenceId))) continue;
+ await followUps.EnqueueAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false);
+ }
}
/// Orphans fix: drop index rows whose sidecar is gone.
diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs
index 5cc5994f..47593854 100644
--- a/src/Titano.Data/Processing/SequenceAnalyser.cs
+++ b/src/Titano.Data/Processing/SequenceAnalyser.cs
@@ -45,9 +45,9 @@ internal enum FramesChange
/// Auto values of the project. The light pass reads the index, probes the decode tier (§8.2) of frames whose tier is not
/// known yet (headers only, once per frame) and reads one file for the GPS fix; the deep pass decodes every frame,
/// checkpointing into .titano/analysis.partial so a cancelled, paused or crashed run resumes, records the tier
-/// each frame decoded at (or that it did not decode) and saves analysis.json atomically. Results land in the
-/// sidecar (status, tiers), the index (status, warning count, compact summary, tier flags, per-frame light statistics)
-/// and project.json.
+/// each frame decoded at (or that it did not decode) and saves analysis.json atomically under the sequence lock.
+/// Results land in the sidecar (status, tiers), the index (status, warning count, compact summary, tier flags,
+/// per-frame light statistics) and project.json.
///
internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions options, SequenceWriter writer, ArchiveIndexer indexer, ProjectStore projects)
{
@@ -94,17 +94,23 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
// the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it
// left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records
// nothing (FramesChangeAsync), and a light analysis of the frames as they are runs instead, here or where they
- // were changed.
+ // were changed; its report is saved only over none of the frames as they are (SaveReport). What a deep pass records
+ // then follows the report analysis.json holds: its own, or the one it left in place.
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false);
+ var saved = deep && SaveReport(folder, report!, change == FramesChange.None && manifest.Versions.Frames == sequence.FramesVersion,
+ manifest.Frames.Count);
if (change != FramesChange.None)
return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], change);
if (deep)
{
+ // A report left in place is read as a light analysis reads it, so the deep status, summary, warnings and
+ // Auto values recorded here describe the file that readers and the report endpoint open.
var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
- if (!Describes(report!, now)) report = null;
+ if (!saved) report = ReadReport(folder, now);
+ else if (!Describes(report!, now)) report = null;
}
else
{
@@ -247,10 +253,30 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
return tiers;
}
+ ///
+ /// Saves a deep pass's report as analysis.json, under the sequence lock, then drops its checkpoint; returns
+ /// whether it saved it. The report replaces the saved one when it measured the frames as they are
+ /// (), or when the saved one describes them no better: none, unreadable, or of another frame
+ /// count than , the sidecar's. A pass overtaken by an upload, or by another process
+ /// sharing the archive, would otherwise replace the report a later pass made of the frames as they are, and the next
+ /// light analysis would drop that one's deep status.
+ ///
+ private static bool SaveReport(SequencePaths folder, DeepAnalysisReport report, bool current, int frameCount)
+ {
+ var save = current || ReadReport(folder, frameCount) is null;
+ if (save) report.Save(folder.AnalysisPath);
+ Checkpoint(folder).Delete();
+ return save;
+ }
+
+ /// The deep pass's checkpoint, .titano/analysis.partial: kept until the pass has saved its report.
+ private static DeepAnalysisCheckpointFile Checkpoint(SequencePaths folder) =>
+ new(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
+
private async Task<(DeepAnalysisReport Report, Dictionary Decoded)> AnalyseDeepAsync(Archive archive, SequenceFrames frames,
IReadOnlyList metadata, SequencePaths folder, IProgress? progress, CancellationToken cancellationToken)
{
- var file = new DeepAnalysisCheckpointFile(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
+ var file = Checkpoint(folder);
// The host's analysis parameters as they are now (Settings › Analysis on the desktop). Those that change per-frame
// results are part of the checkpoint's fingerprint, so a partial run made with other values starts again.
var analysis = options.Current.DeepAnalysis;
@@ -263,11 +289,10 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
.ConfigureAwait(false);
decoded = Decoded(frames, report, source, companions, cancellationToken);
}
- // An unreadable verdict on a frame read from its original goes stale once the frame has a proxy (ReadReport).
+ // An unreadable verdict on a frame read from its original goes stale once the frame has a proxy (ReadReport). The
+ // report is saved under the sequence lock, once the frames it measured are known to be current (SaveReport).
int[] fromProxies = [.. Enumerable.Range(0, frames.Count).Where(i => frames.Proxied?[i] == true)];
report = report with { ProxyFrames = fromProxies.Length > 0 ? fromProxies : null };
- report.Save(folder.AnalysisPath);
- file.Delete();
var columns = report.Frames;
var stats = new FrameLightStats[frames.Count];
diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs
index 7eb69fe2..bf947800 100644
--- a/src/Titano.Data/Repositories/SequenceRepository.cs
+++ b/src/Titano.Data/Repositories/SequenceRepository.cs
@@ -500,11 +500,22 @@ public sealed class SequenceRepository
await session.ScalarAsync($"SELECT id FROM sequences WHERE id = $1{SqlDialect.For(session.Dialect).ForUpdate}", [id], cancellationToken)
.ConfigureAwait(false) is not null;
- /// Replaces the tag links of a sequence whose row the caller has locked (or just upserted).
+ ///
+ /// Replaces the tag links of a sequence whose row the caller has locked (or just upserted); a set the sequence already
+ /// carries (names compared case-insensitively) is left as it is. On PostgreSQL every tag written is locked until
+ /// commit, and the tags are written in the order of their keys whatever order the caller listed them in (a PATCH sends
+ /// them in any order): writers of two sequences that share tags then take those rows in one order, and never wait for
+ /// each other's (PostgreSQL would abort one with a deadlock, and nothing retries it).
+ ///
private async Task ReplaceTagsAsync(IDbSession session, Guid sequenceId, IEnumerable tags, CancellationToken cancellationToken)
{
+ var wanted = TagRepository.NormalizeSet(tags);
+ var keys = wanted.Select(TagRepository.Key).ToList();
+ var stored = await session.QueryListAsync("SELECT t.name_key FROM sequence_tags st JOIN tags t ON t.id = st.tag_id WHERE st.sequence_id = $1",
+ [sequenceId], r => r.GetString(0), cancellationToken).ConfigureAwait(false);
+ if (stored.Count == keys.Count && stored.ToHashSet(StringComparer.Ordinal).SetEquals(keys)) return;
await session.ExecuteAsync("DELETE FROM sequence_tags WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
- foreach (var tag in TagRepository.NormalizeSet(tags))
+ foreach (var tag in wanted.OrderBy(TagRepository.Key, StringComparer.Ordinal))
{
DbParam[] args = [Guid.CreateVersion7(), tag, TagRepository.Key(tag)];
Guid tagId;
diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs
index e73e5444..d8a4ec4d 100644
--- a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs
+++ b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs
@@ -1,6 +1,7 @@
using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data.Archiving;
+using Titano.Data.Jobs;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
@@ -112,7 +113,7 @@ public sealed class ArchiveIntegrityTests(TestContext context)
Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id);
Assert.SequenceEqual(["New folder"], report.UnindexedFolders);
- await fx.Service.ReconcileAsync(id, fx.Ct);
+ await fx.Service.ReconcileAsync(id, null, deepAnalysis: false, fx.Ct);
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.Equal(4, frames.Count);
Assert.Equal(FrameFlags.Missing, frames[0].Flags);
@@ -120,6 +121,38 @@ public sealed class ArchiveIntegrityTests(TestContext context)
Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0));
}
+ [Test]
+ public async Task TheOrphansFixOnAServerDrawsTheFramesItIndexes()
+ {
+ await using var fx = await CreateAsync(context);
+ var id = await IngestSeriesAsync(fx, "a", 3, 1);
+ // The archive's server, which draws what its clients load; its engine runs no analysis.
+ var server = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "server", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true });
+ await using var engine = new JobEngine(fx.Db, server.CreateJobHandlers(),
+ new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
+ await engine.StartAsync(fx.Ct);
+ foreach (var pass in new[] { ArchiveJobs.Thumbnails([id]), ArchiveJobs.Proxies([id]) })
+ await fx.WaitForAsync((await engine.EnqueueAsync(pass, cancellationToken: fx.Ct)).Id, JobState.Done);
+ var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id, fx.Ct))!.Folder);
+ File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50));
+
+ // The fix indexes the photo copied in by hand, and queues the thumbnails and proxies of the frames as they are now;
+ // no analysis, even when asked for one.
+ var before = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Select(j => j.Id).ToHashSet();
+ await server.ReconcileAsync(id, engine, deepAnalysis: true, fx.Ct);
+ var queued = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Where(j => !before.Contains(j.Id)).ToList();
+ Assert.SequenceEqual([JobKind.Proxies, JobKind.Thumbs], queued.Select(j => j.Kind).Order());
+ foreach (var job in queued)
+ {
+ Assert.SequenceEqual([id], job.TargetIds);
+ await fx.WaitForAsync(job.Id, JobState.Done);
+ }
+ var row = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
+ Assert.Equal(4, row.FrameCount);
+ Assert.True(row.Status.HasFlag(SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt), row.Status.ToString());
+ Assert.True(File.Exists(folder.Proxy(3)), "proxy of the new frame");
+ }
+
[Test]
public async Task MaintenanceRemovesOnlyStaleLeftovers()
{
diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
index 0d0dee07..5014e921 100644
--- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
+++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
@@ -508,6 +508,57 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
Assert.Empty((await fx.ScanAsync()).Sequences);
}
+ [Test]
+ public async Task AFramesVersionAnotherIndexReachedForOtherFramesIsIndexedPastIt()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var folder = fx.Paths.Sequence("a");
+ async Task UploadAsync(string name, int minute)
+ {
+ var staged = fx.Stage(Start.AddMinutes(minute), seed: 90 + minute);
+ await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, name, staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+ return (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ }
+ await UploadAsync("IMG_0004.JPG", 1);
+ // A desktop sharing this server's archive, with its own index of the four photos.
+ await using var desktopDb = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(Context.CreateTempDirectory("desktop"), "titano.db")), fx.Ct);
+ var desktop = new ArchiveService(desktopDb, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
+ var desktopArchive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
+ Task DesktopScanAsync() => desktop.Indexer.ScanAsync(desktopArchive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
+ await DesktopScanAsync();
+
+ // The server takes a fifth photo before the desktop scans again.
+ var uploaded = await UploadAsync("IMG_0005.JPG", 2);
+ Assert.Equal(uploaded.FramesVersion - 1, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion);
+
+ // The sidecar is damaged; the desktop's scan rebuilds it from the photos at the first version, the fifth photo under
+ // an id of its own, and writes it one version past its own row: the version the server holds for the photo as it
+ // filed it.
+ await File.WriteAllTextAsync(folder.ManifestPath, "{", fx.Ct);
+ Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences);
+ var rebuilt = ManifestStore.ReadSequence(folder.ManifestPath);
+ Assert.Equal(uploaded.FramesVersion, rebuilt.Versions.Frames);
+ Assert.NotEqual((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[^1].Id, rebuilt.Frames[^1].Id);
+
+ // The server's scan finds other frames at the version it holds: it indexes them one version past it, in both places,
+ // so an editor holding that version for the photo as it was is refused.
+ Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
+ var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal(uploaded.FramesVersion + 1, indexed.FramesVersion);
+ Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
+ Assert.SequenceEqual(rebuilt.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
+ var edit = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Rating = 2 }, null, uploaded.FramesVersion, fx.Ct);
+ Assert.Equal(SequenceEditOutcome.Conflict, edit.Outcome);
+
+ // The desktop follows, and neither index finds anything more to do.
+ Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences);
+ Assert.Equal(indexed.FramesVersion, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion);
+ Assert.Empty((await fx.ScanAsync()).Sequences);
+ Assert.Empty((await DesktopScanAsync()).Sequences);
+ }
+
[Test]
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
{
diff --git a/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs
index 627a70ed..74acf7d3 100644
--- a/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs
+++ b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs
@@ -82,6 +82,81 @@ public sealed class AnalysisFollowUpTests(TestContext context)
Assert.True(ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
}
+ [Test]
+ public async Task PhotosTheOrphansFixIndexesAreAnalysedAgain()
+ {
+ await using var fx = await ProcessingFixture.CreateAsync(context);
+ var sequence = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof");
+ var folder = fx.Folder(sequence);
+ await fx.StartEngineAsync(withArchive: true);
+ var first = await fx.EnqueueAsync(Titano.Data.Processing.ProcessingJobs.LightAnalysis([sequence.Id]));
+ await fx.WaitForAsync(first.Id, JobState.Done);
+ // A server sharing the archive drew the twelve frames and made their proxies.
+ const SequenceStatus images = SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
+ var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
+ ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = manifest.Status | images });
+ await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, images, SequenceStatus.None, fx.Ct);
+
+ // Photos copied into the folder by hand: the Archive screen's orphans check lists them, and its fix indexes them. The
+ // light analysis it queues waits on the first new photo's probe, so what the fix left can be seen.
+ fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 18 }), "Roof", first: 12);
+ fx.Decoder.HoldProbeOf(12);
+ await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, deepAnalysis: false, fx.Ct);
+ const SequenceStatus covering = SequenceStatus.AnalyzedLight | images;
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal((18, SequenceStatus.None), (row.FrameCount, row.Status & covering));
+ Assert.Equal(SequenceStatus.None, ManifestStore.ReadSequence(folder.ManifestPath).Status & covering);
+ var again = (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first.Id);
+ Assert.SequenceEqual([sequence.Id], again.TargetIds);
+ Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.DeepAnalyse, cancellationToken: fx.Ct)).Items);
+
+ await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
+ fx.Decoder.Release();
+ await fx.WaitForAsync(again.Id, JobState.Done);
+ row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "analysed again");
+ Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
+ Assert.Contains("18", ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).Timing.FrameRate.ReasonArgs!);
+ }
+
+ [Test]
+ public async Task PhotosTheOrphansFixIndexesGetTheDeepAnalysisAScanWouldQueue()
+ {
+ await using var fx = await ProcessingFixture.CreateAsync(context);
+ var sequence = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof");
+ var lake = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Lake");
+ var folder = fx.Folder(sequence);
+ await fx.StartEngineAsync(withArchive: true);
+ Task> QueuedDeepAsync() => fx.Db.Jobs.ListQueuedAsync(JobKind.DeepAnalyse, 50, fx.Ct);
+
+ // The host asks for a deep analysis after scans: the orphans fix queues one in the background too, behind the light
+ // analysis, which waits on the first new photo's probe so the queue can be seen.
+ fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 18 }), "Roof", first: 12);
+ fx.Decoder.HoldProbeOf(12);
+ await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, deepAnalysis: true, fx.Ct);
+ var light = (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.Single();
+ var deep = (await QueuedDeepAsync()).Single();
+ Assert.SequenceEqual([sequence.Id], deep.TargetIds);
+ Assert.Equal(JobPriority.Background, deep.Priority);
+ await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
+
+ // The user cancels it, and asks for a deep analysis of both sequences instead. More photos and the fix again: the
+ // one waiting covers them, so no other is queued.
+ await fx.Engine.CancelAsync(deep.Id, fx.Ct);
+ var both = await fx.EnqueueAsync(Titano.Data.Processing.ProcessingJobs.DeepAnalysis([lake.Id, sequence.Id]));
+ fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 20 }), "Roof", first: 18);
+ await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, deepAnalysis: true, fx.Ct);
+ Assert.SequenceEqual([both.Id], (await QueuedDeepAsync()).Select(j => j.Id));
+
+ fx.Decoder.Release();
+ await fx.WaitForAsync(light.Id, JobState.Done);
+ await fx.WaitForAsync(both.Id, JobState.Done);
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal(20, row.FrameCount);
+ Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
+ Assert.Equal(20, Titano.Data.Processing.SequenceAnalyser.ReadReport(folder, 20)?.FrameCount);
+ }
+
[Test]
public async Task AScanAskedForDeepAnalysisQueuesItInTheBackgroundForNewSequencesOnly()
{
diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs
index 93262ac4..fc26bb1e 100644
--- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs
+++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs
@@ -292,6 +292,89 @@ public sealed class AnalysisJobTests(TestContext context)
Assert.Equal(advice, await ProjectJsonAsync(fx, folder));
}
+ [Test]
+ public async Task ADeepAnalysisOvertakenByOneOfTheFramesAsTheyAreLeavesThatOnesReport()
+ {
+ await using var fx = await ProcessingFixture.CreateAsync(context);
+ var sequence = await fx.AddAsync(Flickering(12));
+ var folder = fx.Folder(sequence);
+ await fx.StartEngineAsync(workers: 2);
+
+ // A deep analysis starts on twelve frames; while it runs, a thirteenth photo is indexed and a second deep analysis,
+ // on the other worker, analyses all thirteen and records its report.
+ fx.Decoder.HoldFrame(0);
+ var overtaken = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
+ await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
+ fx.Decoder.Hold = null;
+ await fx.AddAsync(Flickering(13), first: 12);
+ var current = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
+ await fx.WaitForAsync(current.Id, JobState.Done);
+ var report = await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct);
+ Assert.Equal(13, SequenceAnalyser.ReadReport(folder, 13)!.FrameCount);
+ var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.True(analysed.Status.HasFlag(SequenceStatus.AnalyzedDeep), analysed.Status.ToString());
+ fx.Decoder.Release();
+ var done = await fx.WaitForAsync(overtaken.Id, JobState.Done);
+
+ // The first pass measured twelve: it leaves the report of the thirteen in place, so the light analysis it queues
+ // keeps the deep status and its summary.
+ Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail);
+ Assert.Equal(report, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct));
+ Assert.FileMissing(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
+ var light = Assert.Contains(await LightJobsAsync(fx), j => j.TargetIds.Contains(sequence.Id));
+ await fx.WaitForAsync(light.Id, JobState.Done);
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
+ Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status");
+ Assert.Equal(analysed.AnalysisJson, row.AnalysisJson);
+ }
+
+ [Test]
+ public async Task ADeepAnalysisThatLeavesTheReportInPlaceRecordsThatReport()
+ {
+ await using var fx = await ProcessingFixture.CreateAsync(context);
+ var sequence = await fx.AddAsync(Flickering(12));
+ var folder = fx.Folder(sequence);
+ await fx.StartEngineAsync();
+ var first = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
+ await fx.WaitForAsync(first.Id, JobState.Done);
+ var measured = SequenceAnalyser.ReadReport(folder, 12)!;
+
+ // A deep analysis starts on the twelve frames on the only worker. Meanwhile another Titano sharing the archive files
+ // them again, at the next frames version, and saves its own analysis of them; this index's scan indexes them, and
+ // their light analysis waits in the queue.
+ fx.Decoder.HoldFrame(0);
+ var overtaken = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
+ await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
+ fx.Decoder.Hold = null;
+ var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
+ ManifestStore.WriteSequence(folder.ManifestPath, manifest with
+ {
+ Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 },
+ UpdatedAt = DateTimeOffset.UtcNow,
+ });
+ Assert.Contains(sequence.Id, (await fx.Archive.ScanAsync()).Sequences);
+ var theirs = measured with { HeroIndex = measured.HeroIndex == 5 ? 6 : 5 };
+ theirs.Save(folder.AnalysisPath);
+ var report = await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct);
+ var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
+ fx.Decoder.Release();
+ await fx.WaitForAsync(overtaken.Id, JobState.Done);
+
+ // The pass leaves that report in place, and records it rather than its own: the summary in the index is the one of
+ // the file every reader opens, and the light analysis that follows keeps it.
+ Assert.Equal(report, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct));
+ var summary = AnalysisSummary.From(theirs);
+ Assert.NotEqual(AnalysisSummary.From(measured), summary);
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
+ Assert.Equal(summary, AnalysisSummary.FromJson(row.AnalysisJson!));
+ await fx.WaitForAsync(light.Id, JobState.Done);
+ row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
+ Assert.Equal(summary, AnalysisSummary.FromJson(row.AnalysisJson!));
+ }
+
[Test]
public async Task AnAnalysisOvertakenWhileALightAnalysisWaitsRecordsAndThatOneReplacesIt()
{
diff --git a/tests/Titano.Data.Tests/Repositories/SequenceEditTests.cs b/tests/Titano.Data.Tests/Repositories/SequenceEditTests.cs
index bab97147..c923dd47 100644
--- a/tests/Titano.Data.Tests/Repositories/SequenceEditTests.cs
+++ b/tests/Titano.Data.Tests/Repositories/SequenceEditTests.cs
@@ -101,6 +101,51 @@ public abstract class SequenceEditTests(TestContext context) : RepositoryTest(co
var later = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Rating = 3 }, stored.SettingsVersion, Ct);
Assert.Equal((sequence.SettingsVersion + 1, sequence.FramesVersion + 1, 3), (later.Sequence!.SettingsVersion, later.Sequence.FramesVersion, later.Sequence.Rating));
}
+
+ [Test]
+ public async Task WritersOfSequencesSharingTagsListedInOppositeOrdersBothComplete()
+ {
+ var db = await OpenTitanoAsync(multiConnection: true);
+ var archive = TestData.Archive();
+ await db.Archives.UpsertAsync(archive, Ct);
+ var tagged = TestData.Sequence(archive.Id, title: "Tagged");
+ var first = TestData.Sequence(archive.Id, title: "First");
+ var second = TestData.Sequence(archive.Id, title: "Second");
+ await db.Sequences.UpsertAsync(tagged with { Tags = ["Alps", "sunset"] }, Ct);
+ await db.Sequences.UpsertAsync(first, Ct);
+ await db.Sequences.UpsertAsync(second, Ct);
+ var alps = (await db.Tags.ListAsync(Ct)).Single(t => t.Name == "Alps");
+
+ // A transaction holds the first of the two tags (SQLite: its write lock), so both writers below wait until it ends.
+ Task forward, backward;
+ await using (var tx = await db.BeginAsync(Ct))
+ {
+ if (Dialect == Dialect.Postgres)
+ Assert.Equal(alps.Id, await tx.Session.ScalarAsync("SELECT id FROM tags WHERE id = $1 FOR UPDATE", [alps.Id], Ct));
+ else
+ Assert.True(await tx.Tags.RenameAsync(alps.Id, "Alps", Ct));
+ forward = db.Sequences.EditAsync(first.Id, new SequenceEdit { Tags = ["Alps", "sunset"] }, null, Ct);
+ await Task.WhenAny(forward, Task.Delay(300, Ct));
+ // The other writer lists the tags the other way round (a PATCH sends them in any order).
+ backward = db.Sequences.EditAsync(second.Id, new SequenceEdit { Tags = ["SUNSET", "alps"] }, null, Ct);
+ await Task.WhenAny(backward, Task.Delay(300, Ct));
+ Assert.False(forward.IsCompleted || backward.IsCompleted, "both writers wait for the tag");
+ if (Dialect == Dialect.Postgres)
+ {
+ // A writer whose tags are already the stored ones takes no tag at all.
+ var unchanged = db.Sequences.EditAsync(tagged.Id, new SequenceEdit { Tags = ["sunset", "ALPS"], Rating = 2 }, null, Ct);
+ Assert.Equal(unchanged, await Task.WhenAny(unchanged, Task.Delay(TimeSpan.FromSeconds(20), Ct)));
+ Assert.Equal(SequenceEditOutcome.Updated, (await unchanged).Outcome);
+ }
+ await tx.CommitAsync(Ct);
+ }
+
+ // Both take the tags in one order: neither is aborted for a deadlock.
+ Assert.Equal((SequenceEditOutcome.Updated, SequenceEditOutcome.Updated), ((await forward).Outcome, (await backward).Outcome));
+ foreach (var sequence in new[] { tagged, first, second })
+ Assert.SequenceEqual(["Alps", "sunset"], (await db.Sequences.GetAsync(sequence.Id, Ct))!.Tags);
+ Assert.Count(2, await db.Tags.ListAsync(Ct));
+ }
}
public sealed class SqliteSequenceEditTests(TestContext context) : SequenceEditTests(context)