merge: fifth-pass data-layer fixes (Z-1..Z-4)

Merge branch fix5/data-edges into the verification branch.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 23:59:17 +00:00
commit bd9f4a2869
10 files changed
+422 -33

No files matched your search

+3 -1
View File
@@ -1001,7 +1001,9 @@ public sealed class ArchiveViewModel : SectionViewModel
switch (row.Kind)
{
case OrphanKind.UnlistedFiles or OrphanKind.MissingFiles when row.SequenceId is { } id:
await Task.Run(() => archives.ReconcileAsync(id));
// The photos it indexes are analysed in the background, as after a scan.
var (followUps, deep) = (Services.Engine, Services.Settings.DeepAnalysisAfterScan);
await Task.Run(() => archives.ReconcileAsync(id, followUps, deep));
break;
case OrphanKind.MissingSequence when row.SequenceId is { } id:
var confirmed = Services.Dialogs.Confirm(new ConfirmRequest
+56 -17
View File
@@ -105,9 +105,10 @@ public sealed class ArchiveIndexer
/// <summary>
/// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames
/// inserted before indexed ones take the project's frame positions along (<see cref="ProjectPositions"/>). The frames
/// version never goes back: a sidecar read under the lease that is behind the locked row (an older copy put back by
/// hand or by a file-sync tool) is written again one version past the row, so editors and clients holding the row's
/// version see the change.
/// version never goes back, nor names other frames than it did: a sidecar read under the lease that is behind the
/// locked row (an older copy put back by hand or by a file-sync tool), or at the row's version over other frames
/// (another index of the archive bumped a copy put back to it, <see cref="OtherFramesAsync"/>), is written again one
/// version past the row, so editors and clients holding the row's version see the change.
/// </summary>
/// <param name="listed">The folder as the scan listed it before reading <paramref name="manifest"/> without the sequence
/// lease; null when the caller holds the lease and read the sidecar under it.</param>
@@ -131,7 +132,11 @@ public sealed class ArchiveIndexer
if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar
|| (existing is not null && existing.FramesVersion != manifest.Versions.Frames)))
return null;
if (existing is not null && existing.FramesVersion > manifest.Versions.Frames)
// Under the lease, the row's own version may also name other frames: another index of the archive bumped a copy
// put back to it (OtherFramesAsync). A caller without the lease compared them before its read (FramesMovedAsync).
if (existing is not null && (existing.FramesVersion > manifest.Versions.Frames
|| (listed is null && existing.FramesVersion == manifest.Versions.Frames
&& await OtherFramesAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false))))
{
manifest = manifest with
{
@@ -220,16 +225,39 @@ public sealed class ArchiveIndexer
(await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos
!= FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence));
/// <summary>Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).</summary>
public async Task<SequenceManifest> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
/// <summary>
/// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action), as a scan's reconcile
/// does (<see cref="Reconcile"/>): what covered the frames is cleared in the sidecar and the row.
/// </summary>
/// <returns>Whether photos were added, gone or back: the light analysis, thumbnails and proxies are then to be made again
/// (<see cref="ArchiveService.ReconcileAsync"/>). The folder's stamps are recorded, so no scan comes back to it and
/// queues them.</returns>
public async Task<bool> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken)
.ConfigureAwait(false);
var changed = false;
await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m =>
{
var reconciled = Reconcile(folder, m, cancellationToken);
changed = reconciled is not null;
return reconciled;
}, cancellationToken).ConfigureAwait(false);
return changed;
}
/// <summary>
/// The sidecar brought in line with the folder's photos (<see cref="SequenceAdopter.Reconcile"/>); null when nothing
/// changed. Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the
/// sidecar then tells every host, so the follow-ups make them again (the images on a server,
/// <see cref="ArchiveServiceOptions.ImagesAfterScan"/>).
/// </summary>
private static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
SequenceAdopter.Reconcile(sequence, manifest, cancellationToken) is { } reconciled
? reconciled with { Status = reconciled.Status & ~CoveringStatus }
: null;
/// <summary>
/// Applies <paramref name="edit"/> to the sidecar (null writes nothing) and indexes the result, both under the sequence
/// lease. Frames inserted before indexed ones move the project's positions (<see cref="ProjectPositions"/>) without a
@@ -297,14 +325,9 @@ public sealed class ArchiveIndexer
}
if (reconcile || File.Exists(sequence.IngestJournalPath))
{
// Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the
// sidecar then tells every host, so the scan's follow-ups make them again (the images on a server,
// ArchiveServiceOptions.ImagesAfterScan).
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id,
m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled
? reconciled with { Status = reconciled.Status & ~CoveringStatus }
: null,
cancellationToken).ConfigureAwait(false);
// Photos added, gone or back clear what covered the frames, and the scan's follow-ups make it again.
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => Reconcile(sequence, m, cancellationToken), cancellationToken)
.ConfigureAwait(false);
bytes = null;
}
else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false))
@@ -369,7 +392,23 @@ public sealed class ArchiveIndexer
/// </summary>
private async Task<bool> FramesMovedAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
&& indexed.FramesVersion != manifest.Versions.Frames;
&& await OtherFramesAsync(_db, indexed, manifest, cancellationToken).ConfigureAwait(false);
/// <summary>
/// Whether the index row <paramref name="indexed"/> holds other frames than <paramref name="manifest"/>: another frames
/// version, or the same version over other frames (ids, or their order). Each index of a shared archive bumps a sidecar
/// put back behind its own row one version past that row (<see cref="SyncAsync"/>), and another index may hold that
/// very version for the frames as they were before: that index still sees the change, and bumps past it in turn.
/// </summary>
private static async Task<bool> OtherFramesAsync(TitanoRepositories db, Sequence indexed, SequenceManifest manifest, CancellationToken cancellationToken)
{
if (indexed.FramesVersion != manifest.Versions.Frames) return true;
var rows = await db.Frames.ListPositionsAsync(indexed.Id, cancellationToken).ConfigureAwait(false);
if (rows.Count != manifest.Frames.Count) return true;
var listed = new Dictionary<int, Guid>(manifest.Frames.Count);
foreach (var frame in manifest.Frames) listed[frame.I] = frame.Id;
return rows.Any(row => !listed.TryGetValue(row.Index, out var id) || id != row.FrameId);
}
/// <summary>A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.</summary>
private async Task<SequenceManifest> AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
+28 -3
View File
@@ -6,6 +6,7 @@ using Titano.Core.Model;
using Titano.Core.Naming;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
using Titano.Data.Processing;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
@@ -284,11 +285,35 @@ public sealed class ArchiveService : IJobHandlerProvider
public async Task<OrphanReport> FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
/// <summary>Orphans fix: index the unlisted photos of a sequence and flag its missing files.</summary>
public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default)
/// <summary>
/// Orphans fix: index the unlisted photos of a sequence and flag its missing files. Photos added, gone or back clear
/// what covered the frames, as a scan's reconcile does (<see cref="ArchiveIndexer.ReconcileAsync"/>), and what a scan's
/// follow-ups would make again is queued on <paramref name="followUps"/> (null: nothing): the light analysis, a
/// background deep analysis when <paramref name="deepAnalysis"/> asks (as <see cref="ScanParameters.DeepAnalysis"/>
/// does for a scan) and, on a server (<see cref="ArchiveServiceOptions.ImagesAfterScan"/>), the thumbnails and proxies.
/// A kind the engine does not run, or one a queued job of that kind already covers for the sequence, is left out.
/// </summary>
public async Task ReconcileAsync(Guid sequenceId, JobEngine? followUps, bool deepAnalysis, CancellationToken cancellationToken = default)
{
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
if (!await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false) || followUps is null) return;
List<ArchiveJobRequest> requests = [ProcessingJobs.LightAnalysis([sequenceId], sequence.DisplayTitle)];
// BRIEF §10.2: background, low priority.
if (deepAnalysis) requests.Add(ProcessingJobs.DeepAnalysis([sequenceId], sequence.DisplayTitle) with { Priority = JobPriority.Background });
if (_writer.Options.ImagesAfterScan)
{
requests.Add(ArchiveJobs.Thumbnails([sequenceId], sequence.DisplayTitle));
// An archive whose policy makes no proxies would get a pass that does nothing.
if (archive.ProxyLongEdge != ProxyPolicy.Disabled) requests.Add(ArchiveJobs.Proxies([sequenceId], sequence.DisplayTitle));
}
foreach (var request in requests)
{
if (!followUps.Runs(request.Kind)) continue;
// A queued job runs after the fix, on the frames as they are then, whatever else it targets (a scan's follow-ups).
var queued = await _db.Jobs.ListQueuedAsync(request.Kind, 1000, cancellationToken).ConfigureAwait(false);
if (queued.Any(j => j.TargetIds.Contains(sequenceId))) continue;
await followUps.EnqueueAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false);
}
}
/// <summary>Orphans fix: drop index rows whose sidecar is gone.</summary>
+34 -9
View File
@@ -45,9 +45,9 @@ internal enum FramesChange
/// Auto values of the project. The light pass reads the index, probes the decode tier (§8.2) of frames whose tier is not
/// known yet (headers only, once per frame) and reads one file for the GPS fix; the deep pass decodes every frame,
/// checkpointing into <c>.titano/analysis.partial</c> so a cancelled, paused or crashed run resumes, records the tier
/// each frame decoded at (or that it did not decode) and saves <c>analysis.json</c> atomically. Results land in the
/// sidecar (status, tiers), the index (status, warning count, compact summary, tier flags, per-frame light statistics)
/// and <c>project.json</c>.
/// each frame decoded at (or that it did not decode) and saves <c>analysis.json</c> atomically under the sequence lock.
/// Results land in the sidecar (status, tiers), the index (status, warning count, compact summary, tier flags,
/// per-frame light statistics) and <c>project.json</c>.
/// </summary>
internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions options, SequenceWriter writer, ArchiveIndexer indexer, ProjectStore projects)
{
@@ -94,17 +94,23 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
// the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it
// left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records
// nothing (FramesChangeAsync), and a light analysis of the frames as they are runs instead, here or where they
// were changed.
// were changed; its report is saved only over none of the frames as they are (SaveReport). What a deep pass records
// then follows the report analysis.json holds: its own, or the one it left in place.
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false);
var saved = deep && SaveReport(folder, report!, change == FramesChange.None && manifest.Versions.Frames == sequence.FramesVersion,
manifest.Frames.Count);
if (change != FramesChange.None)
return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], change);
if (deep)
{
// A report left in place is read as a light analysis reads it, so the deep status, summary, warnings and
// Auto values recorded here describe the file that readers and the report endpoint open.
var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
if (!Describes(report!, now)) report = null;
if (!saved) report = ReadReport(folder, now);
else if (!Describes(report!, now)) report = null;
}
else
{
@@ -247,10 +253,30 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
return tiers;
}
/// <summary>
/// Saves a deep pass's report as <c>analysis.json</c>, under the sequence lock, then drops its checkpoint; returns
/// whether it saved it. The report replaces the saved one when it measured the frames as they are
/// (<paramref name="current"/>), or when the saved one describes them no better: none, unreadable, or of another frame
/// count than <paramref name="frameCount"/>, the sidecar's. A pass overtaken by an upload, or by another process
/// sharing the archive, would otherwise replace the report a later pass made of the frames as they are, and the next
/// light analysis would drop that one's deep status.
/// </summary>
private static bool SaveReport(SequencePaths folder, DeepAnalysisReport report, bool current, int frameCount)
{
var save = current || ReadReport(folder, frameCount) is null;
if (save) report.Save(folder.AnalysisPath);
Checkpoint(folder).Delete();
return save;
}
/// <summary>The deep pass's checkpoint, <c>.titano/analysis.partial</c>: kept until the pass has saved its report.</summary>
private static DeepAnalysisCheckpointFile Checkpoint(SequencePaths folder) =>
new(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
private async Task<(DeepAnalysisReport Report, Dictionary<Guid, FileTier> Decoded)> AnalyseDeepAsync(Archive archive, SequenceFrames frames,
IReadOnlyList<LightFrame> metadata, SequencePaths folder, IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
{
var file = new DeepAnalysisCheckpointFile(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
var file = Checkpoint(folder);
// The host's analysis parameters as they are now (Settings › Analysis on the desktop). Those that change per-frame
// results are part of the checkpoint's fingerprint, so a partial run made with other values starts again.
var analysis = options.Current.DeepAnalysis;
@@ -263,11 +289,10 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
.ConfigureAwait(false);
decoded = Decoded(frames, report, source, companions, cancellationToken);
}
// An unreadable verdict on a frame read from its original goes stale once the frame has a proxy (ReadReport).
// An unreadable verdict on a frame read from its original goes stale once the frame has a proxy (ReadReport). The
// report is saved under the sequence lock, once the frames it measured are known to be current (SaveReport).
int[] fromProxies = [.. Enumerable.Range(0, frames.Count).Where(i => frames.Proxied?[i] == true)];
report = report with { ProxyFrames = fromProxies.Length > 0 ? fromProxies : null };
report.Save(folder.AnalysisPath);
file.Delete();
var columns = report.Frames;
var stats = new FrameLightStats[frames.Count];
@@ -500,11 +500,22 @@ public sealed class SequenceRepository
await session.ScalarAsync<Guid?>($"SELECT id FROM sequences WHERE id = $1{SqlDialect.For(session.Dialect).ForUpdate}", [id], cancellationToken)
.ConfigureAwait(false) is not null;
/// <summary>Replaces the tag links of a sequence whose row the caller has locked (or just upserted).</summary>
/// <summary>
/// Replaces the tag links of a sequence whose row the caller has locked (or just upserted); a set the sequence already
/// carries (names compared case-insensitively) is left as it is. On PostgreSQL every tag written is locked until
/// commit, and the tags are written in the order of their keys whatever order the caller listed them in (a PATCH sends
/// them in any order): writers of two sequences that share tags then take those rows in one order, and never wait for
/// each other's (PostgreSQL would abort one with a deadlock, and nothing retries it).
/// </summary>
private async Task ReplaceTagsAsync(IDbSession session, Guid sequenceId, IEnumerable<string> tags, CancellationToken cancellationToken)
{
var wanted = TagRepository.NormalizeSet(tags);
var keys = wanted.Select(TagRepository.Key).ToList();
var stored = await session.QueryListAsync("SELECT t.name_key FROM sequence_tags st JOIN tags t ON t.id = st.tag_id WHERE st.sequence_id = $1",
[sequenceId], r => r.GetString(0), cancellationToken).ConfigureAwait(false);
if (stored.Count == keys.Count && stored.ToHashSet(StringComparer.Ordinal).SetEquals(keys)) return;
await session.ExecuteAsync("DELETE FROM sequence_tags WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
foreach (var tag in TagRepository.NormalizeSet(tags))
foreach (var tag in wanted.OrderBy(TagRepository.Key, StringComparer.Ordinal))
{
DbParam[] args = [Guid.CreateVersion7(), tag, TagRepository.Key(tag)];
Guid tagId;
@@ -1,6 +1,7 @@
using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Jobs;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
@@ -112,7 +113,7 @@ public sealed class ArchiveIntegrityTests(TestContext context)
Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id);
Assert.SequenceEqual(["New folder"], report.UnindexedFolders);
await fx.Service.ReconcileAsync(id, fx.Ct);
await fx.Service.ReconcileAsync(id, null, deepAnalysis: false, fx.Ct);
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.Equal(4, frames.Count);
Assert.Equal(FrameFlags.Missing, frames[0].Flags);
@@ -120,6 +121,38 @@ public sealed class ArchiveIntegrityTests(TestContext context)
Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0));
}
[Test]
public async Task TheOrphansFixOnAServerDrawsTheFramesItIndexes()
{
await using var fx = await CreateAsync(context);
var id = await IngestSeriesAsync(fx, "a", 3, 1);
// The archive's server, which draws what its clients load; its engine runs no analysis.
var server = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "server", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true });
await using var engine = new JobEngine(fx.Db, server.CreateJobHandlers(),
new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync(fx.Ct);
foreach (var pass in new[] { ArchiveJobs.Thumbnails([id]), ArchiveJobs.Proxies([id]) })
await fx.WaitForAsync((await engine.EnqueueAsync(pass, cancellationToken: fx.Ct)).Id, JobState.Done);
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id, fx.Ct))!.Folder);
File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50));
// The fix indexes the photo copied in by hand, and queues the thumbnails and proxies of the frames as they are now;
// no analysis, even when asked for one.
var before = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Select(j => j.Id).ToHashSet();
await server.ReconcileAsync(id, engine, deepAnalysis: true, fx.Ct);
var queued = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Where(j => !before.Contains(j.Id)).ToList();
Assert.SequenceEqual([JobKind.Proxies, JobKind.Thumbs], queued.Select(j => j.Kind).Order());
foreach (var job in queued)
{
Assert.SequenceEqual([id], job.TargetIds);
await fx.WaitForAsync(job.Id, JobState.Done);
}
var row = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
Assert.Equal(4, row.FrameCount);
Assert.True(row.Status.HasFlag(SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt), row.Status.ToString());
Assert.True(File.Exists(folder.Proxy(3)), "proxy of the new frame");
}
[Test]
public async Task MaintenanceRemovesOnlyStaleLeftovers()
{
@@ -508,6 +508,57 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AFramesVersionAnotherIndexReachedForOtherFramesIsIndexedPastIt()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var folder = fx.Paths.Sequence("a");
async Task<Sequence> UploadAsync(string name, int minute)
{
var staged = fx.Stage(Start.AddMinutes(minute), seed: 90 + minute);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, name, staged.Size, staged.Sha256, null, null),
fx.Ct);
return (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
}
await UploadAsync("IMG_0004.JPG", 1);
// A desktop sharing this server's archive, with its own index of the four photos.
await using var desktopDb = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(Context.CreateTempDirectory("desktop"), "titano.db")), fx.Ct);
var desktop = new ArchiveService(desktopDb, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
var desktopArchive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
Task<ScanSummary> DesktopScanAsync() => desktop.Indexer.ScanAsync(desktopArchive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
await DesktopScanAsync();
// The server takes a fifth photo before the desktop scans again.
var uploaded = await UploadAsync("IMG_0005.JPG", 2);
Assert.Equal(uploaded.FramesVersion - 1, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion);
// The sidecar is damaged; the desktop's scan rebuilds it from the photos at the first version, the fifth photo under
// an id of its own, and writes it one version past its own row: the version the server holds for the photo as it
// filed it.
await File.WriteAllTextAsync(folder.ManifestPath, "{", fx.Ct);
Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences);
var rebuilt = ManifestStore.ReadSequence(folder.ManifestPath);
Assert.Equal(uploaded.FramesVersion, rebuilt.Versions.Frames);
Assert.NotEqual((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[^1].Id, rebuilt.Frames[^1].Id);
// The server's scan finds other frames at the version it holds: it indexes them one version past it, in both places,
// so an editor holding that version for the photo as it was is refused.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(uploaded.FramesVersion + 1, indexed.FramesVersion);
Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
Assert.SequenceEqual(rebuilt.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
var edit = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Rating = 2 }, null, uploaded.FramesVersion, fx.Ct);
Assert.Equal(SequenceEditOutcome.Conflict, edit.Outcome);
// The desktop follows, and neither index finds anything more to do.
Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences);
Assert.Equal(indexed.FramesVersion, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion);
Assert.Empty((await fx.ScanAsync()).Sequences);
Assert.Empty((await DesktopScanAsync()).Sequences);
}
[Test]
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
{
@@ -82,6 +82,81 @@ public sealed class AnalysisFollowUpTests(TestContext context)
Assert.True(ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
}
[Test]
public async Task PhotosTheOrphansFixIndexesAreAnalysedAgain()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof");
var folder = fx.Folder(sequence);
await fx.StartEngineAsync(withArchive: true);
var first = await fx.EnqueueAsync(Titano.Data.Processing.ProcessingJobs.LightAnalysis([sequence.Id]));
await fx.WaitForAsync(first.Id, JobState.Done);
// A server sharing the archive drew the twelve frames and made their proxies.
const SequenceStatus images = SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = manifest.Status | images });
await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, images, SequenceStatus.None, fx.Ct);
// Photos copied into the folder by hand: the Archive screen's orphans check lists them, and its fix indexes them. The
// light analysis it queues waits on the first new photo's probe, so what the fix left can be seen.
fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 18 }), "Roof", first: 12);
fx.Decoder.HoldProbeOf(12);
await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, deepAnalysis: false, fx.Ct);
const SequenceStatus covering = SequenceStatus.AnalyzedLight | images;
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((18, SequenceStatus.None), (row.FrameCount, row.Status & covering));
Assert.Equal(SequenceStatus.None, ManifestStore.ReadSequence(folder.ManifestPath).Status & covering);
var again = (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first.Id);
Assert.SequenceEqual([sequence.Id], again.TargetIds);
Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.DeepAnalyse, cancellationToken: fx.Ct)).Items);
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
fx.Decoder.Release();
await fx.WaitForAsync(again.Id, JobState.Done);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "analysed again");
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
Assert.Contains("18", ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).Timing.FrameRate.ReasonArgs!);
}
[Test]
public async Task PhotosTheOrphansFixIndexesGetTheDeepAnalysisAScanWouldQueue()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof");
var lake = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Lake");
var folder = fx.Folder(sequence);
await fx.StartEngineAsync(withArchive: true);
Task<IReadOnlyList<Job>> QueuedDeepAsync() => fx.Db.Jobs.ListQueuedAsync(JobKind.DeepAnalyse, 50, fx.Ct);
// The host asks for a deep analysis after scans: the orphans fix queues one in the background too, behind the light
// analysis, which waits on the first new photo's probe so the queue can be seen.
fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 18 }), "Roof", first: 12);
fx.Decoder.HoldProbeOf(12);
await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, deepAnalysis: true, fx.Ct);
var light = (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.Single();
var deep = (await QueuedDeepAsync()).Single();
Assert.SequenceEqual([sequence.Id], deep.TargetIds);
Assert.Equal(JobPriority.Background, deep.Priority);
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
// The user cancels it, and asks for a deep analysis of both sequences instead. More photos and the fix again: the
// one waiting covers them, so no other is queued.
await fx.Engine.CancelAsync(deep.Id, fx.Ct);
var both = await fx.EnqueueAsync(Titano.Data.Processing.ProcessingJobs.DeepAnalysis([lake.Id, sequence.Id]));
fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 20 }), "Roof", first: 18);
await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, deepAnalysis: true, fx.Ct);
Assert.SequenceEqual([both.Id], (await QueuedDeepAsync()).Select(j => j.Id));
fx.Decoder.Release();
await fx.WaitForAsync(light.Id, JobState.Done);
await fx.WaitForAsync(both.Id, JobState.Done);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(20, row.FrameCount);
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
Assert.Equal(20, Titano.Data.Processing.SequenceAnalyser.ReadReport(folder, 20)?.FrameCount);
}
[Test]
public async Task AScanAskedForDeepAnalysisQueuesItInTheBackgroundForNewSequencesOnly()
{
@@ -292,6 +292,89 @@ public sealed class AnalysisJobTests(TestContext context)
Assert.Equal(advice, await ProjectJsonAsync(fx, folder));
}
[Test]
public async Task ADeepAnalysisOvertakenByOneOfTheFramesAsTheyAreLeavesThatOnesReport()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering(12));
var folder = fx.Folder(sequence);
await fx.StartEngineAsync(workers: 2);
// A deep analysis starts on twelve frames; while it runs, a thirteenth photo is indexed and a second deep analysis,
// on the other worker, analyses all thirteen and records its report.
fx.Decoder.HoldFrame(0);
var overtaken = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
fx.Decoder.Hold = null;
await fx.AddAsync(Flickering(13), first: 12);
var current = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.WaitForAsync(current.Id, JobState.Done);
var report = await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct);
Assert.Equal(13, SequenceAnalyser.ReadReport(folder, 13)!.FrameCount);
var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.True(analysed.Status.HasFlag(SequenceStatus.AnalyzedDeep), analysed.Status.ToString());
fx.Decoder.Release();
var done = await fx.WaitForAsync(overtaken.Id, JobState.Done);
// The first pass measured twelve: it leaves the report of the thirteen in place, so the light analysis it queues
// keeps the deep status and its summary.
Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail);
Assert.Equal(report, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct));
Assert.FileMissing(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
var light = Assert.Contains(await LightJobsAsync(fx), j => j.TargetIds.Contains(sequence.Id));
await fx.WaitForAsync(light.Id, JobState.Done);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status");
Assert.Equal(analysed.AnalysisJson, row.AnalysisJson);
}
[Test]
public async Task ADeepAnalysisThatLeavesTheReportInPlaceRecordsThatReport()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering(12));
var folder = fx.Folder(sequence);
await fx.StartEngineAsync();
var first = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.WaitForAsync(first.Id, JobState.Done);
var measured = SequenceAnalyser.ReadReport(folder, 12)!;
// A deep analysis starts on the twelve frames on the only worker. Meanwhile another Titano sharing the archive files
// them again, at the next frames version, and saves its own analysis of them; this index's scan indexes them, and
// their light analysis waits in the queue.
fx.Decoder.HoldFrame(0);
var overtaken = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
fx.Decoder.Hold = null;
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
{
Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 },
UpdatedAt = DateTimeOffset.UtcNow,
});
Assert.Contains(sequence.Id, (await fx.Archive.ScanAsync()).Sequences);
var theirs = measured with { HeroIndex = measured.HeroIndex == 5 ? 6 : 5 };
theirs.Save(folder.AnalysisPath);
var report = await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct);
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
fx.Decoder.Release();
await fx.WaitForAsync(overtaken.Id, JobState.Done);
// The pass leaves that report in place, and records it rather than its own: the summary in the index is the one of
// the file every reader opens, and the light analysis that follows keeps it.
Assert.Equal(report, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct));
var summary = AnalysisSummary.From(theirs);
Assert.NotEqual(AnalysisSummary.From(measured), summary);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
Assert.Equal(summary, AnalysisSummary.FromJson(row.AnalysisJson!));
await fx.WaitForAsync(light.Id, JobState.Done);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
Assert.Equal(summary, AnalysisSummary.FromJson(row.AnalysisJson!));
}
[Test]
public async Task AnAnalysisOvertakenWhileALightAnalysisWaitsRecordsAndThatOneReplacesIt()
{
@@ -101,6 +101,51 @@ public abstract class SequenceEditTests(TestContext context) : RepositoryTest(co
var later = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Rating = 3 }, stored.SettingsVersion, Ct);
Assert.Equal((sequence.SettingsVersion + 1, sequence.FramesVersion + 1, 3), (later.Sequence!.SettingsVersion, later.Sequence.FramesVersion, later.Sequence.Rating));
}
[Test]
public async Task WritersOfSequencesSharingTagsListedInOppositeOrdersBothComplete()
{
var db = await OpenTitanoAsync(multiConnection: true);
var archive = TestData.Archive();
await db.Archives.UpsertAsync(archive, Ct);
var tagged = TestData.Sequence(archive.Id, title: "Tagged");
var first = TestData.Sequence(archive.Id, title: "First");
var second = TestData.Sequence(archive.Id, title: "Second");
await db.Sequences.UpsertAsync(tagged with { Tags = ["Alps", "sunset"] }, Ct);
await db.Sequences.UpsertAsync(first, Ct);
await db.Sequences.UpsertAsync(second, Ct);
var alps = (await db.Tags.ListAsync(Ct)).Single(t => t.Name == "Alps");
// A transaction holds the first of the two tags (SQLite: its write lock), so both writers below wait until it ends.
Task<SequenceEditResult> forward, backward;
await using (var tx = await db.BeginAsync(Ct))
{
if (Dialect == Dialect.Postgres)
Assert.Equal(alps.Id, await tx.Session.ScalarAsync<Guid?>("SELECT id FROM tags WHERE id = $1 FOR UPDATE", [alps.Id], Ct));
else
Assert.True(await tx.Tags.RenameAsync(alps.Id, "Alps", Ct));
forward = db.Sequences.EditAsync(first.Id, new SequenceEdit { Tags = ["Alps", "sunset"] }, null, Ct);
await Task.WhenAny(forward, Task.Delay(300, Ct));
// The other writer lists the tags the other way round (a PATCH sends them in any order).
backward = db.Sequences.EditAsync(second.Id, new SequenceEdit { Tags = ["SUNSET", "alps"] }, null, Ct);
await Task.WhenAny(backward, Task.Delay(300, Ct));
Assert.False(forward.IsCompleted || backward.IsCompleted, "both writers wait for the tag");
if (Dialect == Dialect.Postgres)
{
// A writer whose tags are already the stored ones takes no tag at all.
var unchanged = db.Sequences.EditAsync(tagged.Id, new SequenceEdit { Tags = ["sunset", "ALPS"], Rating = 2 }, null, Ct);
Assert.Equal(unchanged, await Task.WhenAny(unchanged, Task.Delay(TimeSpan.FromSeconds(20), Ct)));
Assert.Equal(SequenceEditOutcome.Updated, (await unchanged).Outcome);
}
await tx.CommitAsync(Ct);
}
// Both take the tags in one order: neither is aborted for a deadlock.
Assert.Equal((SequenceEditOutcome.Updated, SequenceEditOutcome.Updated), ((await forward).Outcome, (await backward).Outcome));
foreach (var sequence in new[] { tagged, first, second })
Assert.SequenceEqual(["Alps", "sunset"], (await db.Sequences.GetAsync(sequence.Id, Ct))!.Tags);
Assert.Count(2, await db.Tags.ListAsync(Ct));
}
}
public sealed class SqliteSequenceEditTests(TestContext context) : SequenceEditTests(context)