diff --git a/src/Titano.Data/Archiving/ArchiveImages.cs b/src/Titano.Data/Archiving/ArchiveImages.cs index adee216a..b70d03d8 100644 --- a/src/Titano.Data/Archiving/ArchiveImages.cs +++ b/src/Titano.Data/Archiving/ArchiveImages.cs @@ -255,7 +255,7 @@ public sealed class ArchiveImages pack.Commit(); } await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false); - await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false); + await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false); return new ImagePassResult(built, reused, undecodable); } @@ -327,7 +327,7 @@ public sealed class ArchiveImages if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false); } await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false); - await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false); + await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false); return new ImagePassResult(built, reused, undecodable, waiting); } @@ -520,15 +520,47 @@ public sealed class ArchiveImages } } - private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken) + /// + /// Sets in the sidecar and the index when the pass covered every frame it read + /// () and the sidecar, as it is under the lease, still lists those frames + /// (); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or + /// renumbered after the pass read have nothing of it, and a scan that cleared the flag for + /// them queues the next pass only while the flag stays clear. The index is written under the same lease, so the flag + /// never lands there after a scan cleared it in both. + /// + private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, SequenceStatus flag, bool complete, + CancellationToken cancellationToken) { - await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => + var folder = paths.Sequence(sequence.Folder); + await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { - var status = complete ? m.Status | flag : m.Status & ~flag; - return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; - }, cancellationToken).ConfigureAwait(false); - await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken) - .ConfigureAwait(false); - await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false); + var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); + var built = complete && SameFrames(manifest.Frames, read.Frames); + var status = built ? manifest.Status | flag : manifest.Status & ~flag; + if (status != manifest.Status) + { + manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; + ManifestStore.WriteSequence(folder.ManifestPath, manifest); + } + await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken) + .ConfigureAwait(false); + await _indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Whether lists the photos did, each at the same index with the + /// same file, size and capture time: thumbnails and proxies are kept by frame index, so photos added, removed, + /// renumbered or replaced since have none of what the pass made. + /// + private static bool SameFrames(IReadOnlyList current, IReadOnlyList read) + { + if (current.Count != read.Count) return false; + for (var i = 0; i < current.Count; i++) + { + var (now, then) = (current[i], read[i]); + if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt) return false; + } + return true; } } diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 263376df..ae7bdddc 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -31,6 +31,9 @@ public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Remo /// public sealed class ArchiveIndexer { + /// What covers the frames as they were: a scan that finds them changed clears it, and its follow-ups make it again. + private const SequenceStatus CoveringStatus = SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt; + private readonly TitanoDatabase _db; private readonly SequenceWriter _writer; @@ -131,29 +134,44 @@ public sealed class ArchiveIndexer _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId, FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken); + /// + /// for a writer that changed, under the sequence lease, only what it owns of the + /// sidecar and the index row (status, decode tiers): recorded only while the index holds the frames of + /// , the sidecar as that writer read and wrote it. Frames another process (a desktop + /// sharing the archive) added, removed or renumbered without this index keep the stamps as they were, so the next + /// scan still indexes them and queues what they need. + /// + internal async Task RecordStampsInStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) + { + if (await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.FramesVersion == manifest.Versions.Frames) + await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + } + /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) { var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); - return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken) + return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, (m, _) => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken) .ConfigureAwait(false); } /// /// Applies to the sidecar (null writes nothing) and indexes the result, both under the sequence - /// lease. Frames inserted before indexed ones move the project's positions () without a - /// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto - /// them rather than writing its old positions back. + /// lease; the edit also gets the sequence's index row as it is then (null when it has none). Frames inserted before + /// indexed ones move the project's positions () without a new settings version; an editor + /// saving its project holds the same lease, so it reads them moved and merges onto them rather than writing its old + /// positions back. /// private async Task UpdateAndSyncAsync(Archive archive, ArchivePaths paths, SequencePaths sequence, Guid sequenceId, - Func edit, CancellationToken cancellationToken) + Func edit, CancellationToken cancellationToken) { await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); - if (edit(manifest) is { } updated) + var indexed = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false); + if (edit(manifest, indexed) is { } updated) { ManifestStore.WriteSequence(sequence.ManifestPath, updated); manifest = updated; @@ -194,21 +212,23 @@ public sealed class ArchiveIndexer ManifestStore.WriteSequence(sequence.ManifestPath, manifest); bytes = null; } - if (reconcile || File.Exists(sequence.IngestJournalPath)) + var reconcilePhotos = reconcile || File.Exists(sequence.IngestJournalPath); + if (reconcilePhotos || await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false)) { - // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, so the scan's - // follow-ups make them again (the images on a server, ArchiveServiceOptions.ImagesAfterScan). - manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, - m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled - ? reconciled with { Status = reconciled.Status & ~(SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt) } - : null, - cancellationToken).ConfigureAwait(false); - bytes = null; - } - else if (await MayMovePositionsAsync(manifest, cancellationToken).ConfigureAwait(false)) - { - // Another process changed the frames: the sidecar is read again under the lease the project moves under. - manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); + // Photos added, gone or back, or frames another process (a desktop sharing the archive) added, removed or + // renumbered together with their sidecar entries, which leaves the statuses its sidecar had: the light analysis, + // thumbnails and proxies no longer cover the frames, so the scan's follow-ups make them again (the images on a + // server, ArchiveServiceOptions.ImagesAfterScan). Decided under the lease, on the sidecar as it is then against + // the index row, under which the project's positions also move. + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, (m, indexed) => + { + var updated = reconcilePhotos ? SequenceAdopter.Reconcile(sequence, m, cancellationToken) : null; + if (updated is null && (indexed is null || indexed.FramesVersion == m.Versions.Frames)) return null; + updated ??= m; + var status = updated.Status & ~CoveringStatus; + if (ReferenceEquals(updated, m) && status == m.Status) return null; + return updated with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; + }, cancellationToken).ConfigureAwait(false); bytes = null; } else @@ -257,9 +277,12 @@ public sealed class ArchiveIndexer } } - /// The index holds a project for this sequence and other frames than the sidecar: its positions may move. - private async Task MayMovePositionsAsync(SequenceManifest manifest, CancellationToken cancellationToken) => - await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { SettingsJson: not null } indexed + /// + /// The index holds this sequence with other frames than the sidecar: another process changed them (its project's + /// positions may move, and what covered the frames no longer does). + /// + private async Task FramesMovedAsync(SequenceManifest manifest, CancellationToken cancellationToken) => + await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.FramesVersion != manifest.Versions.Frames; /// A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept. @@ -309,7 +332,10 @@ public sealed class ArchiveIndexer /// /// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence - /// waiting for its first frame has no sidecar yet and stays. + /// waiting for its first frame has no sidecar yet and stays. A row the folders listed when the rebuild began do not + /// back is looked at again under its lease, as it and its folder are when it would go (): + /// uploads go on during a rebuild, which can take minutes, so a sequence that got its first frame or was created + /// meanwhile has a folder the listing does not hold. /// private async Task RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList folders, Locations locations, CancellationToken cancellationToken) @@ -322,12 +348,46 @@ public sealed class ArchiveIndexer foreach (var location in rows) { if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue; - if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++; - await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false); + IAsyncDisposable held; + try + { + held = await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false); + } + catch (LeaseUnavailableException) + { + // Another process is writing the sequence, so it is there. + continue; + } + await using (held) + { + if (await BackedAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) continue; + if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++; + await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false); + } } return removed; } + /// + /// Whether the index row of keeps its place as it is now: it waits for its first frame, + /// its folder holds a sidecar naming it (one that cannot be read proves nothing), or it is gone already. + /// + private async Task BackedAsync(ArchivePaths paths, Guid sequenceId, CancellationToken cancellationToken) + { + if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row) return true; + if ((await AwaitingFolderAsync(paths, [new SequenceLocation(row.Id, row.Folder, row.Title)], cancellationToken).ConfigureAwait(false)).Count > 0) + return true; + var sidecar = paths.Sequence(row.Folder).ManifestPath; + try + { + return File.Exists(sidecar) && ManifestStore.ReadSequence(sidecar).Id == sequenceId; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + return true; + } + } + /// Which folder each indexed sequence id lives in, updated as the scan goes. private sealed class Locations { diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs index 3cdf58a7..5ece612c 100644 --- a/src/Titano.Data/Archiving/ArchiveVerifier.cs +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -108,22 +108,29 @@ public sealed class ArchiveVerifier pending.Clear(); } - /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt. + /// + /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors + /// while one is. Sidecar and index are written under one lease, the index row with these fields only + /// (), so a settings edit or an ingest committed meanwhile stays. + /// private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken) { var sequence = paths.Sequence(target.Folder); - var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m => + await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false)) { + var m = SequenceWriter.ReadLocked(sequence, target.Id); var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0); - return m with + var now = DateTimeOffset.UtcNow; + var manifest = m with { Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified, - VerifiedAt = DateTimeOffset.UtcNow, - UpdatedAt = DateTimeOffset.UtcNow, + VerifiedAt = now, + UpdatedAt = now, }; - }, cancellationToken).ConfigureAwait(false); - var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false); - await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); - await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + await _db.Sequences.RecordVerifiedAsync(target.Id, clean ? SequenceStatus.Verified : SequenceStatus.HasErrors, + clean ? SequenceStatus.HasErrors : SequenceStatus.Verified, now, cancellationToken).ConfigureAwait(false); + await _indexer.RecordStampsInStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + } } } diff --git a/src/Titano.Data/Archiving/FrameTiers.cs b/src/Titano.Data/Archiving/FrameTiers.cs index 6b7d27ed..e13b0f2c 100644 --- a/src/Titano.Data/Archiving/FrameTiers.cs +++ b/src/Titano.Data/Archiving/FrameTiers.cs @@ -61,6 +61,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi /// /// Whether every such frame follows its proxy even where no tier is recorded (analyses and the /// proxy pass); otherwise only the frames in do. + /// + /// The sidecar and the index are written under one lease, the index from the sidecar read under it, and the index + /// row gets only what tiers change (): a whole row written back + /// would undo what another writer committed meanwhile, an ingest's frames or a settings edit (the server's PATCH + /// takes no lease). + /// /// The archive is offline or read-only. public async Task> RecordAsync(Archive archive, Sequence sequence, IReadOnlyDictionary tiers, bool proxies, CancellationToken cancellationToken) @@ -69,12 +75,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); - var changed = new List(); - ProxyPolicy? policy = null; - var manifest = await writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => + await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { - changed.Clear(); + var m = SequenceWriter.ReadLocked(folder, sequence.Id); + var changed = new List(); var gained = false; + ProxyPolicy? policy = null; var frames = new SidecarFrame[m.Frames.Count]; for (var i = 0; i < frames.Length; i++) { @@ -96,21 +102,25 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi frames[i] = updated; changed.Add(updated); } - if (changed.Count == 0) return null; - return m with + if (changed.Count == 0) return m.Frames; + var manifest = m with { Frames = frames, Status = gained ? m.Status | SequenceStatus.HasWarnings : m.Status, UpdatedAt = DateTimeOffset.UtcNow, }; - }, cancellationToken).ConfigureAwait(false); - if (changed.Count == 0) return manifest.Frames; - await db.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken) - .ConfigureAwait(false); - if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is { } existing) - await db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, folder, existing), cancellationToken).ConfigureAwait(false); - await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); - return manifest.Frames; + ManifestStore.WriteSequence(folder.ManifestPath, manifest); + await using (var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken) + .ConfigureAwait(false); + await tx.Sequences.RecordTiersAsync(sequence.Id, gained ? SequenceStatus.HasWarnings : SequenceStatus.None, + frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + return manifest.Frames; + } } /// diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs index fa1580d7..50e3fdae 100644 --- a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs +++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs @@ -67,10 +67,11 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, /// /// Queues a background (thumbnails or proxies) of each sequence this run indexed whose /// flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the - /// flag then). After a resumed run, which may have indexed sequences before the interruption, of every sequence of the - /// archive without the flag but those in . One pass per sequence, as after an upload: a folder - /// renamed, deleted or damaged before its pass runs fails that pass alone, and the other sequences still get their - /// images. A sequence whose own pass of that kind is already queued is left out. + /// flag then, also when another writer listed them in the sidecar itself). After a resumed run, which may have indexed + /// sequences before the interruption, of every sequence of the archive without the flag but those in + /// . One pass per sequence, as after an upload: a folder renamed, deleted or damaged before + /// its pass runs fails that pass alone, and the other sequences still get their images. A sequence whose own pass of + /// that kind is already queued is left out. /// private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList indexed, bool resumed, IReadOnlySet awaiting) diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index 3ee1a531..92594446 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -195,6 +195,24 @@ public sealed class SequenceRepository await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1", [id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; + /// + /// Records decode tiers written to the sequence's frames (): the status + /// flags in and the preview-only count, in one statement. The frames, their versions and the + /// settings stay as they are, so an ingest or a settings edit committed meanwhile is never written back over. + /// + public async Task RecordTiersAsync(Guid id, SequenceStatus set, int previewOnlyCount, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE sequences SET status = status | $2, preview_only_count = $3, updated_at = $4 WHERE id = $1", + [id, (int)set, previewOnlyCount, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; + + /// + /// Records a verify: status flags set and cleared and the verify time, in one statement; every other field stays as + /// it is (). + /// + public async Task RecordVerifiedAsync(Guid id, SequenceStatus set, SequenceStatus clear, DateTimeOffset verifiedAt, + CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, verified_at = $4, updated_at = $5 WHERE id = $1", + [id, (int)set, (int)clear, verifiedAt, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; + /// /// Records an analysis: status flags set and cleared, the warning count and, for a deep analysis, the compact /// summary ( null keeps the stored one). diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs index 1d1aea42..5cbf7510 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs @@ -55,6 +55,20 @@ internal sealed class ArchiveFixture : IAsyncDisposable public Task ScanAsync(ScanMode mode = ScanMode.Incremental) => Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct); + /// + /// Another process writing the same archive, such as a desktop sharing a server's archive folder: its own index + /// (scanned) and archive service; disposed with the fixture. + /// + public async Task<(ArchiveService Service, Archive Archive)> OtherWriterAsync() + { + var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("other"), "titano.db")), Ct); + _owned.Add(db); + var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "other", LeaseWait = TimeSpan.FromSeconds(20) }); + var archive = await service.OpenAsync(Archive.RootPath, Ct); + await service.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, Ct); + return (service, archive); + } + /// A job engine with the archive handlers, started; disposed with the fixture. public async Task StartEngineAsync() { diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs index 04b6bdae..ddde498f 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -365,6 +365,38 @@ public sealed class ArchiveIngestTests(TestContext context) Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder); } + [Test] + public async Task ARebuildKeepsSequencesThatGetTheirFolderWhileItRuns() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2); + await fx.ScanAsync(); + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + + // Uploads go on while the rebuild runs: after it listed the folders, the sequence waiting for its first frame gets + // it, and a new upload session creates a sequence. + IngestResult? first = null, session = null; + var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, async progress => + { + if (progress.FoldersDone < progress.FoldersTotal || first is not null) return; + first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct); + session = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2), 2), sessionKey: "upload-2"), fx.Ct); + }, fx.Ct); + + // Neither is dropped, though the folders listed at the start hold neither. + Assert.Equal(0, rebuilt.Removed); + Assert.Equal(waiting.Id, Assert.NotNull(first).SequenceId); + var placed = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id)); + Assert.Equal(("2026/2026-09-29 Canon EOS R5", 1, "Harbour sunset"), (placed.Folder, placed.FrameCount, placed.Title)); + var created = Assert.NotNull(await fx.Db.Sequences.GetAsync(Assert.NotNull(session).SequenceId)); + Assert.Equal(1, created.FrameCount); + // The session's next upload lands in the same sequence, not in a second one beside it. + Assert.Equal(created.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "upload-2", fx.Ct)); + var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2).AddSeconds(10), 3), sessionKey: "upload-2"), fx.Ct); + Assert.Equal(created.Id, next.SequenceId); + Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count); + } + [Test] public async Task ADamagedRootSidecarIsRebuiltUnderTheRootSequenceNotOneWaitingForItsFirstFrame() { diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs index 8412d43f..4949fe7d 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs @@ -175,6 +175,101 @@ public sealed class ArchiveJobTests(TestContext context) Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies)); } + [Test] + public async Task AServerScanQueuesTheImagesOfFramesAnotherWriterListedInTheSidecarItself() + { + await using var fx = await CreateAsync(context); + Populate(fx); + await using var engine = ServerEngine(fx); + await engine.StartAsync(fx.Ct); + var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(scan.Id, JobState.Done); + var first = new HashSet(); + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items) + { + await fx.WaitForAsync(pass.Id, JobState.Done); + first.Add(pass.Id); + } + } + var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a"); + Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built"); + + // A desktop sharing the archive imports a photo taken before the others into the sequence: it files the photo and + // lists it in the sidecar itself, whose statuses it keeps, so the scan finds no photo to reconcile. + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddMinutes(-5), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + var sidecar = fx.Paths.Sequence("a").ManifestPath; + Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the other writer keeps the statuses"); + + var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(again.Id, JobState.Done); + + // The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind. + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id)); + Assert.SequenceEqual([a.Id], pass.TargetIds); + await fx.WaitForAsync(pass.Id, JobState.Done); + } + var row = (await fx.Db.Sequences.GetAsync(a.Id))!; + Assert.Equal(7, row.FrameCount); + Assert.True(Built(row.Status), "built again"); + // Every frame has its own thumbnail and proxy: the new first one, and the others it renumbered. + var folder = fx.Paths.Sequence("a"); + using var pack = ThumbPackReader.Open(folder.ThumbsPackPath); + foreach (var frame in ManifestStore.ReadSequence(sidecar).Frames) + { + Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry + && entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size)), $"thumbnail of {frame.File}"); + Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, null), $"proxy of {frame.File}"); + } + } + + [Test] + public async Task AnImagePassMarksTheSequenceBuiltOnlyWhenItsSidecarStillListsTheFramesThePassRead() + { + await using var fx = await CreateAsync(context); + // The pass commits its pack and checkpoints after 200 frames. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201); + await fx.ScanAsync(); + var a = (await LocationsAsync(fx)).Single(); + var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!; + + // While it runs, another writer copies in a photo taken before the others, and a scan reconciles it. + var scanned = false; + var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) => + { + if (scanned) return; + scanned = true; + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(-5), 1, firstSeed: 900, firstNumber: 900); + Assert.SequenceEqual([a.Id], (await fx.ScanAsync()).Sequences); + }, fx.Ct); + + // The pass drew every frame it read, but not the new photo: the sequence is not built. + Assert.True(scanned, "scanned during the pass"); + Assert.Equal((201, 0), (result.Built, result.Undecodable)); + var row = (await fx.Db.Sequences.GetAsync(a.Id))!; + Assert.Equal(202, row.FrameCount); + Assert.False(row.Status.HasFlag(SequenceStatus.ThumbsBuilt), row.Status.ToString()); + Assert.False(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag"); + + // So a scan's follow-up (here a resumed one, which lists the whole archive) queues the next pass, which makes them. + await using var engine = ServerEngine(fx); + var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a"); + await engine.StartAsync(fx.Ct); + await fx.WaitForAsync(resumed.Id, JobState.Done); + var pass = (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items.Single(); + Assert.SequenceEqual([a.Id], pass.TargetIds); + await fx.WaitForAsync(pass.Id, JobState.Done); + Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built"); + using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath); + Assert.Count(202, pack.Entries); + } + [Test] public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages() { diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs new file mode 100644 index 00000000..282f7e59 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -0,0 +1,141 @@ +using Titano.Core.Archiving; +using Titano.Core.Imaging; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Repositories; +using Titano.Data.Tests.Infrastructure; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +/// +/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify stamp) while a settings edit +/// or another writer commits, on both dialects: they write only what they own, under the sequence lease. +/// +[Category("archive")] +public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context) +{ + private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); + + /// One indexed sequence of three photos in a. + private static async Task IndexedAsync(ArchiveFixture fx) + { + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3); + await fx.ScanAsync(); + var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(); + return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!; + } + + /// Records decode tiers as analyses, image passes and renders do. + private static FrameTiers Tiers(ArchiveFixture fx) + { + var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }); + return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer)); + } + + /// The frame decodes only through its 48 × 32 embedded preview. + private static Dictionary Preview(Guid frameId) => + new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) }; + + /// Commits once the writer under test had time to read the row and wait for it. + private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken) + { + await using (tx) + { + await Task.Delay(500, cancellationToken); + await tx.CommitAsync(cancellationToken); + } + } + + [Test] + public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier. + Task record; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct); + Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome); + record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + await Task.WhenAny(record, Task.Delay(500, fx.Ct)); + Assert.False(record.IsCompleted, "the record waits for the row"); + await tx.CommitAsync(fx.Ct); + } + await record; + + // The tier and what follows from it are recorded, and the edit committed meanwhile stays. + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Contains("heroFrame", row.SettingsJson); + Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount)); + Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags); + } + + [Test] + public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet. + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + + // A tier recorded meanwhile changes the tiers here, not the frames this index holds. + await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount)); + + // The next scan still finds the folder changed, and indexes the photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion)); + Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); + } + + [Test] + public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // Hashed by a first verify, so the next one changes no frame and only stamps the sequence. + await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + + // The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence. + Task? commit = null; + await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) => + { + if (done < total || commit is not null) return; + var tx = await fx.Db.BeginAsync(fx.Ct); + await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct); + commit = CommitLaterAsync(tx, fx.Ct); + }, fx.Ct); + await Assert.NotNull(commit); + + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Contains("heroFrame", row.SettingsJson); + Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion); + Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString()); + Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again"); + } +} + +public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs index df1ac4ff..638a963c 100644 --- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs @@ -1,8 +1,11 @@ using System.Net; +using System.Security.Cryptography; using Microsoft.Extensions.DependencyInjection; using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; using Titano.Core.Model; +using Titano.Data; +using Titano.Data.Archiving; using Titano.Protocol; using Titano.Protocol.Client; using Titano.Server.Jobs; @@ -106,6 +109,45 @@ public sealed class ArchiveModuleTests(TestContext context) await UntilImagesAsync(server, editor, archive.Id, "garden"); } + [Test] + public async Task FramesADesktopSharingTheArchiveAddsToASequenceGetThumbnailsAndProxiesAfterThePoll() + { + await using var server = await TestServer.StartAsync(context, options => + { + Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof")); + File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0002.JPG"), Photo()); + }, ArchiveModule.Register); + var archive = server.Services.GetRequiredService(); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + await UntilIndexedAsync(server, archive.Id, "roof"); + await UntilScannedAsync(server); + await UntilImagesAsync(server, editor, archive.Id, "roof"); + await UntilIdleAsync(server); + var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof"); + + // A desktop sharing the archive, with an index of its own, imports a photo taken earlier into the sequence: it files + // the photo and lists it in the sidecar itself, keeping the sidecar's ThumbsBuilt and ProxiesBuilt. + var desktopFolder = context.CreateTempDirectory("desktop"); + await using (var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(desktopFolder, "titano.db")), context.Cancellation)) + { + var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop" }); + var shared = await desktop.OpenAsync(server.Options.ArchivePath, context.Cancellation); + await desktop.Indexer.ScanAsync(shared, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, context.Cancellation); + var photo = Photo(seed: 5); + var staged = Path.Combine(desktopFolder, "IMG_0001.JPG"); + await File.WriteAllBytesAsync(staged, photo); + await desktop.Ingest.CommitAsync(new IngestRequest(shared.Id, roof.Id, null, null, staged, "IMG_0001.JPG", photo.Length, + Convert.ToHexStringLower(SHA256.HashData(photo)), new DateTime(2026, 9, 28, 18, 0, 0), null), context.Cancellation); + } + Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount); + + // The next poll indexes it and queues the images: the new frame and the one it moved get theirs. + Assert.NotNull(await server.Services.GetRequiredService().PollAsync(context.Cancellation)); + await UntilScannedAsync(server); + Assert.Equal(2, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount); + await UntilImagesAsync(server, editor, archive.Id, "roof"); + } + [Test] public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel() { @@ -157,28 +199,51 @@ public sealed class ArchiveModuleTests(TestContext context) } } - /// Waits until the first frame of the sequence in has its thumbnail and proxy served. + /// + /// Waits until every frame of the sequence in has its thumbnail and proxy served and the + /// passes that made them have marked the sequence (a pass writes its files before it marks). + /// private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder) { var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder); - var frame = (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).First(); var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); - foreach (var asset in new[] { "thumb", "proxy" }) + foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id)) { - while (true) + foreach (var asset in new[] { "thumb", "proxy" }) { - using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}"); - if (response.StatusCode == HttpStatusCode.OK) + while (true) { - Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {folder}"); - break; + using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}"); + if (response.StatusCode == HttpStatusCode.OK) + { + Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {frame.FileName}"); + break; + } + Assert.True(DateTime.UtcNow < until, $"no {asset} of {frame.FileName} in {folder}: {(int)response.StatusCode}"); + await Task.Delay(100); } - Assert.True(DateTime.UtcNow < until, $"no {asset} of {folder}: {(int)response.StatusCode}"); - await Task.Delay(100); } } - var built = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status; - Assert.True(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt), $"{folder}: {built}"); + while (Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status is var built + && !(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt))) + { + Assert.True(DateTime.UtcNow < until, $"{folder}: {built}"); + await Task.Delay(50); + } + } + + /// Waits until no job is waiting or running. + private static async Task UntilIdleAsync(TestServer server) + { + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued }) + { + while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0) + { + Assert.True(DateTime.UtcNow < until, $"jobs still {state}"); + await Task.Delay(50); + } + } } /// Waits for the scan queued at start to finish: its sequences are indexed before it ends.