From ed5c0c71c8a581ec56549d92d7e2b3b1c7ec5eb1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 16:29:32 +0000 Subject: [PATCH] fix(archive): an unreadable root sidecar names no sequence waiting for its first frame The index's own predicate for sequences created without photos reads the archive root's sidecar once per call and treats a damaged or unreadable one as naming nobody. A torn root sidecar is therefore rebuilt under the root-adopted sequence instead of a later POST /sequences row, and a sidecar the server may not read fails the root folder alone instead of every scan, rebuild, orphan check and scan follow-up. Maintenance leaves these rows out of temporary-file purging and pack compaction, so the root's thumbs.pack is compacted under the leases of the root's own sequence. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/ArchiveIndexer.cs | 32 ++++- .../Archiving/ArchiveMaintenance.cs | 6 +- .../Archiving/ArchiveIngestTests.cs | 117 ++++++++++++++++++ 3 files changed, 149 insertions(+), 6 deletions(-) diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index ece6e735..f8cfe10e 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -219,21 +219,43 @@ public sealed class ArchiveIndexer } /// - /// The sequences among created without photos that still wait for their first frame - /// (). Their empty folder is not the archive root: scans do not adopt the root - /// under their id, a rebuild keeps them although they have no sidecar yet, and orphan checks do not count them missing. + /// The sequences among created without photos that still wait for their first frame: no + /// folder, no frames, and not the sequence the archive root's sidecar names. Their empty folder is not the archive + /// root: scans do not adopt or rebuild the root under their id, a rebuild keeps them although they have no sidecar yet, + /// orphan checks do not count them missing, and maintenance leaves the root's files to the root's own sequence. Unlike + /// , whose commit reports it, a root sidecar that cannot be read names nobody + /// here: a scan rebuilds a damaged one under the id of the sequence that holds the root's frames, and one it may not + /// read fails the root folder alone rather than the whole scan or rebuild. /// internal async Task> AwaitingFolderAsync(ArchivePaths paths, IEnumerable locations, CancellationToken cancellationToken) { var awaiting = new HashSet(); + (Guid? Id, bool Read) root = (null, false); foreach (var location in locations.Where(l => l.Folder.Length == 0)) { - if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is { } sequence && ArchiveIngest.AwaitsFolder(sequence, paths)) - awaiting.Add(sequence.Id); + if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is not { Folder.Length: 0, FrameCount: 0 } sequence) continue; + // Read once, and only when a row may be waiting: a root sidecar can list a hundred thousand frames. + if (!root.Read) root = (RootSidecarId(paths), true); + if (root.Id != sequence.Id) awaiting.Add(sequence.Id); } return awaiting; } + /// The id the archive root's sidecar names; null when there is none or it cannot be read. + private static Guid? RootSidecarId(ArchivePaths paths) + { + var sidecar = paths.Sequence(string.Empty).ManifestPath; + try + { + return File.Exists(sidecar) ? ManifestStore.ReadSequence(sidecar).Id : null; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + // Indexing the root reports it (a damaged sidecar is rebuilt, an unreadable one fails that folder alone). + return null; + } + } + /// The index holds a project for this sequence and other frames than the sidecar: its positions may move. private async Task MayMovePositionsAsync(SequenceManifest manifest, CancellationToken cancellationToken) => await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { SettingsJson: not null } indexed diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs index 6a75cde9..e9c921c3 100644 --- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -123,7 +123,11 @@ public sealed class ArchiveMaintenance { var paths = new ArchivePaths(archive.RootPath); if (!dryRun) ArchiveVolume.EnsureWritable(paths); - var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + // A sequence waiting for its first frame has no folder yet: the archive root's pack is compacted under the leases of + // the root's own sequence, which its thumbnail passes hold, never under another id's. + var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false); + var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList(); IReadOnlyList leases = [], temps = [], packs = []; int groups = 0, flagChanges = 0; long? reclaimable = null; diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs index c881a068..76033c7a 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -1,3 +1,4 @@ +using System.Runtime.Versioning; using System.Text.Json.Nodes; using Titano.Core.Archiving; using Titano.Core.IO; @@ -35,6 +36,36 @@ public sealed class ArchiveIngestTests(TestContext context) return created; } + /// + /// Makes unreadable until disposed: without read permission where that stops the test process + /// (UnauthorizedAccessException), otherwise (Windows, root) held open without sharing (IOException). + /// + private static IDisposable Unreadable(string path) + { + if (!OperatingSystem.IsWindows()) + { + var mode = File.GetUnixFileMode(path); + File.SetUnixFileMode(path, UnixFileMode.None); + try + { + File.OpenRead(path).Dispose(); + File.SetUnixFileMode(path, mode); + } + catch (UnauthorizedAccessException) + { + return new UnixPermissions(path, mode); + } + } + return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None); + } + + /// Gives a file its permissions back. + [UnsupportedOSPlatform("windows")] + private sealed class UnixPermissions(string path, UnixFileMode mode) : IDisposable + { + public void Dispose() => File.SetUnixFileMode(path, mode); + } + [Test] public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates() { @@ -334,6 +365,92 @@ public sealed class ArchiveIngestTests(TestContext context) Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder); } + [Test] + public async Task ADamagedRootSidecarIsRebuiltUnderTheRootSequenceNotOneWaitingForItsFirstFrame() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var root = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault()); + // Created later, so its id sorts after the root sequence's; then another tool tears the root sidecar. + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + var sidecar = fx.Paths.Sequence(string.Empty).ManifestPath; + File.WriteAllText(sidecar, "{"); + + var summary = await fx.ScanAsync(); + + Assert.Empty(summary.Failed); + Assert.Equal(root.Id, ManifestStore.ReadSequence(sidecar).Id); + Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(sidecar)!, $"{Path.GetFileName(sidecar)}.damaged-*")); + Assert.Equal(2, (await fx.Db.Sequences.GetAsync(root.Id))!.FrameCount); + var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal((string.Empty, 0, "Harbour sunset"), (row.Folder, row.FrameCount, row.Title)); + Assert.Empty(await fx.Db.Frames.ListBySequenceAsync(waiting.Id)); + } + + [Test] + public async Task ARootSidecarThatCannotBeReadFailsTheRootFolderAloneAndDropsNoSequence() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var root = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault()); + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start.AddDays(1), 2, firstSeed: 50); + var sidecar = fx.Paths.Sequence(string.Empty).ManifestPath; + + ScanSummary rebuilt; + using (Unreadable(sidecar)) rebuilt = await fx.ScanAsync(ScanMode.Rebuild); + + Assert.SequenceEqual([string.Empty], rebuilt.Failed.Select(f => f.Folder)); + Assert.Equal(0, rebuilt.Removed); + var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct); + Assert.SequenceEqual(new[] { root.Id, waiting.Id }.Order(), locations.Where(l => l.Folder.Length == 0).Select(l => l.Id).Order()); + Assert.SequenceEqual(["Trips/Comet"], locations.Where(l => l.Folder.Length > 0).Select(l => l.Folder)); + + // Readable again, the root is indexed under its own sequence. + Assert.Empty((await fx.ScanAsync()).Failed); + Assert.Equal(root.Id, ManifestStore.ReadSequence(sidecar).Id); + var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!; + Assert.Equal((string.Empty, 0), (row.Folder, row.FrameCount)); + } + + [Test] + public async Task MaintenanceCompactsTheRootPackUnderTheRootSequencesLeaseOnly() + { + await using var fx = await CreateAsync(context); + // Left over from before the root was adopted, so its id sorts before the root sequence's. + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var root = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(l => l.Id != waiting.Id); + var packPath = fx.Paths.Sequence(string.Empty).ThumbsPackPath; + var hash = new byte[ThumbPack.HashLength]; + using (var pack = ThumbPackWriter.Open(packPath)) + { + // The same entry written again and again: everything but the last copy is garbage. + for (var i = 0; i < 4; i++) pack.Append(0, 48, 32, hash, Photos.Jpeg(Start, 70 + i)); + pack.Commit(); + } + var length = new FileInfo(packPath).Length; + + // A thumbnail pass of the root sequence appends under its own lease: the pack waits for it. + Task maintenance; + using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{root.Id:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) + { + maintenance = fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); + await Task.Delay(500, fx.Ct); + Assert.False(maintenance.IsCompleted); + Assert.Equal(length, new FileInfo(packPath).Length); + } + + var report = await maintenance; + Assert.SequenceEqual([string.Empty], report.CompactedPacks); + using var compacted = ThumbPackReader.Open(packPath); + Assert.Equal(0L, compacted.GarbageBytes); + Assert.NotNull(compacted.Read(0)); + } + [Test] public async Task FramesInsertedBeforeOthersTakeTheProjectPositionsAlong() {