using Titano.Core.Archiving; using Titano.Core.Imaging; using Titano.Core.Model; using Titano.Data.Archiving; using Titano.Data.Repositories; using Titano.Data.Tests.Infrastructure; using Titano.Testing; using static Titano.Data.Tests.Archiving.ArchiveFixture; namespace Titano.Data.Tests.Archiving; /// /// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and /// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own, /// under the sequence lease, and leave what another process changed in the sidecar to the next scan. /// [Category("archive")] public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context) { private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); /// One indexed sequence of three photos in a. private static async Task IndexedAsync(ArchiveFixture fx) { Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3); await fx.ScanAsync(); var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(); return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!; } /// Records decode tiers as analyses, image passes and renders do. private static FrameTiers Tiers(ArchiveFixture fx) { var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }); return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer)); } /// The frame decodes only through its 48 × 32 embedded preview. private static Dictionary Preview(Guid frameId) => new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) }; /// Commits once the writer under test had time to read the row and wait for it. private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken) { await using (tx) { await Task.Delay(500, cancellationToken); await tx.CommitAsync(cancellationToken); } } [Test] public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); // The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier. Task record; await using (var tx = await fx.Db.BeginAsync(fx.Ct)) { var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct); Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome); record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); await Task.WhenAny(record, Task.Delay(500, fx.Ct)); Assert.False(record.IsCompleted, "the record waits for the row"); await tx.CommitAsync(fx.Ct); } await record; // The tier and what follows from it are recorded, and the edit committed meanwhile stays. var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Contains("heroFrame", row.SettingsJson); Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount)); Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags); } [Test] public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); // A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet. var (desktop, archive) = await fx.OtherWriterAsync(); var staged = fx.Stage(Start.AddMinutes(1), seed: 90); await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); // A tier recorded meanwhile changes the tiers here, not the frames this index holds. await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount)); // The next scan still finds the folder changed, and indexes the photo. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion)); Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); } [Test] public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); // A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index. var (desktop, _) = await fx.OtherWriterAsync(); await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct); // A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan. await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount)); } [Test] public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var (desktop, archive) = await fx.OtherWriterAsync(); var staged = fx.Stage(Start.AddMinutes(1), seed: 90); await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); // The rating goes into the sidecar and this index; the frames this index holds stay as they are. var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct); Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating)); Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating); // The next scan still finds the folder changed, and indexes the photo. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating)); Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); } [Test] public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); // The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again. var calls = 0; var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => { if (++calls == 1) { var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct) .GetAwaiter().GetResult(); Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome); } return info with { Rating = 5 }; }, fx.Ct); Assert.Equal(2, calls); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Contains("heroFrame", row.SettingsJson); Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion)); Assert.Equal(rated.SettingsVersion, row.SettingsVersion); var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings)); } [Test] public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var (desktop, archive) = await fx.OtherWriterAsync(); var staged = fx.Stage(Start.AddMinutes(1), seed: 90); await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); // The rename moves the folder and retitles the sequence; the row keeps the frames this index holds. var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct); await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion)); // The next scan still finds the folder changed, and indexes the photo where it is now. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath); row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion)); Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)), "frames in b"); } [Test] public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan() { var fx = await FixtureAsync(); // Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end. Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513); await fx.ScanAsync(); var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!; var (desktop, archive) = await fx.OtherWriterAsync(); var staged = fx.Stage(Start.AddHours(3), seed: 90); await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); // The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end. await Assert.ThrowsAsync(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => throw new OperationCanceledException(), null, fx.Ct)); Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null)); Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); // The next scan still finds the folder changed, and indexes the photo. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); } [Test] public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan() { var fx = await FixtureAsync(); // Two sequences holding the same photo, hashed by a verify. Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3); Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1); await fx.ScanAsync(); await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1]; var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!; // A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags. var (desktop, archive) = await fx.OtherWriterAsync(); var staged = fx.Stage(Start.AddHours(3), seed: 90); await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct); Assert.Equal(1, report.DuplicateFlagChanges); Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); // The next scan still finds the folder changed, and indexes the photo. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); } [Test] public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); // Hashed by a first verify, so the next one changes no frame and only stamps the sequence. await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; // The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence. Task? commit = null; await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) => { if (done < total || commit is not null) return; var tx = await fx.Db.BeginAsync(fx.Ct); await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct); commit = CommitLaterAsync(tx, fx.Ct); }, fx.Ct); await Assert.NotNull(commit); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Contains("heroFrame", row.SettingsJson); Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion); Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString()); Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again"); } } public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context) { protected override Backend Backend => Backend.SqliteFile; } [RequiresEnvironment(PgTestServer.Variable)] public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context) { protected override Backend Backend => Backend.Postgres; }