using Chronos.Core.Archiving; using Chronos.Core.Imaging; using Chronos.Core.Metadata; using Chronos.Core.Model; using Chronos.Data.Archiving; using Chronos.Data.Repositories; using Chronos.Data.Tests.Infrastructure; using Chronos.Testing; using static Chronos.Data.Tests.Archiving.ArchiveFixture; namespace Chronos.Data.Tests.Archiving; /// /// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and /// notes, a rename, an upload) while a settings edit, a scan or another writer commits, on both dialects: they write only /// what they own, under the sequence lease, take the row before the frames, and leave what another process changed in /// the sidecar to the next scan or index it whole. /// [Category("archive")] public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context) { private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); /// One indexed sequence of three photos in a. private static async Task IndexedAsync(ArchiveFixture fx) { Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3); await fx.ScanAsync(); var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(); return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!; } /// Records decode tiers as analyses, image passes and renders do. private static FrameTiers Tiers(ArchiveFixture fx) { var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }); return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer)); } /// The frame decodes only through its 48 × 32 embedded preview. private static Dictionary Preview(Guid frameId) => new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) }; /// /// Retitles the sequence in in its sidecar alone, as another PC does: the next scan reads the /// sidecar and indexes it without the lease. Returns the sidecar as written. /// private static SequenceManifest Retitle(SequencePaths folder) { var retitled = ManifestStore.ReadSequence(folder.ManifestPath) with { Title = "Ridge" }; ManifestStore.WriteSequence(folder.ManifestPath, retitled); return retitled; } /// Commits once the writer under test had time to read the row and wait for it. private static async Task CommitLaterAsync(ChronosTransaction tx, CancellationToken cancellationToken) { await using (tx) { await Task.Delay(500, cancellationToken); await tx.CommitAsync(cancellationToken); } } [Test] public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); // The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier. Task record; await using (var tx = await fx.Db.BeginAsync(fx.Ct)) { var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct); Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome); record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); await Task.WhenAny(record, Task.Delay(500, fx.Ct)); Assert.False(record.IsCompleted, "the record waits for the row"); await tx.CommitAsync(fx.Ct); } await record; // The tier and what follows from it are recorded, and the edit committed meanwhile stays. var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Contains("heroFrame", row.SettingsJson); Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount)); Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags); } [Test] public async Task TiersRecordedWhileAScanHoldsTheRowWaitForItBeforeTheFrames() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); // A scan's sync holds the row, as it does before it rewrites the frames, while an analysis records a tier. Task record; await using (var tx = await fx.Db.BeginAsync(fx.Ct)) { await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); await Task.WhenAny(record, Task.Delay(500, fx.Ct)); Assert.False(record.IsCompleted, "the record waits for the row"); // The record touched no frame yet, so the scan rewrites them all without waiting for it. Had it locked its frame // before the row, each would wait for the other and PostgreSQL would abort one of them. await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct); await tx.CommitAsync(fx.Ct); } await record; Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags); Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount); } [Test] public async Task LightStatsWrittenWhileAScanHoldsTheRowWaitForItBeforeTheFrames() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); // A scan's sync holds the row, as it does before it rewrites the frames, while a light analysis writes its statistics. Task write; await using (var tx = await fx.Db.BeginAsync(fx.Ct)) { await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); write = fx.Db.Frames.UpdateLightStatsAsync([.. frames.Select(f => new FrameLightStats(f.Id, -1.5, 0.01, 0.02))], fx.Ct); await Task.WhenAny(write, Task.Delay(500, fx.Ct)); Assert.False(write.IsCompleted, "the statistics wait for the row"); // The statistics locked no frame yet, so the scan rewrites them all without waiting for them. Frames locked before // the row, in another order than the scan's, would each wait for the other, and PostgreSQL would abort one. await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct); await tx.CommitAsync(fx.Ct); } Assert.Equal(frames.Count, await write); Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.MeanLogLuminance == -1.5), "written"); } [Test] public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); // A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet. var (desktop, archive) = await fx.OtherWriterAsync(); var staged = fx.Stage(Start.AddMinutes(1), seed: 90); await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); // A tier recorded meanwhile changes the tiers here, not the frames this index holds. await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount)); // The next scan still finds the folder changed, and indexes the photo. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion)); Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); } [Test] public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); // A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index. var (desktop, _) = await fx.OtherWriterAsync(); await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct); // A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan. await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount)); } [Test] public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var (desktop, archive) = await fx.OtherWriterAsync(); var staged = fx.Stage(Start.AddMinutes(1), seed: 90); await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); // The rating goes into the sidecar and this index; the frames this index holds stay as they are. var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct); Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating)); Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating); // The next scan still finds the folder changed, and indexes the photo. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating)); Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); } [Test] public async Task AnUploadAfterAnotherWriterVerifiedTheSequenceIndexesWhatTheVerifyFound() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); // A desktop sharing the archive verifies the sequence: the photos' first hashes go into the sidecar and its own index. var (desktop, archive) = await fx.OtherWriterAsync(); await desktop.Verifier.VerifyAsync(archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); // A photo uploaded here before this index's next scan is appended after the others. var staged = fx.Stage(Start.AddMinutes(1), seed: 90); await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); // The upload indexes the sequence whole from the sidecar, hashes and verify included, and leaves the folder in step. var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); Assert.Count(4, frames); Assert.True(frames.All(f => f.Sha256 is not null), "hashed"); Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.Sha256)), frames.Select(f => (f.Id, f.Sha256))); Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified"); Assert.Empty((await fx.ScanAsync()).Sequences); } [Test] public async Task AnUploadAfterAPhotoWasCopiedInByHandLeavesItToTheNextScan() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); // Copied into the folder by hand, over a share: only a scan that reconciles the folder with its photos indexes it. var folder = fx.Paths.Sequence("a"); await File.WriteAllBytesAsync(folder.File("IMG_0009.JPG"), Photos.Jpeg(Start.AddMinutes(2), 91), fx.Ct); // A photo uploaded before this index's next scan is indexed with the sidecar's frames. var staged = fx.Stage(Start.AddMinutes(1), seed: 90); await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); // The upload left the folder changed, so the next incremental scan reconciles it and indexes the copied photo. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); Assert.Count(5, frames); Assert.Contains("IMG_0009.JPG", frames.Select(f => f.FileName)); Assert.Empty((await fx.ScanAsync()).Sequences); } [Test] public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); // The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again. var calls = 0; var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => { if (++calls == 1) { var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct) .GetAwaiter().GetResult(); Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome); } return info with { Rating = 5 }; }, fx.Ct); Assert.Equal(2, calls); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Contains("heroFrame", row.SettingsJson); Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion)); Assert.Equal(rated.SettingsVersion, row.SettingsVersion); var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings)); } [Test] public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var (desktop, archive) = await fx.OtherWriterAsync(); var staged = fx.Stage(Start.AddMinutes(1), seed: 90); await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); // The rename moves the folder and retitles the sequence; the row keeps the frames this index holds. var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct); await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion)); // The next scan still finds the folder changed, and indexes the photo where it is now. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath); row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion)); Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)), "frames in b"); } [Test] public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan() { var fx = await FixtureAsync(); // Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end. Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513); await fx.ScanAsync(); var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!; var (desktop, archive) = await fx.OtherWriterAsync(); var staged = fx.Stage(Start.AddHours(3), seed: 90); await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); // The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end. await Assert.ThrowsAsync(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => throw new OperationCanceledException(), null, fx.Ct)); Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null)); Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); // The next scan still finds the folder changed, and indexes the photo. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); } [Test] public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan() { var fx = await FixtureAsync(); // Two sequences holding the same photo, hashed by a verify. Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3); Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1); await fx.ScanAsync(); await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1]; var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!; // A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags. var (desktop, archive) = await fx.OtherWriterAsync(); var staged = fx.Stage(Start.AddHours(3), seed: 90); await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct); Assert.Equal(1, report.DuplicateFlagChanges); Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); // The next scan still finds the folder changed, and indexes the photo. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); } [Test] public async Task AVerifyKeepsTheTiersRecordedWhileItRan() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); // The first verify of photos a scan adopted hashes them all; a thumbnail pass records a preview-only frame meanwhile. var recorded = false; var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) => { if (recorded) return; recorded = true; await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); }, fx.Ct); Assert.Equal(3L, report.Hashed); // The sidecar and the index keep the tier's flag and count beside the hashes. var indexed = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], indexed.Select(f => f.Flags)); Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], sidecar.Frames.Select(f => f.Flags)); Assert.True(indexed.All(f => f.Sha256 is not null), "hashed"); Assert.SequenceEqual(sidecar.Frames.Select(f => f.Sha256), indexed.Select(f => f.Sha256)); Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount); } [Test] public async Task AVerifyLeavesAFrameAnUploadRepairedWhileItRanUnflagged() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); // Hashed by a first verify, so an upload of the same photo repairs the frame once its file is gone. await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); var folder = fx.Paths.Sequence("a"); var lost = ManifestStore.ReadSequence(folder.ManifestPath).Frames[1]; File.Delete(folder.File(lost.File)); IngestRequest Upload((string Path, long Size, string Sha256) staged) => new(fx.Archive.Id, sequence.Id, null, null, staged.Path, lost.File, staged.Size, staged.Sha256, null, null); // The verify finds the file missing, and an upload puts it back before the verify writes what it found. var checkedFrames = 0; var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) => { if (++checkedFrames != 2) return; Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Repaired, "repaired"); }, fx.Ct); Assert.SequenceEqual([(lost.Id, FrameHealth.Missing)], report.Problems.Select(p => (p.FrameId, p.Health))); // The frame stays unflagged and the sequence verified, so the next copy is a duplicate and the file stays in place. Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(lost.Id, fx.Ct))!.Flags); Assert.Equal(FrameFlags.None, ManifestStore.ReadSequence(folder.ManifestPath).Frames[1].Flags); Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified"); Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Duplicate, "duplicate"); Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*")); } [Test] public async Task AScanThatReadTheSidecarBeforeAnUploadCommittedReadsItAgainUnderTheLease() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var folder = fx.Paths.Sequence("a"); var read = Retitle(folder); // The upload's transaction holds the row (and SQLite's write lock) when the scan, which read the sidecar, reaches it. Task scan; SequenceManifest uploaded; await using (var tx = await fx.Db.BeginAsync(fx.Ct)) { var row = await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); scan = fx.ScanAsync(); await Task.WhenAny(scan, Task.Delay(500, fx.Ct)); Assert.False(scan.IsCompleted, "the scan waits for the row"); // Under the lease, the upload filed its photo and wrote the sidecar; this transaction indexes them. var photo = folder.File("IMG_0004.JPG"); await File.WriteAllBytesAsync(photo, Photos.Jpeg(Start.AddMinutes(1), 90), fx.Ct); var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), "IMG_0004.JPG", "IMG_0004.JPG", new FileInfo(photo).Length, null, MetadataReader.Read(photo, fx.Ct)); uploaded = SequenceFrames.Insert(read, [frame]); ManifestStore.WriteSequence(folder.ManifestPath, uploaded); await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(uploaded, fx.Archive.Id, folder, row), fx.Ct); await tx.Frames.BulkInsertAsync([SequenceMapping.ToFrame(uploaded.Frames[^1], sequence.Id, folder)], fx.Ct); await tx.CommitAsync(fx.Ct); } Assert.SequenceEqual([sequence.Id], (await scan).Sequences); // The scan wrote nothing from its older read: the photo, its frames version and the title are indexed. var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal((4, uploaded.Versions.Frames, "Ridge"), (indexed.FrameCount, indexed.FramesVersion, indexed.Title)); Assert.SequenceEqual(uploaded.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); Assert.Empty((await fx.ScanAsync()).Sequences); } [Test] public async Task AScanThatReadTheSidecarBeforeATierWasRecordedReadsItAgainUnderTheLease() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var folder = fx.Paths.Sequence("a"); var read = Retitle(folder); // The tier writer's transaction holds the row when the scan, which read the sidecar, reaches it. Task scan; await using (var tx = await fx.Db.BeginAsync(fx.Ct)) { await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); scan = fx.ScanAsync(); await Task.WhenAny(scan, Task.Delay(500, fx.Ct)); Assert.False(scan.IsCompleted, "the scan waits for the row"); // Under the lease, the writer gave the first frame its preview tier in the sidecar; this transaction indexes it. var tier = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32); var tiered = SequenceFrames.Update(read, new Dictionary> { [read.Frames[0].Id] = f => SequenceFrames.WithTier(f, tier) }); ManifestStore.WriteSequence(folder.ManifestPath, tiered); var first = tiered.Frames[0]; await tx.Sequences.RecordTiersAsync(sequence.Id, SequenceStatus.None, 1, fx.Ct); await tx.Frames.UpdateTiersAsync([new FrameTierFields(first.Id, first.Flags, first.PreviewW ?? 0, first.PreviewH ?? 0, first.Format)], fx.Ct); await tx.CommitAsync(fx.Ct); } Assert.SequenceEqual([sequence.Id], (await scan).Sequences); // The scan wrote nothing from its older read: the tier stays, and the title is indexed. var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal(("Ridge", 1), (indexed.Title, indexed.PreviewOnlyCount)); Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags); Assert.Empty((await fx.ScanAsync()).Sequences); } [Test] public async Task AnOlderSidecarPutBackIsIndexedPastTheFramesVersionTheIndexHolds() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var folder = fx.Paths.Sequence("a"); var older = await File.ReadAllBytesAsync(folder.ManifestPath, fx.Ct); var staged = fx.Stage(Start.AddMinutes(1), seed: 90); await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), fx.Ct); var uploaded = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; // A file-sync tool puts the sidecar back as it was before the upload; the photo stays. await File.WriteAllBytesAsync(folder.ManifestPath, older, fx.Ct); Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); // Indexed as the sidecar lists it, one frames version past the index's, in both places. var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal((3, uploaded.FramesVersion + 1), (indexed.FrameCount, indexed.FramesVersion)); Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames); Assert.Empty((await fx.ScanAsync()).Sequences); } [Test] public async Task AFramesVersionAnotherIndexReachedForOtherFramesIsIndexedPastIt() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); var folder = fx.Paths.Sequence("a"); async Task UploadAsync(string name, int minute) { var staged = fx.Stage(Start.AddMinutes(minute), seed: 90 + minute); await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, name, staged.Size, staged.Sha256, null, null), fx.Ct); return (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; } await UploadAsync("IMG_0004.JPG", 1); // A desktop sharing this server's archive, with its own index of the four photos. await using var desktopDb = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(Context.CreateTempDirectory("desktop"), "chronos.db")), fx.Ct); var desktop = new ArchiveService(desktopDb, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) }); var desktopArchive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct); Task DesktopScanAsync() => desktop.Indexer.ScanAsync(desktopArchive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct); await DesktopScanAsync(); // The server takes a fifth photo before the desktop scans again. var uploaded = await UploadAsync("IMG_0005.JPG", 2); Assert.Equal(uploaded.FramesVersion - 1, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion); // The sidecar is damaged; the desktop's scan rebuilds it from the photos at the first version, the fifth photo under // an id of its own, and writes it one version past its own row: the version the server holds for the photo as it // filed it. await File.WriteAllTextAsync(folder.ManifestPath, "{", fx.Ct); Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences); var rebuilt = ManifestStore.ReadSequence(folder.ManifestPath); Assert.Equal(uploaded.FramesVersion, rebuilt.Versions.Frames); Assert.NotEqual((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[^1].Id, rebuilt.Frames[^1].Id); // The server's scan finds other frames at the version it holds: it indexes them one version past it, in both places, // so an editor holding that version for the photo as it was is refused. Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal(uploaded.FramesVersion + 1, indexed.FramesVersion); Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames); Assert.SequenceEqual(rebuilt.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); var edit = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Rating = 2 }, null, uploaded.FramesVersion, fx.Ct); Assert.Equal(SequenceEditOutcome.Conflict, edit.Outcome); // The desktop follows, and neither index finds anything more to do. Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences); Assert.Equal(indexed.FramesVersion, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion); Assert.Empty((await fx.ScanAsync()).Sequences); Assert.Empty((await DesktopScanAsync()).Sequences); } [Test] public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() { var fx = await FixtureAsync(); var sequence = await IndexedAsync(fx); // Hashed by a first verify, so the next one changes no frame and only stamps the sequence. await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; // The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence. Task? commit = null; await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) => { if (done < total || commit is not null) return; var tx = await fx.Db.BeginAsync(fx.Ct); await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct); commit = CommitLaterAsync(tx, fx.Ct); }, fx.Ct); await Assert.NotNull(commit); var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Contains("heroFrame", row.SettingsJson); Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion); Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString()); Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again"); } } public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context) { protected override Backend Backend => Backend.SqliteFile; } [RequiresEnvironment(PgTestServer.Variable)] public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context) { protected override Backend Backend => Backend.Postgres; }