Files
Chronos/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
T
Claude 94ef3c88db fix(archive): partial writers keep the folder changed for the scan; image flags follow queued passes and hashed photos
Writers that change part of a sequence (decode tiers, image-pass flags,
analyses, verify flushes and stamps, duplicate flags, title/tags/rating/
notes edits, mass rename) now ask ArchiveIndexer.InStepAsync before their
own sidecar write and record the folder's scan stamps afterwards only when
the index held the folder as it was: the stored stamps are still the
folder's and the row holds the sidecar's frames. What another process
changed in the sidecar alone (frames, a new title) stays visible to the
next incremental scan, which writes the whole row from the sidecar.

EditSequenceInfoAsync reads sidecar and row under the lease and writes the
row through SequenceRepository.EditAsync at the settings version it read,
applying the edit again on a conflict, then the sidecar from the stored
row; settings and frames committed meanwhile stay. MassRenamer writes only
folder and title (RelocateAsync, whose null title now keeps the current
one). Verify flushes and duplicate flags run under one lease.

A scan that finds frames another writer listed in the sidecar itself
clears AnalyzedLight, ThumbsBuilt and ProxiesBuilt only in its own row and
only on a server (ImagesAfterScan); the shared sidecar keeps the writer's
flags, so a desktop's poll no longer makes the server redo its passes.

An image pass overtaken by an upload keeps its flag while a pass of the
same kind is queued for the sequence. ThumbPack.IsFrom accepts a thumbnail
drawn before a verify hashed its photo, in the pass, in SameFrames and in
the server's thumbnail lookup, which answered 404 for every such frame.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 20:16:58 +00:00

287 lines
16 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Titano.Core.Archiving;
using Titano.Core.Imaging;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Repositories;
using Titano.Data.Tests.Infrastructure;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
/// <summary>
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
/// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own,
/// under the sequence lease, and leave what another process changed in the sidecar to the next scan.
/// </summary>
[Category("archive")]
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
{
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
/// <summary>One indexed sequence of three photos in <c>a</c>.</summary>
private static async Task<Sequence> IndexedAsync(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
await fx.ScanAsync();
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single();
return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
}
/// <summary>Records decode tiers as analyses, image passes and renders do.</summary>
private static FrameTiers Tiers(ArchiveFixture fx)
{
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
}
/// <summary>The frame decodes only through its 48 × 32 embedded preview.</summary>
private static Dictionary<Guid, FileTier> Preview(Guid frameId) =>
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
/// <summary>Commits <paramref name="tx"/> once the writer under test had time to read the row and wait for it.</summary>
private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken)
{
await using (tx)
{
await Task.Delay(500, cancellationToken);
await tx.CommitAsync(cancellationToken);
}
}
[Test]
public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier.
Task record;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct);
Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome);
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
Assert.False(record.IsCompleted, "the record waits for the row");
await tx.CommitAsync(fx.Ct);
}
await record;
// The tier and what follows from it are recorded, and the edit committed meanwhile stays.
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount));
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
}
[Test]
public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// A tier recorded meanwhile changes the tiers here, not the frames this index holds.
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount));
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion));
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index.
var (desktop, _) = await fx.OtherWriterAsync();
await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct);
// A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan.
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount));
}
[Test]
public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rating goes into the sidecar and this index; the frames this index holds stay as they are.
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct);
Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating));
Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating));
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again.
var calls = 0;
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info =>
{
if (++calls == 1)
{
var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct)
.GetAwaiter().GetResult();
Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome);
}
return info with { Rating = 5 };
}, fx.Ct);
Assert.Equal(2, calls);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion));
Assert.Equal(rated.SettingsVersion, row.SettingsVersion);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings));
}
[Test]
public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rename moves the folder and retitles the sequence; the row keeps the frames this index holds.
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct);
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion));
// The next scan still finds the folder changed, and indexes the photo where it is now.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion));
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)),
"frames in b");
}
[Test]
public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan()
{
var fx = await FixtureAsync();
// Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513);
await fx.ScanAsync();
var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!;
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end.
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default,
(_, _, _) => throw new OperationCanceledException(), null, fx.Ct));
Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null));
Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
// Two sequences holding the same photo, hashed by a verify.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1);
await fx.ScanAsync();
await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1];
var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!;
// A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(1, report.DuplicateFlagChanges);
Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// Hashed by a first verify, so the next one changes no frame and only stamps the sequence.
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
// The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence.
Task? commit = null;
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) =>
{
if (done < total || commit is not null) return;
var tx = await fx.Db.BeginAsync(fx.Ct);
await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct);
commit = CommitLaterAsync(tx, fx.Ct);
}, fx.Ct);
await Assert.NotNull(commit);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion);
Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString());
Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again");
}
}
public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
{
protected override Backend Backend => Backend.Postgres;
}