Files
Chronos/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
T
Claude e369d1d20a fix(archive): lock the sequence row before its frames in every index write
FrameTiers.RecordAsync updated the frames of a sequence before its row,
while a scan's sync locks the row (FOR UPDATE) before rewriting every
frame. On PostgreSQL the two deadlocked and one job was aborted with
40P01. The tier writer now records the row first; the verify's integrity
flags (next commit) and maintenance's duplicate flags take the row lock
in the same transaction as their frames, so every writer of a sequence's
frames follows one order. A PostgreSQL test that deadlocked with the old
order covers it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 21:34:08 +00:00

314 lines
17 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Titano.Core.Archiving;
using Titano.Core.Imaging;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Repositories;
using Titano.Data.Tests.Infrastructure;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
/// <summary>
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
/// notes, a rename, an upload) while a settings edit, a scan or another writer commits, on both dialects: they write only
/// what they own, under the sequence lease, take the row before the frames, and leave what another process changed in
/// the sidecar to the next scan or index it whole.
/// </summary>
[Category("archive")]
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
{
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
/// <summary>One indexed sequence of three photos in <c>a</c>.</summary>
private static async Task<Sequence> IndexedAsync(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
await fx.ScanAsync();
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single();
return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
}
/// <summary>Records decode tiers as analyses, image passes and renders do.</summary>
private static FrameTiers Tiers(ArchiveFixture fx)
{
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
}
/// <summary>The frame decodes only through its 48 × 32 embedded preview.</summary>
private static Dictionary<Guid, FileTier> Preview(Guid frameId) =>
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
/// <summary>Commits <paramref name="tx"/> once the writer under test had time to read the row and wait for it.</summary>
private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken)
{
await using (tx)
{
await Task.Delay(500, cancellationToken);
await tx.CommitAsync(cancellationToken);
}
}
[Test]
public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier.
Task record;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct);
Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome);
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
Assert.False(record.IsCompleted, "the record waits for the row");
await tx.CommitAsync(fx.Ct);
}
await record;
// The tier and what follows from it are recorded, and the edit committed meanwhile stays.
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount));
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
}
[Test]
public async Task TiersRecordedWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A scan's sync holds the row, as it does before it rewrites the frames, while an analysis records a tier.
Task record;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
Assert.False(record.IsCompleted, "the record waits for the row");
// The record touched no frame yet, so the scan rewrites them all without waiting for it. Had it locked its frame
// before the row, each would wait for the other and PostgreSQL would abort one of them.
await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
await tx.CommitAsync(fx.Ct);
}
await record;
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags);
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
}
[Test]
public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// A tier recorded meanwhile changes the tiers here, not the frames this index holds.
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount));
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion));
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index.
var (desktop, _) = await fx.OtherWriterAsync();
await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct);
// A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan.
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount));
}
[Test]
public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rating goes into the sidecar and this index; the frames this index holds stay as they are.
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct);
Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating));
Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating));
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again.
var calls = 0;
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info =>
{
if (++calls == 1)
{
var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct)
.GetAwaiter().GetResult();
Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome);
}
return info with { Rating = 5 };
}, fx.Ct);
Assert.Equal(2, calls);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion));
Assert.Equal(rated.SettingsVersion, row.SettingsVersion);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings));
}
[Test]
public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rename moves the folder and retitles the sequence; the row keeps the frames this index holds.
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct);
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion));
// The next scan still finds the folder changed, and indexes the photo where it is now.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion));
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)),
"frames in b");
}
[Test]
public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan()
{
var fx = await FixtureAsync();
// Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513);
await fx.ScanAsync();
var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!;
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end.
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default,
(_, _, _) => throw new OperationCanceledException(), null, fx.Ct));
Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null));
Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
// Two sequences holding the same photo, hashed by a verify.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1);
await fx.ScanAsync();
await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1];
var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!;
// A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(1, report.DuplicateFlagChanges);
Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// Hashed by a first verify, so the next one changes no frame and only stamps the sequence.
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
// The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence.
Task? commit = null;
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) =>
{
if (done < total || commit is not null) return;
var tx = await fx.Db.BeginAsync(fx.Ct);
await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct);
commit = CommitLaterAsync(tx, fx.Ct);
}, fx.Ct);
await Assert.NotNull(commit);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion);
Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString());
Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again");
}
}
public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
{
protected override Backend Backend => Backend.Postgres;
}