FrameTiers.RecordAsync updated the frames of a sequence before its row, while a scan's sync locks the row (FOR UPDATE) before rewriting every frame. On PostgreSQL the two deadlocked and one job was aborted with 40P01. The tier writer now records the row first; the verify's integrity flags (next commit) and maintenance's duplicate flags take the row lock in the same transaction as their frames, so every writer of a sequence's frames follows one order. A PostgreSQL test that deadlocked with the old order covers it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
314 lines
17 KiB
C#
314 lines
17 KiB
C#
using Titano.Core.Archiving;
|
||
using Titano.Core.Imaging;
|
||
using Titano.Core.Model;
|
||
using Titano.Data.Archiving;
|
||
using Titano.Data.Repositories;
|
||
using Titano.Data.Tests.Infrastructure;
|
||
using Titano.Testing;
|
||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||
|
||
namespace Titano.Data.Tests.Archiving;
|
||
|
||
/// <summary>
|
||
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
|
||
/// notes, a rename, an upload) while a settings edit, a scan or another writer commits, on both dialects: they write only
|
||
/// what they own, under the sequence lease, take the row before the frames, and leave what another process changed in
|
||
/// the sidecar to the next scan or index it whole.
|
||
/// </summary>
|
||
[Category("archive")]
|
||
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
|
||
{
|
||
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
|
||
|
||
/// <summary>One indexed sequence of three photos in <c>a</c>.</summary>
|
||
private static async Task<Sequence> IndexedAsync(ArchiveFixture fx)
|
||
{
|
||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
|
||
await fx.ScanAsync();
|
||
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single();
|
||
return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
|
||
}
|
||
|
||
/// <summary>Records decode tiers as analyses, image passes and renders do.</summary>
|
||
private static FrameTiers Tiers(ArchiveFixture fx)
|
||
{
|
||
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
||
return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
|
||
}
|
||
|
||
/// <summary>The frame decodes only through its 48 × 32 embedded preview.</summary>
|
||
private static Dictionary<Guid, FileTier> Preview(Guid frameId) =>
|
||
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
|
||
|
||
/// <summary>Commits <paramref name="tx"/> once the writer under test had time to read the row and wait for it.</summary>
|
||
private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken)
|
||
{
|
||
await using (tx)
|
||
{
|
||
await Task.Delay(500, cancellationToken);
|
||
await tx.CommitAsync(cancellationToken);
|
||
}
|
||
}
|
||
|
||
[Test]
|
||
public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
|
||
// The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier.
|
||
Task record;
|
||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||
{
|
||
var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct);
|
||
Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome);
|
||
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
|
||
Assert.False(record.IsCompleted, "the record waits for the row");
|
||
await tx.CommitAsync(fx.Ct);
|
||
}
|
||
await record;
|
||
|
||
// The tier and what follows from it are recorded, and the edit committed meanwhile stays.
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Contains("heroFrame", row.SettingsJson);
|
||
Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount));
|
||
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
|
||
}
|
||
|
||
[Test]
|
||
public async Task TiersRecordedWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
|
||
// A scan's sync holds the row, as it does before it rewrites the frames, while an analysis records a tier.
|
||
Task record;
|
||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||
{
|
||
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
|
||
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
|
||
Assert.False(record.IsCompleted, "the record waits for the row");
|
||
// The record touched no frame yet, so the scan rewrites them all without waiting for it. Had it locked its frame
|
||
// before the row, each would wait for the other and PostgreSQL would abort one of them.
|
||
await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
|
||
await tx.CommitAsync(fx.Ct);
|
||
}
|
||
await record;
|
||
|
||
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags);
|
||
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
}
|
||
|
||
[Test]
|
||
public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
|
||
// A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
|
||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
|
||
// A tier recorded meanwhile changes the tiers here, not the frames this index holds.
|
||
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount));
|
||
|
||
// The next scan still finds the folder changed, and indexes the photo.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion));
|
||
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||
}
|
||
|
||
[Test]
|
||
public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
|
||
// A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index.
|
||
var (desktop, _) = await fx.OtherWriterAsync();
|
||
await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct);
|
||
|
||
// A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan.
|
||
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
|
||
// The rating goes into the sidecar and this index; the frames this index holds stay as they are.
|
||
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct);
|
||
Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating));
|
||
Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating);
|
||
|
||
// The next scan still finds the folder changed, and indexes the photo.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating));
|
||
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
|
||
// The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again.
|
||
var calls = 0;
|
||
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info =>
|
||
{
|
||
if (++calls == 1)
|
||
{
|
||
var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct)
|
||
.GetAwaiter().GetResult();
|
||
Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome);
|
||
}
|
||
return info with { Rating = 5 };
|
||
}, fx.Ct);
|
||
|
||
Assert.Equal(2, calls);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Contains("heroFrame", row.SettingsJson);
|
||
Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion));
|
||
Assert.Equal(rated.SettingsVersion, row.SettingsVersion);
|
||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||
Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings));
|
||
}
|
||
|
||
[Test]
|
||
public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
|
||
// The rename moves the folder and retitles the sequence; the row keeps the frames this index holds.
|
||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct);
|
||
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion));
|
||
|
||
// The next scan still finds the folder changed, and indexes the photo where it is now.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath);
|
||
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion));
|
||
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)),
|
||
"frames in b");
|
||
}
|
||
|
||
[Test]
|
||
public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
// Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end.
|
||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513);
|
||
await fx.ScanAsync();
|
||
var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!;
|
||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||
var staged = fx.Stage(Start.AddHours(3), seed: 90);
|
||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
|
||
// The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end.
|
||
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default,
|
||
(_, _, _) => throw new OperationCanceledException(), null, fx.Ct));
|
||
Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null));
|
||
Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||
|
||
// The next scan still finds the folder changed, and indexes the photo.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||
}
|
||
|
||
[Test]
|
||
public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
// Two sequences holding the same photo, hashed by a verify.
|
||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
|
||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1);
|
||
await fx.ScanAsync();
|
||
await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||
var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1];
|
||
var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!;
|
||
|
||
// A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags.
|
||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||
var staged = fx.Stage(Start.AddHours(3), seed: 90);
|
||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.Equal(1, report.DuplicateFlagChanges);
|
||
Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||
|
||
// The next scan still finds the folder changed, and indexes the photo.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
// Hashed by a first verify, so the next one changes no frame and only stamps the sequence.
|
||
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||
sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
|
||
// The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence.
|
||
Task? commit = null;
|
||
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) =>
|
||
{
|
||
if (done < total || commit is not null) return;
|
||
var tx = await fx.Db.BeginAsync(fx.Ct);
|
||
await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct);
|
||
commit = CommitLaterAsync(tx, fx.Ct);
|
||
}, fx.Ct);
|
||
await Assert.NotNull(commit);
|
||
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Contains("heroFrame", row.SettingsJson);
|
||
Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion);
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString());
|
||
Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again");
|
||
}
|
||
}
|
||
|
||
public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
|
||
{
|
||
protected override Backend Backend => Backend.SqliteFile;
|
||
}
|
||
|
||
[RequiresEnvironment(PgTestServer.Variable)]
|
||
public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
|
||
{
|
||
protected override Backend Backend => Backend.Postgres;
|
||
}
|