using Chronos.Core.Archiving;
using Chronos.Core.Imaging;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Repositories;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
///
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
/// notes, a rename, an upload) while a settings edit, a scan or another writer commits, on both dialects: they write only
/// what they own, under the sequence lease, take the row before the frames, and leave what another process changed in
/// the sidecar to the next scan or index it whole.
///
[Category("archive")]
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
{
private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
/// One indexed sequence of three photos in a.
private static async Task IndexedAsync(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
await fx.ScanAsync();
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single();
return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
}
/// Records decode tiers as analyses, image passes and renders do.
private static FrameTiers Tiers(ArchiveFixture fx)
{
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
}
/// The frame decodes only through its 48 × 32 embedded preview.
private static Dictionary Preview(Guid frameId) =>
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
///
/// Retitles the sequence in in its sidecar alone, as another PC does: the next scan reads the
/// sidecar and indexes it without the lease. Returns the sidecar as written.
///
private static SequenceManifest Retitle(SequencePaths folder)
{
var retitled = ManifestStore.ReadSequence(folder.ManifestPath) with { Title = "Ridge" };
ManifestStore.WriteSequence(folder.ManifestPath, retitled);
return retitled;
}
/// Commits once the writer under test had time to read the row and wait for it.
private static async Task CommitLaterAsync(ChronosTransaction tx, CancellationToken cancellationToken)
{
await using (tx)
{
await Task.Delay(500, cancellationToken);
await tx.CommitAsync(cancellationToken);
}
}
[Test]
public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier.
Task record;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct);
Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome);
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
Assert.False(record.IsCompleted, "the record waits for the row");
await tx.CommitAsync(fx.Ct);
}
await record;
// The tier and what follows from it are recorded, and the edit committed meanwhile stays.
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount));
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
}
[Test]
public async Task TiersRecordedWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A scan's sync holds the row, as it does before it rewrites the frames, while an analysis records a tier.
Task record;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
Assert.False(record.IsCompleted, "the record waits for the row");
// The record touched no frame yet, so the scan rewrites them all without waiting for it. Had it locked its frame
// before the row, each would wait for the other and PostgreSQL would abort one of them.
await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
await tx.CommitAsync(fx.Ct);
}
await record;
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags);
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
}
[Test]
public async Task LightStatsWrittenWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A scan's sync holds the row, as it does before it rewrites the frames, while a light analysis writes its statistics.
Task write;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
write = fx.Db.Frames.UpdateLightStatsAsync([.. frames.Select(f => new FrameLightStats(f.Id, -1.5, 0.01, 0.02))], fx.Ct);
await Task.WhenAny(write, Task.Delay(500, fx.Ct));
Assert.False(write.IsCompleted, "the statistics wait for the row");
// The statistics locked no frame yet, so the scan rewrites them all without waiting for them. Frames locked before
// the row, in another order than the scan's, would each wait for the other, and PostgreSQL would abort one.
await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
await tx.CommitAsync(fx.Ct);
}
Assert.Equal(frames.Count, await write);
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.MeanLogLuminance == -1.5), "written");
}
[Test]
public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// A tier recorded meanwhile changes the tiers here, not the frames this index holds.
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount));
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion));
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index.
var (desktop, _) = await fx.OtherWriterAsync();
await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct);
// A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan.
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount));
}
[Test]
public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rating goes into the sidecar and this index; the frames this index holds stay as they are.
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct);
Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating));
Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating));
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task AnUploadAfterAnotherWriterVerifiedTheSequenceIndexesWhatTheVerifyFound()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// A desktop sharing the archive verifies the sequence: the photos' first hashes go into the sidecar and its own index.
var (desktop, archive) = await fx.OtherWriterAsync();
await desktop.Verifier.VerifyAsync(archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
// A photo uploaded here before this index's next scan is appended after the others.
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The upload indexes the sequence whole from the sidecar, hashes and verify included, and leaves the folder in step.
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
Assert.Count(4, frames);
Assert.True(frames.All(f => f.Sha256 is not null), "hashed");
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.Sha256)), frames.Select(f => (f.Id, f.Sha256)));
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified");
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AnUploadAfterAPhotoWasCopiedInByHandLeavesItToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// Copied into the folder by hand, over a share: only a scan that reconciles the folder with its photos indexes it.
var folder = fx.Paths.Sequence("a");
await File.WriteAllBytesAsync(folder.File("IMG_0009.JPG"), Photos.Jpeg(Start.AddMinutes(2), 91), fx.Ct);
// A photo uploaded before this index's next scan is indexed with the sidecar's frames.
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The upload left the folder changed, so the next incremental scan reconciles it and indexes the copied photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
Assert.Count(5, frames);
Assert.Contains("IMG_0009.JPG", frames.Select(f => f.FileName));
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again.
var calls = 0;
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info =>
{
if (++calls == 1)
{
var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct)
.GetAwaiter().GetResult();
Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome);
}
return info with { Rating = 5 };
}, fx.Ct);
Assert.Equal(2, calls);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion));
Assert.Equal(rated.SettingsVersion, row.SettingsVersion);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings));
}
[Test]
public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rename moves the folder and retitles the sequence; the row keeps the frames this index holds.
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct);
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion));
// The next scan still finds the folder changed, and indexes the photo where it is now.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion));
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)),
"frames in b");
}
[Test]
public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan()
{
var fx = await FixtureAsync();
// Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513);
await fx.ScanAsync();
var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!;
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end.
await Assert.ThrowsAsync(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default,
(_, _, _) => throw new OperationCanceledException(), null, fx.Ct));
Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null));
Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
// Two sequences holding the same photo, hashed by a verify.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1);
await fx.ScanAsync();
await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1];
var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!;
// A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(1, report.DuplicateFlagChanges);
Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task AVerifyKeepsTheTiersRecordedWhileItRan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// The first verify of photos a scan adopted hashes them all; a thumbnail pass records a preview-only frame meanwhile.
var recorded = false;
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) =>
{
if (recorded) return;
recorded = true;
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
}, fx.Ct);
Assert.Equal(3L, report.Hashed);
// The sidecar and the index keep the tier's flag and count beside the hashes.
var indexed = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], indexed.Select(f => f.Flags));
Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], sidecar.Frames.Select(f => f.Flags));
Assert.True(indexed.All(f => f.Sha256 is not null), "hashed");
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Sha256), indexed.Select(f => f.Sha256));
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
}
[Test]
public async Task AVerifyLeavesAFrameAnUploadRepairedWhileItRanUnflagged()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// Hashed by a first verify, so an upload of the same photo repairs the frame once its file is gone.
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
var folder = fx.Paths.Sequence("a");
var lost = ManifestStore.ReadSequence(folder.ManifestPath).Frames[1];
File.Delete(folder.File(lost.File));
IngestRequest Upload((string Path, long Size, string Sha256) staged) =>
new(fx.Archive.Id, sequence.Id, null, null, staged.Path, lost.File, staged.Size, staged.Sha256, null, null);
// The verify finds the file missing, and an upload puts it back before the verify writes what it found.
var checkedFrames = 0;
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) =>
{
if (++checkedFrames != 2) return;
Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Repaired, "repaired");
}, fx.Ct);
Assert.SequenceEqual([(lost.Id, FrameHealth.Missing)], report.Problems.Select(p => (p.FrameId, p.Health)));
// The frame stays unflagged and the sequence verified, so the next copy is a duplicate and the file stays in place.
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(lost.Id, fx.Ct))!.Flags);
Assert.Equal(FrameFlags.None, ManifestStore.ReadSequence(folder.ManifestPath).Frames[1].Flags);
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified");
Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Duplicate, "duplicate");
Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*"));
}
[Test]
public async Task AScanThatReadTheSidecarBeforeAnUploadCommittedReadsItAgainUnderTheLease()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var folder = fx.Paths.Sequence("a");
var read = Retitle(folder);
// The upload's transaction holds the row (and SQLite's write lock) when the scan, which read the sidecar, reaches it.
Task scan;
SequenceManifest uploaded;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
var row = await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
scan = fx.ScanAsync();
await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
Assert.False(scan.IsCompleted, "the scan waits for the row");
// Under the lease, the upload filed its photo and wrote the sidecar; this transaction indexes them.
var photo = folder.File("IMG_0004.JPG");
await File.WriteAllBytesAsync(photo, Photos.Jpeg(Start.AddMinutes(1), 90), fx.Ct);
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), "IMG_0004.JPG", "IMG_0004.JPG", new FileInfo(photo).Length, null,
MetadataReader.Read(photo, fx.Ct));
uploaded = SequenceFrames.Insert(read, [frame]);
ManifestStore.WriteSequence(folder.ManifestPath, uploaded);
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(uploaded, fx.Archive.Id, folder, row), fx.Ct);
await tx.Frames.BulkInsertAsync([SequenceMapping.ToFrame(uploaded.Frames[^1], sequence.Id, folder)], fx.Ct);
await tx.CommitAsync(fx.Ct);
}
Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
// The scan wrote nothing from its older read: the photo, its frames version and the title are indexed.
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, uploaded.Versions.Frames, "Ridge"), (indexed.FrameCount, indexed.FramesVersion, indexed.Title));
Assert.SequenceEqual(uploaded.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AScanThatReadTheSidecarBeforeATierWasRecordedReadsItAgainUnderTheLease()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var folder = fx.Paths.Sequence("a");
var read = Retitle(folder);
// The tier writer's transaction holds the row when the scan, which read the sidecar, reaches it.
Task scan;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
scan = fx.ScanAsync();
await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
Assert.False(scan.IsCompleted, "the scan waits for the row");
// Under the lease, the writer gave the first frame its preview tier in the sidecar; this transaction indexes it.
var tier = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32);
var tiered = SequenceFrames.Update(read, new Dictionary> { [read.Frames[0].Id] = f => SequenceFrames.WithTier(f, tier) });
ManifestStore.WriteSequence(folder.ManifestPath, tiered);
var first = tiered.Frames[0];
await tx.Sequences.RecordTiersAsync(sequence.Id, SequenceStatus.None, 1, fx.Ct);
await tx.Frames.UpdateTiersAsync([new FrameTierFields(first.Id, first.Flags, first.PreviewW ?? 0, first.PreviewH ?? 0, first.Format)], fx.Ct);
await tx.CommitAsync(fx.Ct);
}
Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
// The scan wrote nothing from its older read: the tier stays, and the title is indexed.
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", 1), (indexed.Title, indexed.PreviewOnlyCount));
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AnOlderSidecarPutBackIsIndexedPastTheFramesVersionTheIndexHolds()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var folder = fx.Paths.Sequence("a");
var older = await File.ReadAllBytesAsync(folder.ManifestPath, fx.Ct);
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var uploaded = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
// A file-sync tool puts the sidecar back as it was before the upload; the photo stays.
await File.WriteAllBytesAsync(folder.ManifestPath, older, fx.Ct);
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
// Indexed as the sidecar lists it, one frames version past the index's, in both places.
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((3, uploaded.FramesVersion + 1), (indexed.FrameCount, indexed.FramesVersion));
Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AFramesVersionAnotherIndexReachedForOtherFramesIsIndexedPastIt()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var folder = fx.Paths.Sequence("a");
async Task UploadAsync(string name, int minute)
{
var staged = fx.Stage(Start.AddMinutes(minute), seed: 90 + minute);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, name, staged.Size, staged.Sha256, null, null),
fx.Ct);
return (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
}
await UploadAsync("IMG_0004.JPG", 1);
// A desktop sharing this server's archive, with its own index of the four photos.
await using var desktopDb = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(Context.CreateTempDirectory("desktop"), "chronos.db")), fx.Ct);
var desktop = new ArchiveService(desktopDb, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
var desktopArchive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
Task DesktopScanAsync() => desktop.Indexer.ScanAsync(desktopArchive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
await DesktopScanAsync();
// The server takes a fifth photo before the desktop scans again.
var uploaded = await UploadAsync("IMG_0005.JPG", 2);
Assert.Equal(uploaded.FramesVersion - 1, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion);
// The sidecar is damaged; the desktop's scan rebuilds it from the photos at the first version, the fifth photo under
// an id of its own, and writes it one version past its own row: the version the server holds for the photo as it
// filed it.
await File.WriteAllTextAsync(folder.ManifestPath, "{", fx.Ct);
Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences);
var rebuilt = ManifestStore.ReadSequence(folder.ManifestPath);
Assert.Equal(uploaded.FramesVersion, rebuilt.Versions.Frames);
Assert.NotEqual((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[^1].Id, rebuilt.Frames[^1].Id);
// The server's scan finds other frames at the version it holds: it indexes them one version past it, in both places,
// so an editor holding that version for the photo as it was is refused.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(uploaded.FramesVersion + 1, indexed.FramesVersion);
Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
Assert.SequenceEqual(rebuilt.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
var edit = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Rating = 2 }, null, uploaded.FramesVersion, fx.Ct);
Assert.Equal(SequenceEditOutcome.Conflict, edit.Outcome);
// The desktop follows, and neither index finds anything more to do.
Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences);
Assert.Equal(indexed.FramesVersion, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion);
Assert.Empty((await fx.ScanAsync()).Sequences);
Assert.Empty((await DesktopScanAsync()).Sequences);
}
[Test]
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// Hashed by a first verify, so the next one changes no frame and only stamps the sequence.
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
// The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence.
Task? commit = null;
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) =>
{
if (done < total || commit is not null) return;
var tx = await fx.Db.BeginAsync(fx.Ct);
await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct);
commit = CommitLaterAsync(tx, fx.Ct);
}, fx.Ct);
await Assert.NotNull(commit);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion);
Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString());
Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again");
}
}
public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
{
protected override Backend Backend => Backend.Postgres;
}