CI / Build and test (push) Successful in 11m36s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 12s
Release / Build, test and publish (push) Successful in 14m56s
CI / PostgreSQL 16 integration (push) Successful in 10m11s
CI / PostgreSQL 17 integration (push) Successful in 10m15s
CI / Docker image (push) Successful in 45s
Release / Server image (push) Successful in 4m55s
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
600 lines
35 KiB
C#
600 lines
35 KiB
C#
using Chronos.Core.Archiving;
|
||
using Chronos.Core.Imaging;
|
||
using Chronos.Core.Metadata;
|
||
using Chronos.Core.Model;
|
||
using Chronos.Data.Archiving;
|
||
using Chronos.Data.Repositories;
|
||
using Chronos.Data.Tests.Infrastructure;
|
||
using Chronos.Testing;
|
||
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
|
||
|
||
namespace Chronos.Data.Tests.Archiving;
|
||
|
||
/// <summary>
|
||
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
|
||
/// notes, a rename, an upload) while a settings edit, a scan or another writer commits, on both dialects: they write only
|
||
/// what they own, under the sequence lease, take the row before the frames, and leave what another process changed in
|
||
/// the sidecar to the next scan or index it whole.
|
||
/// </summary>
|
||
[Category("archive")]
|
||
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
|
||
{
|
||
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
|
||
|
||
/// <summary>One indexed sequence of three photos in <c>a</c>.</summary>
|
||
private static async Task<Sequence> IndexedAsync(ArchiveFixture fx)
|
||
{
|
||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
|
||
await fx.ScanAsync();
|
||
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single();
|
||
return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
|
||
}
|
||
|
||
/// <summary>Records decode tiers as analyses, image passes and renders do.</summary>
|
||
private static FrameTiers Tiers(ArchiveFixture fx)
|
||
{
|
||
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
||
return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
|
||
}
|
||
|
||
/// <summary>The frame decodes only through its 48 × 32 embedded preview.</summary>
|
||
private static Dictionary<Guid, FileTier> Preview(Guid frameId) =>
|
||
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
|
||
|
||
/// <summary>
|
||
/// Retitles the sequence in <paramref name="folder"/> in its sidecar alone, as another PC does: the next scan reads the
|
||
/// sidecar and indexes it without the lease. Returns the sidecar as written.
|
||
/// </summary>
|
||
private static SequenceManifest Retitle(SequencePaths folder)
|
||
{
|
||
var retitled = ManifestStore.ReadSequence(folder.ManifestPath) with { Title = "Ridge" };
|
||
ManifestStore.WriteSequence(folder.ManifestPath, retitled);
|
||
return retitled;
|
||
}
|
||
|
||
/// <summary>Commits <paramref name="tx"/> once the writer under test had time to read the row and wait for it.</summary>
|
||
private static async Task CommitLaterAsync(ChronosTransaction tx, CancellationToken cancellationToken)
|
||
{
|
||
await using (tx)
|
||
{
|
||
await Task.Delay(500, cancellationToken);
|
||
await tx.CommitAsync(cancellationToken);
|
||
}
|
||
}
|
||
|
||
[Test]
|
||
public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
|
||
// The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier.
|
||
Task record;
|
||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||
{
|
||
var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct);
|
||
Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome);
|
||
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
|
||
Assert.False(record.IsCompleted, "the record waits for the row");
|
||
await tx.CommitAsync(fx.Ct);
|
||
}
|
||
await record;
|
||
|
||
// The tier and what follows from it are recorded, and the edit committed meanwhile stays.
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Contains("heroFrame", row.SettingsJson);
|
||
Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount));
|
||
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
|
||
}
|
||
|
||
[Test]
|
||
public async Task TiersRecordedWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
|
||
// A scan's sync holds the row, as it does before it rewrites the frames, while an analysis records a tier.
|
||
Task record;
|
||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||
{
|
||
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
|
||
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
|
||
Assert.False(record.IsCompleted, "the record waits for the row");
|
||
// The record touched no frame yet, so the scan rewrites them all without waiting for it. Had it locked its frame
|
||
// before the row, each would wait for the other and PostgreSQL would abort one of them.
|
||
await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
|
||
await tx.CommitAsync(fx.Ct);
|
||
}
|
||
await record;
|
||
|
||
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags);
|
||
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
}
|
||
|
||
[Test]
|
||
public async Task LightStatsWrittenWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
|
||
// A scan's sync holds the row, as it does before it rewrites the frames, while a light analysis writes its statistics.
|
||
Task<int> write;
|
||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||
{
|
||
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
|
||
write = fx.Db.Frames.UpdateLightStatsAsync([.. frames.Select(f => new FrameLightStats(f.Id, -1.5, 0.01, 0.02))], fx.Ct);
|
||
await Task.WhenAny(write, Task.Delay(500, fx.Ct));
|
||
Assert.False(write.IsCompleted, "the statistics wait for the row");
|
||
// The statistics locked no frame yet, so the scan rewrites them all without waiting for them. Frames locked before
|
||
// the row, in another order than the scan's, would each wait for the other, and PostgreSQL would abort one.
|
||
await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
|
||
await tx.CommitAsync(fx.Ct);
|
||
}
|
||
Assert.Equal(frames.Count, await write);
|
||
|
||
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.MeanLogLuminance == -1.5), "written");
|
||
}
|
||
|
||
[Test]
|
||
public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
|
||
// A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
|
||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
|
||
// A tier recorded meanwhile changes the tiers here, not the frames this index holds.
|
||
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount));
|
||
|
||
// The next scan still finds the folder changed, and indexes the photo.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion));
|
||
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||
}
|
||
|
||
[Test]
|
||
public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
|
||
// A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index.
|
||
var (desktop, _) = await fx.OtherWriterAsync();
|
||
await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct);
|
||
|
||
// A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan.
|
||
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
|
||
// The rating goes into the sidecar and this index; the frames this index holds stay as they are.
|
||
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct);
|
||
Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating));
|
||
Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating);
|
||
|
||
// The next scan still finds the folder changed, and indexes the photo.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating));
|
||
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnUploadAfterAnotherWriterVerifiedTheSequenceIndexesWhatTheVerifyFound()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
// A desktop sharing the archive verifies the sequence: the photos' first hashes go into the sidecar and its own index.
|
||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||
await desktop.Verifier.VerifyAsync(archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||
|
||
// A photo uploaded here before this index's next scan is appended after the others.
|
||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
|
||
// The upload indexes the sequence whole from the sidecar, hashes and verify included, and leaves the folder in step.
|
||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
Assert.Count(4, frames);
|
||
Assert.True(frames.All(f => f.Sha256 is not null), "hashed");
|
||
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.Sha256)), frames.Select(f => (f.Id, f.Sha256)));
|
||
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified");
|
||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnUploadAfterAPhotoWasCopiedInByHandLeavesItToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
// Copied into the folder by hand, over a share: only a scan that reconciles the folder with its photos indexes it.
|
||
var folder = fx.Paths.Sequence("a");
|
||
await File.WriteAllBytesAsync(folder.File("IMG_0009.JPG"), Photos.Jpeg(Start.AddMinutes(2), 91), fx.Ct);
|
||
|
||
// A photo uploaded before this index's next scan is indexed with the sidecar's frames.
|
||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||
|
||
// The upload left the folder changed, so the next incremental scan reconciles it and indexes the copied photo.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
Assert.Count(5, frames);
|
||
Assert.Contains("IMG_0009.JPG", frames.Select(f => f.FileName));
|
||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
|
||
// The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again.
|
||
var calls = 0;
|
||
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info =>
|
||
{
|
||
if (++calls == 1)
|
||
{
|
||
var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct)
|
||
.GetAwaiter().GetResult();
|
||
Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome);
|
||
}
|
||
return info with { Rating = 5 };
|
||
}, fx.Ct);
|
||
|
||
Assert.Equal(2, calls);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Contains("heroFrame", row.SettingsJson);
|
||
Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion));
|
||
Assert.Equal(rated.SettingsVersion, row.SettingsVersion);
|
||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||
Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings));
|
||
}
|
||
|
||
[Test]
|
||
public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
|
||
// The rename moves the folder and retitles the sequence; the row keeps the frames this index holds.
|
||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct);
|
||
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion));
|
||
|
||
// The next scan still finds the folder changed, and indexes the photo where it is now.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath);
|
||
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion));
|
||
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)),
|
||
"frames in b");
|
||
}
|
||
|
||
[Test]
|
||
public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
// Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end.
|
||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513);
|
||
await fx.ScanAsync();
|
||
var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!;
|
||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||
var staged = fx.Stage(Start.AddHours(3), seed: 90);
|
||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
|
||
// The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end.
|
||
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default,
|
||
(_, _, _) => throw new OperationCanceledException(), null, fx.Ct));
|
||
Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null));
|
||
Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||
|
||
// The next scan still finds the folder changed, and indexes the photo.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||
}
|
||
|
||
[Test]
|
||
public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
// Two sequences holding the same photo, hashed by a verify.
|
||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
|
||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1);
|
||
await fx.ScanAsync();
|
||
await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||
var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1];
|
||
var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!;
|
||
|
||
// A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags.
|
||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||
var staged = fx.Stage(Start.AddHours(3), seed: 90);
|
||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.Equal(1, report.DuplicateFlagChanges);
|
||
Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||
|
||
// The next scan still finds the folder changed, and indexes the photo.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AVerifyKeepsTheTiersRecordedWhileItRan()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
|
||
// The first verify of photos a scan adopted hashes them all; a thumbnail pass records a preview-only frame meanwhile.
|
||
var recorded = false;
|
||
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async _ =>
|
||
{
|
||
if (recorded) return;
|
||
recorded = true;
|
||
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||
}, fx.Ct);
|
||
Assert.Equal(3L, report.Hashed);
|
||
|
||
// The sidecar and the index keep the tier's flag and count beside the hashes.
|
||
var indexed = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||
Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], indexed.Select(f => f.Flags));
|
||
Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], sidecar.Frames.Select(f => f.Flags));
|
||
Assert.True(indexed.All(f => f.Sha256 is not null), "hashed");
|
||
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Sha256), indexed.Select(f => f.Sha256));
|
||
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AVerifyLeavesAFrameAnUploadRepairedWhileItRanUnflagged()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
// Hashed by a first verify, so an upload of the same photo repairs the frame once its file is gone.
|
||
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||
var folder = fx.Paths.Sequence("a");
|
||
var lost = ManifestStore.ReadSequence(folder.ManifestPath).Frames[1];
|
||
File.Delete(folder.File(lost.File));
|
||
IngestRequest Upload((string Path, long Size, string Sha256) staged) =>
|
||
new(fx.Archive.Id, sequence.Id, null, null, staged.Path, lost.File, staged.Size, staged.Sha256, null, null);
|
||
|
||
// The verify finds the file missing, and an upload puts it back before the verify writes what it found.
|
||
var checkedFrames = 0;
|
||
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async _ =>
|
||
{
|
||
if (++checkedFrames != 2) return;
|
||
Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Repaired, "repaired");
|
||
}, fx.Ct);
|
||
Assert.SequenceEqual([(lost.Id, FrameHealth.Missing)], report.Problems.Select(p => (p.FrameId, p.Health)));
|
||
|
||
// The frame stays unflagged and the sequence verified, so the next copy is a duplicate and the file stays in place.
|
||
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(lost.Id, fx.Ct))!.Flags);
|
||
Assert.Equal(FrameFlags.None, ManifestStore.ReadSequence(folder.ManifestPath).Frames[1].Flags);
|
||
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified");
|
||
Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Duplicate, "duplicate");
|
||
Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*"));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AScanThatReadTheSidecarBeforeAnUploadCommittedReadsItAgainUnderTheLease()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var folder = fx.Paths.Sequence("a");
|
||
var read = Retitle(folder);
|
||
|
||
// The upload's transaction holds the row (and SQLite's write lock) when the scan, which read the sidecar, reaches it.
|
||
Task<ScanSummary> scan;
|
||
SequenceManifest uploaded;
|
||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||
{
|
||
var row = await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
|
||
scan = fx.ScanAsync();
|
||
await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
|
||
Assert.False(scan.IsCompleted, "the scan waits for the row");
|
||
// Under the lease, the upload filed its photo and wrote the sidecar; this transaction indexes them.
|
||
var photo = folder.File("IMG_0004.JPG");
|
||
await File.WriteAllBytesAsync(photo, Photos.Jpeg(Start.AddMinutes(1), 90), fx.Ct);
|
||
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), "IMG_0004.JPG", "IMG_0004.JPG", new FileInfo(photo).Length, null,
|
||
MetadataReader.Read(photo, fx.Ct));
|
||
uploaded = SequenceFrames.Insert(read, [frame]);
|
||
ManifestStore.WriteSequence(folder.ManifestPath, uploaded);
|
||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(uploaded, fx.Archive.Id, folder, row), fx.Ct);
|
||
await tx.Frames.BulkInsertAsync([SequenceMapping.ToFrame(uploaded.Frames[^1], sequence.Id, folder)], fx.Ct);
|
||
await tx.CommitAsync(fx.Ct);
|
||
}
|
||
Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
|
||
|
||
// The scan wrote nothing from its older read: the photo, its frames version and the title are indexed.
|
||
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal((4, uploaded.Versions.Frames, "Ridge"), (indexed.FrameCount, indexed.FramesVersion, indexed.Title));
|
||
Assert.SequenceEqual(uploaded.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AScanThatReadTheSidecarBeforeATierWasRecordedReadsItAgainUnderTheLease()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var folder = fx.Paths.Sequence("a");
|
||
var read = Retitle(folder);
|
||
|
||
// The tier writer's transaction holds the row when the scan, which read the sidecar, reaches it.
|
||
Task<ScanSummary> scan;
|
||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||
{
|
||
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
|
||
scan = fx.ScanAsync();
|
||
await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
|
||
Assert.False(scan.IsCompleted, "the scan waits for the row");
|
||
// Under the lease, the writer gave the first frame its preview tier in the sidecar; this transaction indexes it.
|
||
var tier = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32);
|
||
var tiered = SequenceFrames.Update(read, new Dictionary<Guid, Func<SidecarFrame, SidecarFrame>> { [read.Frames[0].Id] = f => SequenceFrames.WithTier(f, tier) });
|
||
ManifestStore.WriteSequence(folder.ManifestPath, tiered);
|
||
var first = tiered.Frames[0];
|
||
await tx.Sequences.RecordTiersAsync(sequence.Id, SequenceStatus.None, 1, fx.Ct);
|
||
await tx.Frames.UpdateTiersAsync([new FrameTierFields(first.Id, first.Flags, first.PreviewW ?? 0, first.PreviewH ?? 0, first.Format)], fx.Ct);
|
||
await tx.CommitAsync(fx.Ct);
|
||
}
|
||
Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
|
||
|
||
// The scan wrote nothing from its older read: the tier stays, and the title is indexed.
|
||
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(("Ridge", 1), (indexed.Title, indexed.PreviewOnlyCount));
|
||
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
|
||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnOlderSidecarPutBackIsIndexedPastTheFramesVersionTheIndexHolds()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var folder = fx.Paths.Sequence("a");
|
||
var older = await File.ReadAllBytesAsync(folder.ManifestPath, fx.Ct);
|
||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
var uploaded = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
|
||
// A file-sync tool puts the sidecar back as it was before the upload; the photo stays.
|
||
await File.WriteAllBytesAsync(folder.ManifestPath, older, fx.Ct);
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
|
||
// Indexed as the sidecar lists it, one frames version past the index's, in both places.
|
||
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal((3, uploaded.FramesVersion + 1), (indexed.FrameCount, indexed.FramesVersion));
|
||
Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
|
||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AFramesVersionAnotherIndexReachedForOtherFramesIsIndexedPastIt()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
var folder = fx.Paths.Sequence("a");
|
||
async Task<Sequence> UploadAsync(string name, int minute)
|
||
{
|
||
var staged = fx.Stage(Start.AddMinutes(minute), seed: 90 + minute);
|
||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, name, staged.Size, staged.Sha256, null, null),
|
||
fx.Ct);
|
||
return (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
}
|
||
await UploadAsync("IMG_0004.JPG", 1);
|
||
// A desktop sharing this server's archive, with its own index of the four photos.
|
||
await using var desktopDb = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(Context.CreateTempDirectory("desktop"), "chronos.db")), fx.Ct);
|
||
var desktop = new ArchiveService(desktopDb, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
|
||
var desktopArchive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
|
||
Task<ScanSummary> DesktopScanAsync() => desktop.Indexer.ScanAsync(desktopArchive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
|
||
await DesktopScanAsync();
|
||
|
||
// The server takes a fifth photo before the desktop scans again.
|
||
var uploaded = await UploadAsync("IMG_0005.JPG", 2);
|
||
Assert.Equal(uploaded.FramesVersion - 1, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion);
|
||
|
||
// The sidecar is damaged; the desktop's scan rebuilds it from the photos at the first version, the fifth photo under
|
||
// an id of its own, and writes it one version past its own row: the version the server holds for the photo as it
|
||
// filed it.
|
||
await File.WriteAllTextAsync(folder.ManifestPath, "{", fx.Ct);
|
||
Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences);
|
||
var rebuilt = ManifestStore.ReadSequence(folder.ManifestPath);
|
||
Assert.Equal(uploaded.FramesVersion, rebuilt.Versions.Frames);
|
||
Assert.NotEqual((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[^1].Id, rebuilt.Frames[^1].Id);
|
||
|
||
// The server's scan finds other frames at the version it holds: it indexes them one version past it, in both places,
|
||
// so an editor holding that version for the photo as it was is refused.
|
||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Equal(uploaded.FramesVersion + 1, indexed.FramesVersion);
|
||
Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
|
||
Assert.SequenceEqual(rebuilt.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||
var edit = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Rating = 2 }, null, uploaded.FramesVersion, fx.Ct);
|
||
Assert.Equal(SequenceEditOutcome.Conflict, edit.Outcome);
|
||
|
||
// The desktop follows, and neither index finds anything more to do.
|
||
Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences);
|
||
Assert.Equal(indexed.FramesVersion, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion);
|
||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||
Assert.Empty((await DesktopScanAsync()).Sequences);
|
||
}
|
||
|
||
[Test]
|
||
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
|
||
{
|
||
var fx = await FixtureAsync();
|
||
var sequence = await IndexedAsync(fx);
|
||
// Hashed by a first verify, so the next one changes no frame and only stamps the sequence.
|
||
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||
sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
|
||
// The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence.
|
||
Task? commit = null;
|
||
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async p =>
|
||
{
|
||
if (p.BytesDone < p.BytesTotal || commit is not null) return;
|
||
var tx = await fx.Db.BeginAsync(fx.Ct);
|
||
await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct);
|
||
commit = CommitLaterAsync(tx, fx.Ct);
|
||
}, fx.Ct);
|
||
await Assert.NotNull(commit);
|
||
|
||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||
Assert.Contains("heroFrame", row.SettingsJson);
|
||
Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion);
|
||
Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString());
|
||
Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again");
|
||
}
|
||
}
|
||
|
||
public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
|
||
{
|
||
protected override Backend Backend => Backend.SqliteFile;
|
||
}
|
||
|
||
[RequiresEnvironment(PgTestServer.Variable)]
|
||
public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
|
||
{
|
||
protected override Backend Backend => Backend.Postgres;
|
||
}
|