Files
Chronos/tests/Chronos.Data.Tests/Archiving/PartialRowWriterTests.cs
T
Alby96andClaude Opus 5.5 f2b58522b9
CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
refactor!: rename Titano to Chronos and move CI and releases to Gitea (#6)
The owner renamed the product and moved the repository to the self-hosted
Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control,
and asked for Actions and a first automatic release with the server package
and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md).

- Chronos everywhere: a byte-exact rename of contents and paths, then a
  per-language review of the forms the old name's ending had dictated.
- Gitea instead of GitHub: the update check, licence, issue and support
  links read ChronosReleasesUrl; the update check takes the highest stable
  version, offers a pre-release build its final release and reports a
  repository it cannot see as a failed check. EULA, security policy and
  Gitea issue forms live at the repository root.
- Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release,
  nightly image and performance gates in the .NET SDK container with git
  checkout and the system SQLite. The release cross-compiles the Windows
  programs, checks their icon, version and DPI manifest, packs installers
  and zips, publishes the Gitea release and pushes the server image
  (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows
  runner (WINDOWS_RUNNER).
- New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the
  docker-exec script, which Gitea's service containers do not allow; the
  throw-away server runs without durability) and pe-check.
- Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR,
  3 on the shared runner; the weekly performance gates keep theirs).

BREAKING CHANGE: programs, settings, data folders, file types, environment
variables and the Docker image use the new name; nothing is migrated from
Titano, which never had a release.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-10-23 12:45:18 +02:00

600 lines
35 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Chronos.Core.Archiving;
using Chronos.Core.Imaging;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Repositories;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
/// notes, a rename, an upload) while a settings edit, a scan or another writer commits, on both dialects: they write only
/// what they own, under the sequence lease, take the row before the frames, and leave what another process changed in
/// the sidecar to the next scan or index it whole.
/// </summary>
[Category("archive")]
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
{
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
/// <summary>One indexed sequence of three photos in <c>a</c>.</summary>
private static async Task<Sequence> IndexedAsync(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
await fx.ScanAsync();
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single();
return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
}
/// <summary>Records decode tiers as analyses, image passes and renders do.</summary>
private static FrameTiers Tiers(ArchiveFixture fx)
{
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
}
/// <summary>The frame decodes only through its 48 × 32 embedded preview.</summary>
private static Dictionary<Guid, FileTier> Preview(Guid frameId) =>
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
/// <summary>
/// Retitles the sequence in <paramref name="folder"/> in its sidecar alone, as another PC does: the next scan reads the
/// sidecar and indexes it without the lease. Returns the sidecar as written.
/// </summary>
private static SequenceManifest Retitle(SequencePaths folder)
{
var retitled = ManifestStore.ReadSequence(folder.ManifestPath) with { Title = "Ridge" };
ManifestStore.WriteSequence(folder.ManifestPath, retitled);
return retitled;
}
/// <summary>Commits <paramref name="tx"/> once the writer under test had time to read the row and wait for it.</summary>
private static async Task CommitLaterAsync(ChronosTransaction tx, CancellationToken cancellationToken)
{
await using (tx)
{
await Task.Delay(500, cancellationToken);
await tx.CommitAsync(cancellationToken);
}
}
[Test]
public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier.
Task record;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct);
Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome);
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
Assert.False(record.IsCompleted, "the record waits for the row");
await tx.CommitAsync(fx.Ct);
}
await record;
// The tier and what follows from it are recorded, and the edit committed meanwhile stays.
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount));
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
}
[Test]
public async Task TiersRecordedWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A scan's sync holds the row, as it does before it rewrites the frames, while an analysis records a tier.
Task record;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
Assert.False(record.IsCompleted, "the record waits for the row");
// The record touched no frame yet, so the scan rewrites them all without waiting for it. Had it locked its frame
// before the row, each would wait for the other and PostgreSQL would abort one of them.
await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
await tx.CommitAsync(fx.Ct);
}
await record;
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags);
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
}
[Test]
public async Task LightStatsWrittenWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A scan's sync holds the row, as it does before it rewrites the frames, while a light analysis writes its statistics.
Task<int> write;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
write = fx.Db.Frames.UpdateLightStatsAsync([.. frames.Select(f => new FrameLightStats(f.Id, -1.5, 0.01, 0.02))], fx.Ct);
await Task.WhenAny(write, Task.Delay(500, fx.Ct));
Assert.False(write.IsCompleted, "the statistics wait for the row");
// The statistics locked no frame yet, so the scan rewrites them all without waiting for them. Frames locked before
// the row, in another order than the scan's, would each wait for the other, and PostgreSQL would abort one.
await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
await tx.CommitAsync(fx.Ct);
}
Assert.Equal(frames.Count, await write);
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.MeanLogLuminance == -1.5), "written");
}
[Test]
public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// A tier recorded meanwhile changes the tiers here, not the frames this index holds.
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount));
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion));
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index.
var (desktop, _) = await fx.OtherWriterAsync();
await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct);
// A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan.
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount));
}
[Test]
public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rating goes into the sidecar and this index; the frames this index holds stay as they are.
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct);
Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating));
Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating));
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task AnUploadAfterAnotherWriterVerifiedTheSequenceIndexesWhatTheVerifyFound()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// A desktop sharing the archive verifies the sequence: the photos' first hashes go into the sidecar and its own index.
var (desktop, archive) = await fx.OtherWriterAsync();
await desktop.Verifier.VerifyAsync(archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
// A photo uploaded here before this index's next scan is appended after the others.
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The upload indexes the sequence whole from the sidecar, hashes and verify included, and leaves the folder in step.
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
Assert.Count(4, frames);
Assert.True(frames.All(f => f.Sha256 is not null), "hashed");
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.Sha256)), frames.Select(f => (f.Id, f.Sha256)));
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified");
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AnUploadAfterAPhotoWasCopiedInByHandLeavesItToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// Copied into the folder by hand, over a share: only a scan that reconciles the folder with its photos indexes it.
var folder = fx.Paths.Sequence("a");
await File.WriteAllBytesAsync(folder.File("IMG_0009.JPG"), Photos.Jpeg(Start.AddMinutes(2), 91), fx.Ct);
// A photo uploaded before this index's next scan is indexed with the sidecar's frames.
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The upload left the folder changed, so the next incremental scan reconciles it and indexes the copied photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
Assert.Count(5, frames);
Assert.Contains("IMG_0009.JPG", frames.Select(f => f.FileName));
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again.
var calls = 0;
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info =>
{
if (++calls == 1)
{
var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct)
.GetAwaiter().GetResult();
Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome);
}
return info with { Rating = 5 };
}, fx.Ct);
Assert.Equal(2, calls);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion));
Assert.Equal(rated.SettingsVersion, row.SettingsVersion);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings));
}
[Test]
public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rename moves the folder and retitles the sequence; the row keeps the frames this index holds.
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct);
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion));
// The next scan still finds the folder changed, and indexes the photo where it is now.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion));
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)),
"frames in b");
}
[Test]
public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan()
{
var fx = await FixtureAsync();
// Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513);
await fx.ScanAsync();
var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!;
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end.
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default,
(_, _, _) => throw new OperationCanceledException(), null, fx.Ct));
Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null));
Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
// Two sequences holding the same photo, hashed by a verify.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1);
await fx.ScanAsync();
await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1];
var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!;
// A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(1, report.DuplicateFlagChanges);
Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task AVerifyKeepsTheTiersRecordedWhileItRan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// The first verify of photos a scan adopted hashes them all; a thumbnail pass records a preview-only frame meanwhile.
var recorded = false;
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) =>
{
if (recorded) return;
recorded = true;
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
}, fx.Ct);
Assert.Equal(3L, report.Hashed);
// The sidecar and the index keep the tier's flag and count beside the hashes.
var indexed = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], indexed.Select(f => f.Flags));
Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], sidecar.Frames.Select(f => f.Flags));
Assert.True(indexed.All(f => f.Sha256 is not null), "hashed");
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Sha256), indexed.Select(f => f.Sha256));
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
}
[Test]
public async Task AVerifyLeavesAFrameAnUploadRepairedWhileItRanUnflagged()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// Hashed by a first verify, so an upload of the same photo repairs the frame once its file is gone.
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
var folder = fx.Paths.Sequence("a");
var lost = ManifestStore.ReadSequence(folder.ManifestPath).Frames[1];
File.Delete(folder.File(lost.File));
IngestRequest Upload((string Path, long Size, string Sha256) staged) =>
new(fx.Archive.Id, sequence.Id, null, null, staged.Path, lost.File, staged.Size, staged.Sha256, null, null);
// The verify finds the file missing, and an upload puts it back before the verify writes what it found.
var checkedFrames = 0;
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) =>
{
if (++checkedFrames != 2) return;
Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Repaired, "repaired");
}, fx.Ct);
Assert.SequenceEqual([(lost.Id, FrameHealth.Missing)], report.Problems.Select(p => (p.FrameId, p.Health)));
// The frame stays unflagged and the sequence verified, so the next copy is a duplicate and the file stays in place.
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(lost.Id, fx.Ct))!.Flags);
Assert.Equal(FrameFlags.None, ManifestStore.ReadSequence(folder.ManifestPath).Frames[1].Flags);
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified");
Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Duplicate, "duplicate");
Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*"));
}
[Test]
public async Task AScanThatReadTheSidecarBeforeAnUploadCommittedReadsItAgainUnderTheLease()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var folder = fx.Paths.Sequence("a");
var read = Retitle(folder);
// The upload's transaction holds the row (and SQLite's write lock) when the scan, which read the sidecar, reaches it.
Task<ScanSummary> scan;
SequenceManifest uploaded;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
var row = await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
scan = fx.ScanAsync();
await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
Assert.False(scan.IsCompleted, "the scan waits for the row");
// Under the lease, the upload filed its photo and wrote the sidecar; this transaction indexes them.
var photo = folder.File("IMG_0004.JPG");
await File.WriteAllBytesAsync(photo, Photos.Jpeg(Start.AddMinutes(1), 90), fx.Ct);
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), "IMG_0004.JPG", "IMG_0004.JPG", new FileInfo(photo).Length, null,
MetadataReader.Read(photo, fx.Ct));
uploaded = SequenceFrames.Insert(read, [frame]);
ManifestStore.WriteSequence(folder.ManifestPath, uploaded);
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(uploaded, fx.Archive.Id, folder, row), fx.Ct);
await tx.Frames.BulkInsertAsync([SequenceMapping.ToFrame(uploaded.Frames[^1], sequence.Id, folder)], fx.Ct);
await tx.CommitAsync(fx.Ct);
}
Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
// The scan wrote nothing from its older read: the photo, its frames version and the title are indexed.
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, uploaded.Versions.Frames, "Ridge"), (indexed.FrameCount, indexed.FramesVersion, indexed.Title));
Assert.SequenceEqual(uploaded.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AScanThatReadTheSidecarBeforeATierWasRecordedReadsItAgainUnderTheLease()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var folder = fx.Paths.Sequence("a");
var read = Retitle(folder);
// The tier writer's transaction holds the row when the scan, which read the sidecar, reaches it.
Task<ScanSummary> scan;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
scan = fx.ScanAsync();
await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
Assert.False(scan.IsCompleted, "the scan waits for the row");
// Under the lease, the writer gave the first frame its preview tier in the sidecar; this transaction indexes it.
var tier = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32);
var tiered = SequenceFrames.Update(read, new Dictionary<Guid, Func<SidecarFrame, SidecarFrame>> { [read.Frames[0].Id] = f => SequenceFrames.WithTier(f, tier) });
ManifestStore.WriteSequence(folder.ManifestPath, tiered);
var first = tiered.Frames[0];
await tx.Sequences.RecordTiersAsync(sequence.Id, SequenceStatus.None, 1, fx.Ct);
await tx.Frames.UpdateTiersAsync([new FrameTierFields(first.Id, first.Flags, first.PreviewW ?? 0, first.PreviewH ?? 0, first.Format)], fx.Ct);
await tx.CommitAsync(fx.Ct);
}
Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
// The scan wrote nothing from its older read: the tier stays, and the title is indexed.
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", 1), (indexed.Title, indexed.PreviewOnlyCount));
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AnOlderSidecarPutBackIsIndexedPastTheFramesVersionTheIndexHolds()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var folder = fx.Paths.Sequence("a");
var older = await File.ReadAllBytesAsync(folder.ManifestPath, fx.Ct);
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var uploaded = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
// A file-sync tool puts the sidecar back as it was before the upload; the photo stays.
await File.WriteAllBytesAsync(folder.ManifestPath, older, fx.Ct);
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
// Indexed as the sidecar lists it, one frames version past the index's, in both places.
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((3, uploaded.FramesVersion + 1), (indexed.FrameCount, indexed.FramesVersion));
Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AFramesVersionAnotherIndexReachedForOtherFramesIsIndexedPastIt()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var folder = fx.Paths.Sequence("a");
async Task<Sequence> UploadAsync(string name, int minute)
{
var staged = fx.Stage(Start.AddMinutes(minute), seed: 90 + minute);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, name, staged.Size, staged.Sha256, null, null),
fx.Ct);
return (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
}
await UploadAsync("IMG_0004.JPG", 1);
// A desktop sharing this server's archive, with its own index of the four photos.
await using var desktopDb = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(Context.CreateTempDirectory("desktop"), "chronos.db")), fx.Ct);
var desktop = new ArchiveService(desktopDb, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
var desktopArchive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
Task<ScanSummary> DesktopScanAsync() => desktop.Indexer.ScanAsync(desktopArchive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
await DesktopScanAsync();
// The server takes a fifth photo before the desktop scans again.
var uploaded = await UploadAsync("IMG_0005.JPG", 2);
Assert.Equal(uploaded.FramesVersion - 1, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion);
// The sidecar is damaged; the desktop's scan rebuilds it from the photos at the first version, the fifth photo under
// an id of its own, and writes it one version past its own row: the version the server holds for the photo as it
// filed it.
await File.WriteAllTextAsync(folder.ManifestPath, "{", fx.Ct);
Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences);
var rebuilt = ManifestStore.ReadSequence(folder.ManifestPath);
Assert.Equal(uploaded.FramesVersion, rebuilt.Versions.Frames);
Assert.NotEqual((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[^1].Id, rebuilt.Frames[^1].Id);
// The server's scan finds other frames at the version it holds: it indexes them one version past it, in both places,
// so an editor holding that version for the photo as it was is refused.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(uploaded.FramesVersion + 1, indexed.FramesVersion);
Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
Assert.SequenceEqual(rebuilt.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
var edit = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Rating = 2 }, null, uploaded.FramesVersion, fx.Ct);
Assert.Equal(SequenceEditOutcome.Conflict, edit.Outcome);
// The desktop follows, and neither index finds anything more to do.
Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences);
Assert.Equal(indexed.FramesVersion, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion);
Assert.Empty((await fx.ScanAsync()).Sequences);
Assert.Empty((await DesktopScanAsync()).Sequences);
}
[Test]
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// Hashed by a first verify, so the next one changes no frame and only stamps the sequence.
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
// The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence.
Task? commit = null;
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) =>
{
if (done < total || commit is not null) return;
var tx = await fx.Db.BeginAsync(fx.Ct);
await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct);
commit = CommitLaterAsync(tx, fx.Ct);
}, fx.Ct);
await Assert.NotNull(commit);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion);
Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString());
Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again");
}
}
public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
{
protected override Backend Backend => Backend.Postgres;
}