A desktop sharing the archive that imports into an existing sequence lists the new frames in the sidecar itself and keeps its ThumbsBuilt and ProxiesBuilt. The server scan cleared the flags only when SequenceAdopter.Reconcile found unindexed photos, so it queued no image pass: the new frames, and every frame they renumbered, answered 404 on /thumb and /proxy until a pass was forced. A scan now also clears AnalyzedLight, ThumbsBuilt and ProxiesBuilt when the sidecar's frames version differs from the index row's, decided under the sequence lease, and its follow-ups queue the light analysis and both passes. Writers that change only part of a sequence wrote the whole index row back from a sidecar read under a lease they had already released (decode tiers) or without any (verify stamp): an ingest or a PATCH committed in between was undone, and a row written from a sidecar the index had not synced claimed its frames version, hiding the change from the scan. FrameTiers.RecordAsync and ArchiveVerifier now write the sidecar and only their own columns (SequenceRepository.RecordTiersAsync, RecordVerifiedAsync) under one lease. They and the image passes record the folder's scan stamps only while the index holds the sidecar's frames version (ArchiveIndexer.RecordStampsInStepAsync), so the next scan still sees frames another process added. An image pass decided ThumbsBuilt/ProxiesBuilt from the sidecar it read at its start and set the flag again after a scan had cleared it for photos added meanwhile, so no pass was queued for them. MarkAsync now holds the lease for the sidecar and the index, and sets the flag only while the sidecar lists the photos the pass read at the same indexes; otherwise it clears it. A rebuild deleted every row whose folder was not among those listed when it began, so a sequence that got its first frame or was created by an upload session while it ran lost its row (404 for the Uploader, a second '(2)' sequence for the session). A row is now checked again under its lease, as it and its folder are then, before it goes. Tests: a desktop's ArchiveIngest commit followed by a server scan (Data and Server, /thumb and /proxy answer 200 for every frame); a thumbnail pass with a scan's reconcile in between; tier records and a verify stamp interleaved with a PATCH and with another writer's frames, on SQLite and PostgreSQL; a rebuild during which uploads place and create sequences. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
621 lines
35 KiB
C#
621 lines
35 KiB
C#
using System.Runtime.Versioning;
|
|
using System.Text.Json.Nodes;
|
|
using Titano.Core.Archiving;
|
|
using Titano.Core.IO;
|
|
using Titano.Core.Model;
|
|
using Titano.Core.Project;
|
|
using Titano.Data.Archiving;
|
|
using Titano.Data.Repositories;
|
|
using Titano.Testing;
|
|
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Titano.Data.Tests.Archiving;
|
|
|
|
[Category("archive")]
|
|
public sealed class ArchiveIngestTests(TestContext context)
|
|
{
|
|
private static IngestRequest Request(ArchiveFixture fx, (string Path, long Size, string Sha256) staged, string? sessionKey = "upload-1",
|
|
Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") =>
|
|
new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null);
|
|
|
|
/// <summary>A sequence as <c>POST /api/v1/sequences</c> creates it: a row without a folder or frames.</summary>
|
|
private static async Task<Sequence> CreateWithoutPhotosAsync(ArchiveFixture fx, string title)
|
|
{
|
|
var created = new Sequence
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
ArchiveId = fx.Archive.Id,
|
|
Title = title,
|
|
Folder = string.Empty,
|
|
Rating = 4,
|
|
Origin = SequenceOrigin.Server,
|
|
OwnerUserId = Guid.CreateVersion7(),
|
|
UpdatedAt = DateTimeOffset.UtcNow,
|
|
};
|
|
await fx.Db.Sequences.UpsertAsync(created, fx.Ct);
|
|
return created;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Makes <paramref name="path"/> unreadable until disposed: without read permission where that stops the test process
|
|
/// (UnauthorizedAccessException), otherwise (Windows, root) held open without sharing (IOException).
|
|
/// </summary>
|
|
private static IDisposable Unreadable(string path)
|
|
{
|
|
if (!OperatingSystem.IsWindows())
|
|
{
|
|
var mode = File.GetUnixFileMode(path);
|
|
File.SetUnixFileMode(path, UnixFileMode.None);
|
|
try
|
|
{
|
|
File.OpenRead(path).Dispose();
|
|
File.SetUnixFileMode(path, mode);
|
|
}
|
|
catch (UnauthorizedAccessException)
|
|
{
|
|
return new UnixPermissions(path, mode);
|
|
}
|
|
}
|
|
return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None);
|
|
}
|
|
|
|
/// <summary>Gives a file its permissions back.</summary>
|
|
[UnsupportedOSPlatform("windows")]
|
|
private sealed class UnixPermissions(string path, UnixFileMode mode) : IDisposable
|
|
{
|
|
public void Dispose() => File.SetUnixFileMode(path, mode);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var first = fx.Stage(Start, 1);
|
|
var a = await fx.Service.Ingest.CommitAsync(Request(fx, first), fx.Ct);
|
|
var b = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), name: "IMG_0002.JPG"), fx.Ct);
|
|
|
|
Assert.Equal(a.SequenceId, b.SequenceId);
|
|
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", a.RelativePath);
|
|
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-10_0002.JPG", b.RelativePath);
|
|
Assert.Equal((0, 1), (a.Index, b.Index));
|
|
Assert.False(a.Duplicate);
|
|
Assert.FileMissing(first.Path);
|
|
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("2026/2026-09-28 Canon EOS R5").ManifestPath);
|
|
Assert.Equal(a.SequenceId, manifest.Id);
|
|
Assert.SequenceEqual(["upload-1"], manifest.SessionKeys);
|
|
Assert.Equal("IMG_0002.JPG", manifest.Frames[1].Original);
|
|
Assert.Equal(first.Sha256, manifest.Frames[0].Sha256);
|
|
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(a.SequenceId))!.FrameCount);
|
|
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheSameContentIsReportedAsDuplicateAndNotCopiedTwice()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
|
|
var again = fx.Stage(Start, 7);
|
|
var duplicate = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct);
|
|
Assert.True(duplicate.Duplicate);
|
|
Assert.Equal(original.FrameId, duplicate.FrameId);
|
|
Assert.Equal(original.RelativePath, duplicate.RelativePath);
|
|
Assert.FileExists(again.Path);
|
|
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ParallelCommitsToOneSequenceGiveGapFreeIndexesAndAValidSidecar()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
const int count = 40;
|
|
var staged = Enumerable.Range(0, count).Select(i => fx.Stage(Start.AddSeconds(10 * i), 1000 + i)).ToList();
|
|
// Arrival order differs from capture order, as with parallel chunked uploads.
|
|
var order = Enumerable.Range(0, count).OrderBy(i => (i * 17) % count).ToList();
|
|
var results = await Task.WhenAll(order.Select(i => Task.Run(() =>
|
|
fx.Service.Ingest.CommitAsync(Request(fx, staged[i], name: $"IMG_{i:0000}.JPG"), fx.Ct))));
|
|
|
|
Assert.Count(1, results.Select(r => r.SequenceId).Distinct());
|
|
Assert.False(results.Any(r => r.Duplicate));
|
|
var sequenceId = results[0].SequenceId;
|
|
var row = (await fx.Db.Sequences.GetAsync(sequenceId))!;
|
|
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
|
|
Assert.SequenceEqual(Enumerable.Range(0, count), manifest.Frames.Select(f => f.I));
|
|
Assert.SequenceEqual(Enumerable.Range(0, count).Select(i => Start.AddSeconds(10 * i)), manifest.Frames.Select(f => f.CapturedAt!.Value));
|
|
Assert.Count(count, manifest.Frames.Select(f => f.File).Distinct());
|
|
Assert.True(manifest.Frames.All(f => File.Exists(fx.Paths.Sequence(row.Folder).File(f.File))));
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId);
|
|
Assert.SequenceEqual(manifest.Frames.Select(f => (f.Id, f.I)), frames.Select(f => (f.Id, f.Index)));
|
|
Assert.Equal(count, row.FrameCount);
|
|
Assert.Empty(Directory.EnumerateFiles(Path.Combine(fx.Paths.StagingDirectory, "uploads")));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ExplicitTargetsUseTheGivenSequenceOrTitle()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var titled = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), title: "Harbour"), fx.Ct);
|
|
var sameTitle = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(5), 2), title: "harbour "), fx.Ct);
|
|
var byId = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(9), 3), sequenceId: titled.SequenceId), fx.Ct);
|
|
Assert.Equal(titled.SequenceId, sameTitle.SequenceId);
|
|
Assert.Equal(titled.SequenceId, byId.SequenceId);
|
|
Assert.Equal("Harbour", (await fx.Db.Sequences.GetAsync(titled.SequenceId))!.Title);
|
|
await Assert.ThrowsAsync<ArgumentException>(() => fx.Service.Ingest.CommitAsync(
|
|
new IngestRequest(fx.Archive.Id, titled.SequenceId, "Both", null, fx.Stage(Start, 4).Path, "x.jpg", 1, "00", null, null), fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFrameCapturedEarlierIsInsertedInOrderAndTheIndexFollows()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct);
|
|
var early = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 2)), fx.Ct);
|
|
Assert.Equal(0, early.Index);
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(late.SequenceId);
|
|
Assert.SequenceEqual([early.FrameId, late.FrameId], frames.Select(f => f.Id));
|
|
Assert.SequenceEqual([0, 1], frames.Select(f => f.Index));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ABatchInterruptedBetweenMoveAndSidecarIsCompletedOnTheNextCommit()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1)), fx.Ct);
|
|
var row = (await fx.Db.Sequences.GetAsync(first.SequenceId))!;
|
|
var sequence = fx.Paths.Sequence(row.Folder);
|
|
// Simulated crash: the journal was written and the file moved, but sequence.json was not rewritten.
|
|
var bytes = Photos.Jpeg(Start.AddSeconds(10), 2);
|
|
File.WriteAllBytes(sequence.File("originals/moved.JPG"), bytes);
|
|
var orphan = new SidecarFrame { Id = Guid.CreateVersion7(), File = "originals/moved.JPG", Size = bytes.Length, CapturedAt = Start.AddSeconds(10) };
|
|
IngestJournal.Write(sequence, new IngestJournal([orphan]));
|
|
|
|
var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(20), 3)), fx.Ct);
|
|
Assert.Equal(2, next.Index);
|
|
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
|
|
Assert.SequenceEqual([first.FrameId, orphan.Id, next.FrameId], manifest.Frames.Select(f => f.Id));
|
|
Assert.FileMissing(sequence.IngestJournalPath);
|
|
Assert.Equal(3, (await fx.Db.Frames.ListBySequenceAsync(first.SequenceId)).Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AMissingFrameIsRepairedByTheSameContentInsteadOfCountingAsDuplicate()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
|
|
var row = (await fx.Db.Sequences.GetAsync(original.SequenceId))!;
|
|
var file = Path.Combine(fx.Archive.RootPath, original.RelativePath);
|
|
var bytes = await File.ReadAllBytesAsync(file, fx.Ct);
|
|
File.Delete(file);
|
|
await fx.Service.Indexer.ReconcileAsync(fx.Archive, row, fx.Ct);
|
|
Assert.Equal(FrameFlags.Missing, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
|
|
|
|
var again = fx.Stage(Start, 7);
|
|
var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct);
|
|
|
|
Assert.False(repaired.Duplicate, "the server holds no intact copy: this one is filed");
|
|
Assert.Equal((false, true), (original.Repaired, repaired.Repaired));
|
|
Assert.Equal((original.FrameId, original.Index, original.RelativePath), (repaired.FrameId, repaired.Index, repaired.RelativePath));
|
|
Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct));
|
|
Assert.FileMissing(again.Path);
|
|
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
|
|
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
|
|
Assert.Equal(FrameFlags.None, manifest.Frames.Single().Flags);
|
|
Assert.False((await fx.Db.Sequences.GetAsync(original.SequenceId))!.Status.HasFlag(SequenceStatus.HasErrors));
|
|
|
|
// Intact again: the next copy is a duplicate and stays in staging.
|
|
var third = fx.Stage(Start, 7);
|
|
Assert.True((await fx.Service.Ingest.CommitAsync(Request(fx, third), fx.Ct)).Duplicate);
|
|
Assert.FileExists(third.Path);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ACorruptFrameIsRepairedAndItsDamagedFileKeptAside()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
|
|
var file = Path.Combine(fx.Archive.RootPath, original.RelativePath);
|
|
var bytes = await File.ReadAllBytesAsync(file, fx.Ct);
|
|
var damaged = bytes.ToArray();
|
|
damaged[^10] ^= 0xFF;
|
|
await File.WriteAllBytesAsync(file, damaged, fx.Ct);
|
|
await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
|
Assert.Equal(FrameFlags.Corrupt, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
|
|
|
|
var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
|
|
|
|
Assert.Equal((false, true), (repaired.Duplicate, repaired.Repaired));
|
|
Assert.Equal(original.FrameId, repaired.FrameId);
|
|
Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct));
|
|
var aside = Directory.GetFiles(Path.GetDirectoryName(file)!, Path.GetFileName(file) + ".damaged-*");
|
|
Assert.Count(1, aside);
|
|
Assert.SequenceEqual(damaged, await File.ReadAllBytesAsync(aside[0], fx.Ct));
|
|
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
|
|
// The kept file is no photo to a scan: the sequence still has one frame.
|
|
Assert.Equal(0, (await fx.ScanAsync(ScanMode.Full)).Failed.Count);
|
|
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceCreatedWithoutPhotosGetsItsFolderWithTheFirstFrame()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
// As POST /api/v1/sequences creates it: a row without a folder.
|
|
var created = new Sequence
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
ArchiveId = fx.Archive.Id,
|
|
Title = "Harbour sunset",
|
|
Folder = string.Empty,
|
|
Tags = ["harbour"],
|
|
Rating = 4,
|
|
Origin = SequenceOrigin.Server,
|
|
OwnerUserId = Guid.CreateVersion7(),
|
|
UpdatedAt = DateTimeOffset.UtcNow,
|
|
};
|
|
await fx.Db.Sequences.UpsertAsync(created, fx.Ct);
|
|
|
|
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), sequenceId: created.Id), fx.Ct);
|
|
var second = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), sequenceId: created.Id, name: "IMG_0002.JPG"), fx.Ct);
|
|
|
|
Assert.Equal((created.Id, created.Id), (first.SequenceId, second.SequenceId));
|
|
var row = (await fx.Db.Sequences.GetAsync(created.Id))!;
|
|
Assert.Equal("2026/2026-09-28 Canon EOS R5", row.Folder);
|
|
Assert.Equal(("Harbour sunset", 4, SequenceOrigin.Server, created.OwnerUserId, 2), (row.Title, row.Rating, row.Origin, row.OwnerUserId, row.FrameCount));
|
|
Assert.SequenceEqual(["harbour"], row.Tags);
|
|
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", first.RelativePath);
|
|
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
|
|
Assert.Equal((created.Id, "Harbour sunset", created.OwnerUserId.ToString()), (manifest.Id, manifest.Title, manifest.CreatedBy));
|
|
Assert.SequenceEqual([first.FrameId, second.FrameId], manifest.Frames.Select(f => f.Id));
|
|
Assert.FileMissing(fx.Paths.Sequence(string.Empty).ManifestPath);
|
|
// The index can be rebuilt from the sidecar it now has.
|
|
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceAdoptedAtTheRootIsNeverMovedWhenItsSidecarIsGone()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Photos.WriteSeries(fx.Archive.RootPath, Start, 2);
|
|
await fx.ScanAsync();
|
|
var adopted = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault());
|
|
Assert.Equal(string.Empty, adopted.Folder);
|
|
File.Delete(fx.Paths.Sequence(string.Empty).ManifestPath);
|
|
|
|
// It held photos, so it is no sequence waiting for a folder: the commit reports the missing sidecar.
|
|
var staged = fx.Stage(Start.AddMinutes(5), 3);
|
|
await Assert.ThrowsAsync<IOException>(() => fx.Service.Ingest.CommitAsync(Request(fx, staged, sequenceId: adopted.Id), fx.Ct));
|
|
|
|
var row = (await fx.Db.Sequences.GetAsync(adopted.Id))!;
|
|
Assert.Equal((string.Empty, 2), (row.Folder, row.FrameCount));
|
|
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(adopted.Id));
|
|
Assert.False(Directory.Exists(Path.Combine(fx.Archive.RootPath, "2026")), "no folder was claimed for it");
|
|
Assert.FileExists(staged.Path);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceWaitingForItsFirstFrameNeverLendsItsIdToTheArchiveRoot()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
|
// Loose photos at the root, which a scan adopts in place.
|
|
Photos.WriteSeries(fx.Archive.RootPath, Start.AddDays(-1), 2);
|
|
|
|
await fx.ScanAsync();
|
|
|
|
var root = ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath);
|
|
Assert.NotEqual(waiting.Id, root.Id);
|
|
var rootRow = (await fx.Db.Sequences.GetAsync(root.Id))!;
|
|
Assert.Equal((string.Empty, 2), (rootRow.Folder, rootRow.FrameCount));
|
|
var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
|
|
Assert.Equal((string.Empty, 0, "Harbour sunset"), (row.Folder, row.FrameCount, row.Title));
|
|
|
|
// Its first frame still gets a folder of its own, and the root keeps its photos.
|
|
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), sequenceId: waiting.Id), fx.Ct);
|
|
Assert.Equal(waiting.Id, first.SequenceId);
|
|
var placed = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
|
|
Assert.Equal(("2026/2026-09-28 Canon EOS R5", 1), (placed.Folder, placed.FrameCount));
|
|
Assert.Equal(root.Id, ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath).Id);
|
|
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(root.Id))!.FrameCount);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ARootSequenceWhoseSidecarIsGoneIsAdoptedAgainUnderItsOwnId()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Photos.WriteSeries(fx.Archive.RootPath, Start, 2);
|
|
await fx.ScanAsync();
|
|
var adopted = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault());
|
|
// Created later, so its id sorts after the root sequence's.
|
|
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
|
File.Delete(fx.Paths.Sequence(string.Empty).ManifestPath);
|
|
|
|
await fx.ScanAsync();
|
|
|
|
Assert.Equal(adopted.Id, ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath).Id);
|
|
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(adopted.Id))!.FrameCount);
|
|
var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
|
|
Assert.Equal((string.Empty, 0), (row.Folder, row.FrameCount));
|
|
}
|
|
|
|
[Test]
|
|
public async Task RebuildOrphansAndVerifyLeaveASequenceWaitingForItsFirstFrame()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2);
|
|
await fx.ScanAsync();
|
|
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
|
|
|
// It has no sidecar yet, which is no reason to drop it.
|
|
var rebuilt = await fx.ScanAsync(ScanMode.Rebuild);
|
|
Assert.Equal(0, rebuilt.Removed);
|
|
var orphans = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
|
|
Assert.Empty(orphans.MissingSequences);
|
|
Assert.Empty(orphans.UnindexedFolders);
|
|
Assert.Equal(0, await fx.Service.RemoveMissingSequencesAsync(fx.Archive.Id, [waiting.Id], fx.Ct));
|
|
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
|
Assert.Equal((1, 2L), (report.Sequences, report.Checked));
|
|
var kept = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id));
|
|
Assert.Equal(("Harbour sunset", 4), (kept.Title, kept.Rating));
|
|
|
|
// The archive root is not its folder: loose photos there are a folder the index does not know yet.
|
|
Photos.WriteSeries(fx.Archive.RootPath, Start.AddDays(-1), 1, firstSeed: 50);
|
|
Assert.SequenceEqual([string.Empty], (await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).UnindexedFolders);
|
|
|
|
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct);
|
|
Assert.Equal(waiting.Id, first.SequenceId);
|
|
Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ARebuildKeepsSequencesThatGetTheirFolderWhileItRuns()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2);
|
|
await fx.ScanAsync();
|
|
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
|
|
|
// Uploads go on while the rebuild runs: after it listed the folders, the sequence waiting for its first frame gets
|
|
// it, and a new upload session creates a sequence.
|
|
IngestResult? first = null, session = null;
|
|
var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, async progress =>
|
|
{
|
|
if (progress.FoldersDone < progress.FoldersTotal || first is not null) return;
|
|
first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct);
|
|
session = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2), 2), sessionKey: "upload-2"), fx.Ct);
|
|
}, fx.Ct);
|
|
|
|
// Neither is dropped, though the folders listed at the start hold neither.
|
|
Assert.Equal(0, rebuilt.Removed);
|
|
Assert.Equal(waiting.Id, Assert.NotNull(first).SequenceId);
|
|
var placed = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id));
|
|
Assert.Equal(("2026/2026-09-29 Canon EOS R5", 1, "Harbour sunset"), (placed.Folder, placed.FrameCount, placed.Title));
|
|
var created = Assert.NotNull(await fx.Db.Sequences.GetAsync(Assert.NotNull(session).SequenceId));
|
|
Assert.Equal(1, created.FrameCount);
|
|
// The session's next upload lands in the same sequence, not in a second one beside it.
|
|
Assert.Equal(created.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "upload-2", fx.Ct));
|
|
var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2).AddSeconds(10), 3), sessionKey: "upload-2"), fx.Ct);
|
|
Assert.Equal(created.Id, next.SequenceId);
|
|
Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADamagedRootSidecarIsRebuiltUnderTheRootSequenceNotOneWaitingForItsFirstFrame()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Photos.WriteSeries(fx.Archive.RootPath, Start, 2);
|
|
await fx.ScanAsync();
|
|
var root = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault());
|
|
// Created later, so its id sorts after the root sequence's; then another tool tears the root sidecar.
|
|
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
|
var sidecar = fx.Paths.Sequence(string.Empty).ManifestPath;
|
|
File.WriteAllText(sidecar, "{");
|
|
|
|
var summary = await fx.ScanAsync();
|
|
|
|
Assert.Empty(summary.Failed);
|
|
Assert.Equal(root.Id, ManifestStore.ReadSequence(sidecar).Id);
|
|
Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(sidecar)!, $"{Path.GetFileName(sidecar)}.damaged-*"));
|
|
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(root.Id))!.FrameCount);
|
|
var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
|
|
Assert.Equal((string.Empty, 0, "Harbour sunset"), (row.Folder, row.FrameCount, row.Title));
|
|
Assert.Empty(await fx.Db.Frames.ListBySequenceAsync(waiting.Id));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ARootSidecarThatCannotBeReadFailsTheRootFolderAloneAndDropsNoSequence()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Photos.WriteSeries(fx.Archive.RootPath, Start, 2);
|
|
await fx.ScanAsync();
|
|
var root = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault());
|
|
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
|
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start.AddDays(1), 2, firstSeed: 50);
|
|
var sidecar = fx.Paths.Sequence(string.Empty).ManifestPath;
|
|
|
|
ScanSummary rebuilt;
|
|
using (Unreadable(sidecar)) rebuilt = await fx.ScanAsync(ScanMode.Rebuild);
|
|
|
|
Assert.SequenceEqual([string.Empty], rebuilt.Failed.Select(f => f.Folder));
|
|
Assert.Equal(0, rebuilt.Removed);
|
|
var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct);
|
|
Assert.SequenceEqual(new[] { root.Id, waiting.Id }.Order(), locations.Where(l => l.Folder.Length == 0).Select(l => l.Id).Order());
|
|
Assert.SequenceEqual(["Trips/Comet"], locations.Where(l => l.Folder.Length > 0).Select(l => l.Folder));
|
|
|
|
// Readable again, the root is indexed under its own sequence.
|
|
Assert.Empty((await fx.ScanAsync()).Failed);
|
|
Assert.Equal(root.Id, ManifestStore.ReadSequence(sidecar).Id);
|
|
var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
|
|
Assert.Equal((string.Empty, 0), (row.Folder, row.FrameCount));
|
|
}
|
|
|
|
[Test]
|
|
public async Task MaintenanceCompactsTheRootPackUnderTheRootSequencesLeaseOnly()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
// Left over from before the root was adopted, so its id sorts before the root sequence's.
|
|
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
|
Photos.WriteSeries(fx.Archive.RootPath, Start, 2);
|
|
await fx.ScanAsync();
|
|
var root = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(l => l.Id != waiting.Id);
|
|
var packPath = fx.Paths.Sequence(string.Empty).ThumbsPackPath;
|
|
var hash = new byte[ThumbPack.HashLength];
|
|
using (var pack = ThumbPackWriter.Open(packPath))
|
|
{
|
|
// The same entry written again and again: everything but the last copy is garbage.
|
|
for (var i = 0; i < 4; i++) pack.Append(0, 48, 32, hash, Photos.Jpeg(Start, 70 + i));
|
|
pack.Commit();
|
|
}
|
|
var length = new FileInfo(packPath).Length;
|
|
|
|
// A thumbnail pass of the root sequence appends under its own lease: the pack waits for it.
|
|
Task<MaintenanceReport> maintenance;
|
|
using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{root.Id:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct))
|
|
{
|
|
maintenance = fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
await Task.Delay(500, fx.Ct);
|
|
Assert.False(maintenance.IsCompleted);
|
|
Assert.Equal(length, new FileInfo(packPath).Length);
|
|
}
|
|
|
|
var report = await maintenance;
|
|
Assert.SequenceEqual([string.Empty], report.CompactedPacks);
|
|
using var compacted = ThumbPackReader.Open(packPath);
|
|
Assert.Equal(0L, compacted.GarbageBytes);
|
|
Assert.NotNull(compacted.Read(0));
|
|
}
|
|
|
|
[Test]
|
|
public async Task FramesInsertedBeforeOthersTakeTheProjectPositionsAlong()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var results = new List<IngestResult>();
|
|
for (var i = 0; i < 5; i++) results.Add(await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1 + i), 10 + i)), fx.Ct));
|
|
var sequenceId = results[0].SequenceId;
|
|
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(sequenceId))!.Folder);
|
|
var project = new ProjectSettings
|
|
{
|
|
ExcludedFrames = [3],
|
|
HeroFrame = 1,
|
|
Camera = new CameraSettings { Keyframes = [new CameraKey(0, 0.5, 0.5, 1, 0), new CameraKey(4, 0.4, 0.5, 1.5, 0)] },
|
|
}.ToJson();
|
|
AtomicFile.WriteAllText(folder.ProjectPath, project);
|
|
var edited = await fx.Db.Sequences.EditAsync(sequenceId, new SequenceEdit { SettingsJson = project, HeroFrameId = results[1].FrameId }, null, fx.Ct);
|
|
var version = edited.Sequence!.SettingsVersion;
|
|
|
|
// Two photos captured before every other one, and one between the third and the fourth, arrive later.
|
|
await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 20)), fx.Ct);
|
|
await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(30), 21)), fx.Ct);
|
|
await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(3.5), 22)), fx.Ct);
|
|
|
|
var row = (await fx.Db.Sequences.GetAsync(sequenceId))!;
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId);
|
|
var moved = ProjectSettings.FromJson(row.SettingsJson!);
|
|
Assert.SequenceEqual([results[3].FrameId], moved.ExcludedFrames.Select(i => frames[i].Id));
|
|
Assert.Equal(results[1].FrameId, frames[moved.HeroFrame!.Value].Id);
|
|
Assert.SequenceEqual([results[0].FrameId, results[4].FrameId], moved.Camera.Keyframes.Select(k => frames[k.Frame].Id));
|
|
Assert.Equal(version, row.SettingsVersion);
|
|
Assert.Equal(results[1].FrameId, row.HeroFrameId);
|
|
Assert.True(JsonNode.DeepEquals(JsonNode.Parse(row.SettingsJson!), JsonNode.Parse(File.ReadAllText(folder.ProjectPath))), "project.json mirrors the index");
|
|
}
|
|
|
|
[Test]
|
|
public async Task PositionsWaitingForTheirPhotosTakeTheirPlaceWhenTheyArrive()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var results = new List<IngestResult>();
|
|
for (var i = 0; i < 3; i++) results.Add(await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1 + i), 30 + i)), fx.Ct));
|
|
var sequenceId = results[0].SequenceId;
|
|
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(sequenceId))!.Folder);
|
|
// Settings synced from a copy that also holds an earlier and a later photo: what names them waits for them.
|
|
var early = fx.Stage(Start, 40);
|
|
var late = fx.Stage(Start.AddMinutes(9), 41);
|
|
var project = new ProjectSettings
|
|
{
|
|
ExcludedFrames = [1],
|
|
Camera = new CameraSettings { Keyframes = [new CameraKey(0, 0.5, 0.5, 1, 0)] },
|
|
PendingFrames =
|
|
[
|
|
new PendingFrame { Sha256 = early.Sha256, Size = early.Size, Excluded = true },
|
|
new PendingFrame { Sha256 = late.Sha256, Size = late.Size, Hero = true, Keyframes = [new CameraKey(0, 0.3, 0.6, 2, 0)] },
|
|
],
|
|
}.ToJson();
|
|
AtomicFile.WriteAllText(folder.ProjectPath, project);
|
|
var version = (await fx.Db.Sequences.EditAsync(sequenceId, new SequenceEdit { SettingsJson = project }, null, fx.Ct)).Sequence!.SettingsVersion;
|
|
|
|
// The later photo is appended in step with the index; the earlier one is inserted before every other.
|
|
var appended = await fx.Service.Ingest.CommitAsync(Request(fx, late), fx.Ct);
|
|
var halfway = ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequenceId))!.SettingsJson!);
|
|
Assert.Equal(3, halfway.HeroFrame);
|
|
Assert.Count(1, halfway.PendingFrames!);
|
|
Assert.Equal(early.Sha256, halfway.PendingFrames![0].Sha256);
|
|
var inserted = await fx.Service.Ingest.CommitAsync(Request(fx, early), fx.Ct);
|
|
|
|
var row = (await fx.Db.Sequences.GetAsync(sequenceId))!;
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId);
|
|
var placed = ProjectSettings.FromJson(row.SettingsJson!);
|
|
Assert.SequenceEqual([inserted.FrameId, results[1].FrameId], placed.ExcludedFrames.Select(i => frames[i].Id));
|
|
Assert.Equal(appended.FrameId, frames[placed.HeroFrame!.Value].Id);
|
|
Assert.SequenceEqual([results[0].FrameId, appended.FrameId], placed.Camera.Keyframes.Select(k => frames[k.Frame].Id));
|
|
Assert.Equal(0.3, placed.Camera.Keyframes[1].CenterX);
|
|
Assert.Null(placed.PendingFrames);
|
|
Assert.Equal(appended.FrameId, row.HeroFrameId);
|
|
Assert.Equal(version, row.SettingsVersion);
|
|
Assert.True(JsonNode.DeepEquals(JsonNode.Parse(row.SettingsJson!), JsonNode.Parse(File.ReadAllText(folder.ProjectPath))), "project.json mirrors the index");
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosAFolderScanFindsMoveTheProjectPositionsButUnknownMembersStay()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct);
|
|
await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(2), 2)), fx.Ct);
|
|
var row = (await fx.Db.Sequences.GetAsync(late.SequenceId))!;
|
|
var folder = fx.Paths.Sequence(row.Folder);
|
|
// A project with a member from a newer Titano; only the index copy holds it (the API edited it).
|
|
const string project = "{ \"excludedFrames\": [1], \"heroFrame\": 0, \"futureOption\": { \"keep\": true } }";
|
|
await fx.Db.Sequences.EditAsync(late.SequenceId, new SequenceEdit { SettingsJson = project }, null, fx.Ct);
|
|
|
|
File.WriteAllBytes(folder.File("originals/early.JPG"), Photos.Jpeg(Start, 3));
|
|
await fx.Service.Indexer.ReconcileAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(late.SequenceId))!, fx.Ct);
|
|
|
|
var settings = (await fx.Db.Sequences.GetAsync(late.SequenceId))!.SettingsJson!;
|
|
var moved = ProjectSettings.FromJson(settings);
|
|
Assert.SequenceEqual([2], moved.ExcludedFrames);
|
|
Assert.Equal(1, moved.HeroFrame);
|
|
Assert.Contains("futureOption", settings);
|
|
Assert.True(JsonNode.DeepEquals(JsonNode.Parse(settings), JsonNode.Parse(File.ReadAllText(folder.ProjectPath))), "project.json mirrors the index");
|
|
}
|
|
|
|
[Test]
|
|
public async Task AScanMovesTheProjectPositionsOnlyBetweenProjectSaves()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct);
|
|
await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(2), 2)), fx.Ct);
|
|
var id = late.SequenceId;
|
|
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
|
var version = (await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = "{ \"excludedFrames\": [1] }" }, null, fx.Ct)).Sequence!.SettingsVersion;
|
|
|
|
// Another process files a photo captured before both into the sidecar; this index learns of it from a scan.
|
|
var photos = ArchiveScanner.PhotoDirectory(folder);
|
|
var stamp = Directory.GetLastWriteTimeUtc(photos);
|
|
File.WriteAllBytes(folder.File("originals/early.JPG"), Photos.Jpeg(Start, 3));
|
|
ManifestStore.WriteSequence(folder.ManifestPath, SequenceAdopter.Reconcile(folder, ManifestStore.ReadSequence(folder.ManifestPath), fx.Ct)!);
|
|
Directory.SetLastWriteTimeUtc(photos, stamp);
|
|
|
|
// An editor saving the project holds the sequence lease (LocalSequenceSource.SaveProjectAsync).
|
|
var save = await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{id:N}", "editor", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct);
|
|
var scan = Task.Run(() => fx.ScanAsync());
|
|
await Task.Delay(500, fx.Ct);
|
|
Assert.False(scan.IsCompleted, "the scan waits for the save");
|
|
// The save stores the positions its editor knew, as of the frames before the scan.
|
|
Assert.Equal(SequenceEditOutcome.Updated,
|
|
(await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = "{ \"excludedFrames\": [0, 1] }" }, version, fx.Ct)).Outcome);
|
|
save.Dispose();
|
|
Assert.Equal(1, (await scan).Indexed);
|
|
|
|
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
|
Assert.Equal(3, row.FrameCount);
|
|
Assert.SequenceEqual([1, 2], ProjectSettings.FromJson(row.SettingsJson!).ExcludedFrames);
|
|
}
|
|
}
|