Files
Alby96andClaude Opus 5.5 0075065884
CI / Build and test (push) Successful in 10m37s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 9s
Release / Build, test and publish (push) Successful in 16m19s
CI / PostgreSQL 16 integration (push) Successful in 10m19s
CI / PostgreSQL 17 integration (push) Successful in 11m27s
CI / Docker image (push) Successful in 57s
Release / Server image (push) Successful in 1m34s
feat: work board, no usage data, archive folder tree, deploy guide (owner review 4 + owner review 1 phase 6) (#62)
Owner review 4 and owner review 1 phase 6.

- Work board in Videos (desktop and admin panel): Published and Workspace lanes (#29).
- Anonymous usage counts removed: Chronos sends nothing about its use (#30).
- The archive's whole folder tree is created inside any archive root, on the server and the desktop (#31).
- Video cards start with the preset; sizes keep number and unit together (#28).
- Deploy guide docs/deploy (Docker, Unraid, desktop), BRIEF §24, original documentation in docs/reference/original (#22).

Closes #22 #28 #29 #30 #31

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-11-06 18:46:22 +01:00

1322 lines
79 KiB
C#

using System.Security.Cryptography;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Import;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Repositories;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>Deleting photos of a sequence into the archive's trash, restoring them and emptying the trash (owner review 2, R6).</summary>
[Category("archive")]
public abstract class FrameTrashTests(TestContext context) : DatabaseTest(context)
{
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
/// <summary>A sequence of <paramref name="count"/> uploaded photos (hashed, under <c>originals/</c>), ten seconds apart.</summary>
private static async Task<(Sequence Sequence, List<Frame> Frames)> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1", int seed = 100)
{
Guid? id = null;
for (var i = 0; i < count; i++)
{
var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i), seed + i);
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG",
size, sha, null, null), fx.Ct);
id = result.SequenceId;
}
return ((await fx.Db.Sequences.GetAsync(id!.Value))!, [.. await fx.Db.Frames.ListBySequenceAsync(id.Value)]);
}
private static async Task<FrameDeleteResult> DeleteAsync(ArchiveFixture fx, Sequence sequence, params Frame[] frames)
{
var plan = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [.. frames.Select(f => f.Id)], sequence.FramesVersion, fx.Ct);
Assert.True(plan.CanDelete, string.Join(", ", plan.Blocked));
return await fx.Service.FrameTrash.DeleteAsync(plan, "alberto", null, fx.Ct);
}
private static Job Running(JobKind kind, IReadOnlyList<Guid> targets) => new()
{
Id = Guid.CreateVersion7(),
Kind = kind,
TargetIds = targets,
State = JobState.Running,
NodeId = "test#0",
Created = DateTimeOffset.UtcNow,
Started = DateTimeOffset.UtcNow,
};
private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
private static string EntryFolder(ArchiveFixture fx, TrashEntry entry) => FrameTrash.Location(fx.Paths, entry.Name).Folder;
private static async Task DeleteSequenceAsync(ArchiveFixture fx, Guid id) =>
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([id], null, fx.Ct), null, null, fx.Ct);
/// <summary>A sequence a scan adopted in place from photos written into <paramref name="folder"/>: its frames are not hashed.</summary>
private static async Task<(Sequence Sequence, List<Frame> Frames)> AdoptedAsync(ArchiveFixture fx, string folder, int count, int seed = 1)
{
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, folder), Start, count, seed);
await fx.ScanAsync();
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == folder).Id;
return ((await fx.Db.Sequences.GetAsync(id))!, [.. await fx.Db.Frames.ListBySequenceAsync(id)]);
}
/// <summary>The journal a restore of <paramref name="entry"/> into its sequence writes before its first move (names free: as they were).</summary>
private static FrameRestoreJournal RestoreJournal(ArchiveFixture fx, TrashEntry entry)
{
var (folder, record) = FrameTrash.Location(fx.Paths, entry.Name);
var photos = FrameTrash.ReadEntry(fx.Paths, entry.Name)!.Photos;
return new FrameRestoreJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(folder),
[.. photos.Select(p => new RestoredPhoto(p.Frame.Id, [.. p.Files.Select(f => new FrameRemovalMove(f.From, fx.Paths.ToRelative(Path.Combine(folder, f.Path))))]))]);
}
/// <summary>Makes every <paramref name="operation"/> on <paramref name="table"/> fail, as a lost database connection would.</summary>
private async Task FailAsync(ArchiveFixture fx, string table, string operation)
{
await using var session = await fx.Db.Database.OpenAsync(fx.Ct);
await session.ExecuteScriptAsync(ForDialect(
$"CREATE TRIGGER fail_{table} BEFORE {operation} ON {table} BEGIN SELECT RAISE(ABORT, 'injected'); END;",
$"CREATE FUNCTION fail_{table}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected'; END $$; " +
$"CREATE TRIGGER fail_{table} BEFORE {operation} ON {table} FOR EACH ROW EXECUTE FUNCTION fail_{table}();"), fx.Ct);
}
private async Task HealAsync(ArchiveFixture fx, string table)
{
await using var session = await fx.Db.Database.OpenAsync(fx.Ct);
await session.ExecuteScriptAsync(ForDialect($"DROP TRIGGER fail_{table};", $"DROP TRIGGER fail_{table} ON {table};"), fx.Ct);
}
[Test]
public async Task TheDryRunCountsThePhotosAndChangesNothing()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var plan = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [frames[1].Id, frames[3].Id], sequence.FramesVersion, fx.Ct);
Assert.True(plan.CanDelete);
Assert.SequenceEqual([frames[1].Id, frames[3].Id], plan.Frames.Select(f => f.Id));
Assert.Equal(frames[1].Size + frames[3].Size, plan.Bytes);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.False(Directory.EnumerateFileSystemEntries(FrameTrash.Directory(fx.Paths)).Any(), "a dry run moves nothing");
foreach (var frame in frames) Assert.FileExists(fx.Paths.FromRelative(frame.RelativePath));
}
[Test]
public async Task APlanSaysWhyPhotosCannotGo()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var trash = fx.Service.FrameTrash;
Assert.SequenceEqual([FrameDeleteBlock.WouldEmpty], (await trash.PlanDeleteAsync(sequence.Id, [.. frames.Select(f => f.Id)], null, fx.Ct)).Blocked);
Assert.SequenceEqual([FrameDeleteBlock.FramesChanged], (await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id], sequence.FramesVersion - 1, fx.Ct)).Blocked);
Assert.SequenceEqual([FrameDeleteBlock.FramesChanged], (await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id, Guid.CreateVersion7()], null, fx.Ct)).Blocked);
await Assert.ThrowsAsync<ArgumentException>(() => trash.PlanDeleteAsync(Guid.CreateVersion7(), [frames[0].Id], null, fx.Ct));
// A job on the sequence, or on its whole archive, holds it.
await fx.Db.Jobs.EnqueueAsync(Running(JobKind.Render, [sequence.Id]));
var busy = await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id], null, fx.Ct);
Assert.SequenceEqual([FrameDeleteBlock.Busy], busy.Blocked);
Assert.False(busy.CanDelete);
await Assert.ThrowsAsync<StaleDeletePlanException>(() => trash.DeleteAsync(busy, null, null, fx.Ct));
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task DeletingMovesThePhotosToTheTrashAndTombstonesThem()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
var folder = fx.Paths.Sequence(sequence.Folder);
// Proxies are named by frame index: they follow their frames.
Directory.CreateDirectory(folder.Proxies);
for (var i = 0; i < frames.Count; i++) File.WriteAllBytes(folder.Proxy(i), [(byte)i]);
await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt, SequenceStatus.None);
var result = await DeleteAsync(fx, sequence, frames[1], frames[3]);
foreach (var gone in new[] { frames[1], frames[3] }) Assert.FileMissing(fx.Paths.FromRelative(gone.RelativePath));
var entry = result.Entry;
Assert.Equal(TrashKind.Photos, entry.Kind);
Assert.True(FrameTrash.IsPhotoEntry(entry.Name));
Assert.Equal(2, entry.Frames);
Assert.Equal(frames[1].Size + frames[3].Size, entry.Bytes);
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), entry.Name[FrameTrash.EntryPrefix.Length..]);
Assert.Equal(frames[1].Sha256, Sha(Path.Combine(entryFolder, "originals", Path.GetFileName(frames[1].RelativePath))));
Assert.FileMissing(folder.RemovalJournalPath);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
Assert.SequenceEqual([frames[0].Id, frames[2].Id, frames[4].Id], manifest.Frames.Select(f => f.Id));
Assert.SequenceEqual([0, 1, 2], manifest.Frames.Select(f => f.I));
Assert.Equal(sequence.FramesVersion + 1, manifest.Versions.Frames);
Assert.SequenceEqual([frames[1].Sha256, frames[3].Sha256], Assert.NotNull(manifest.Removed).Select(r => r.Sha256));
Assert.True(manifest.Removed.All(r => r.Reason == RemovalReason.Deleted));
var row = result.Sequence;
Assert.Equal(3, row.FrameCount);
Assert.Equal(manifest.Versions.Frames, row.FramesVersion);
Assert.Equal(SequenceStatus.None, row.Status & (SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt));
Assert.SequenceEqual([frames[0].Id, frames[2].Id, frames[4].Id], (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.SequenceEqual([frames[1].Sha256, frames[3].Sha256], (await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id)).Select(r => r.Sha256).Order());
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[3].Sha256!.ToUpperInvariant()));
Assert.SequenceEqual<byte>([0, 2, 4], Enumerable.Range(0, 3).Select(i => File.ReadAllBytes(folder.Proxy(i))[0]));
Assert.FileMissing(folder.Proxy(3));
Assert.FileMissing(folder.Proxy(4));
var listed = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == entry.Name);
Assert.Equal(sequence.DisplayTitle, listed.Title);
Assert.Equal("alberto", listed.DeletedBy);
}
[Test]
public async Task PositionsOnDeletedPhotosWaitAndComeBackWithThem()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
var folder = fx.Paths.Sequence(sequence.Folder);
var project = new ProjectSettings { ExcludedFrames = [1, 4], HeroFrame = 3 }.ToJson();
var version = (await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = project, HeroFrameId = frames[3].Id }, null, fx.Ct))
.Sequence!.SettingsVersion;
var deleted = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[1], frames[3]);
var row = deleted.Sequence;
var parked = ProjectSettings.FromJson(row.SettingsJson!);
Assert.SequenceEqual([2], parked.ExcludedFrames, "the exclusion of the fifth photo follows it");
Assert.Null(parked.HeroFrame);
Assert.Count(2, Assert.NotNull(parked.PendingFrames));
Assert.True(parked.HeroPending, "the hero waits for its photo");
Assert.Null(row.HeroFrameId);
Assert.Equal(version, row.SettingsVersion);
Assert.True(File.Exists(folder.ProjectPath), "project.json mirrors the index");
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(TrashKind.Photos, restored.Kind);
Assert.Equal(2, restored.Photos);
Assert.Empty(restored.Kept);
var back = (await fx.Db.Sequences.GetAsync(sequence.Id))!;
var placed = ProjectSettings.FromJson(back.SettingsJson!);
Assert.SequenceEqual([1, 4], placed.ExcludedFrames);
Assert.Equal(3, placed.HeroFrame);
Assert.Null(placed.PendingFrames);
Assert.Equal(frames[3].Id, back.HeroFrameId);
Assert.Equal(version, back.SettingsVersion);
}
[Test]
public async Task ARestorePutsThePhotosBackWithTheirIdsAndDropsTheirTombstones()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var folder = fx.Paths.Sequence(sequence.Folder);
Directory.CreateDirectory(folder.Proxies);
for (var i = 0; i < frames.Count; i++) File.WriteAllBytes(folder.Proxy(i), [(byte)i]);
var deleted = await DeleteAsync(fx, sequence, frames[0], frames[2]);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.False(restored.NewSequence);
Assert.Equal(sequence.Id, restored.Sequence.Id);
var now = await fx.Db.Frames.ListBySequenceAsync(sequence.Id);
Assert.SequenceEqual(frames.Select(f => f.Id), now.Select(f => f.Id));
Assert.SequenceEqual(frames.Select(f => f.RelativePath), now.Select(f => f.RelativePath));
Assert.SequenceEqual(frames.Select(f => f.Sha256), now.Select(f => f.Sha256));
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.Null(ManifestStore.ReadSequence(folder.ManifestPath).Removed);
Assert.Empty(await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id));
Assert.Equal(sequence.FramesVersion + 2, restored.Sequence.FramesVersion);
Assert.DoesNotContain(deleted.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
Assert.False(Directory.Exists(Path.Combine(FrameTrash.Directory(fx.Paths), deleted.Entry.Name[FrameTrash.EntryPrefix.Length..])));
// The proxies of the frames that stayed are back at their index; those of the photos that went are made again.
Assert.SequenceEqual<byte>([1], File.ReadAllBytes(folder.Proxy(1)));
Assert.SequenceEqual<byte>([3], File.ReadAllBytes(folder.Proxy(3)));
Assert.FileMissing(folder.Proxy(0));
}
[Test]
public async Task ARestoreKeepsAPhotoWhoseFileNoLongerMatchesItsRecord()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), deleted.Entry.Name[FrameTrash.EntryPrefix.Length..]);
var damaged = Path.Combine(entryFolder, "originals", Path.GetFileName(frames[2].RelativePath));
var bytes = File.ReadAllBytes(damaged);
bytes[^10] ^= 0x01;
File.WriteAllBytes(damaged, bytes);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
Assert.SequenceEqual([new KeptPhoto(frames[2].OriginalFileName!, KeptReason.Mismatch)], restored.Kept);
Assert.SequenceEqual([frames[0].Id, frames[1].Id, frames[3].Id], (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.SequenceEqual([frames[2].Sha256], (await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id)).Select(r => r.Sha256));
var left = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal(1, left.Frames);
Assert.FileExists(damaged);
// Nothing left that matches: the entry stays as it is.
await Assert.ThrowsAsync<TrashMismatchException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct));
Assert.FileExists(damaged);
}
[Test]
public async Task PhotosOfASequenceThatIsGoneComeBackAsANewOne()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
var whole = await fx.Service.PlanDeleteAsync([sequence.Id], null, fx.Ct);
await fx.Service.DeleteAsync(whole, null, null, fx.Ct);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.True(restored.NewSequence);
Assert.NotEqual(sequence.Id, restored.Sequence.Id);
Assert.Equal(sequence.DisplayTitle, restored.Sequence.DisplayTitle);
var frame = (await fx.Db.Frames.ListBySequenceAsync(restored.Sequence.Id)).Single();
Assert.Equal(frames[1].Sha256, frame.Sha256);
Assert.Equal(frames[1].Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
// The whole sequence is still in the trash, restorable on its own.
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Kind == TrashKind.Sequence && e.SequenceId == sequence.Id);
}
[Test]
public async Task TombstonesSurviveARebuildFromTheSidecars()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
await DeleteAsync(fx, sequence, frames[0]);
await fx.ReopenWithEmptyIndexAsync();
await fx.ScanAsync(ScanMode.Rebuild);
var removals = await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id);
Assert.SequenceEqual([frames[0].Sha256], removals.Select(r => r.Sha256));
Assert.Equal(RemovalReason.Deleted, removals[0].Reason);
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task TombstonesAreReplacedWholeAndGoWithTheirSequence()
{
var fx = await FixtureAsync();
var (sequence, _) = await UploadedAsync(fx, 1);
var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero);
var many = Enumerable.Range(0, 450).Select(i => new FrameRemoval(sequence.Id, i.ToString("x64", System.Globalization.CultureInfo.InvariantCulture),
i % 2 == 0 ? null : new string('A', 64), i % 2 == 0 ? RemovalReason.Deleted : RemovalReason.Replaced, at)).ToList();
Assert.Equal(450, await fx.Db.FrameRemovals.SyncSequenceAsync(sequence.Id, many, fx.Ct));
var stored = await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id, fx.Ct);
Assert.Count(450, stored);
Assert.Equal((new string('a', 64), RemovalReason.Replaced), (stored.Single(r => r.Sha256 == many[1].Sha256).ReplacementSha256, stored[1].Reason));
Assert.Equal(1, await fx.Db.FrameRemovals.SyncSequenceAsync(sequence.Id, [many[3], many[3]], fx.Ct));
Assert.Count(1, await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id, fx.Ct));
await fx.Db.Sequences.DeleteAsync(sequence.Id, fx.Ct);
Assert.Empty(await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id, fx.Ct));
}
[Test]
public async Task ADeletionACrashInterruptedIsUndoneByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
// The journal was written and one of two files moved when the process died; the sidecar still lists both frames.
var entry = Path.Combine(FrameTrash.Directory(fx.Paths), $"{sequence.Id:N}-20261027120000");
var record = entry + ".json";
Directory.CreateDirectory(Path.Combine(entry, "originals"));
File.WriteAllText(record, "{}");
var moves = frames.Take(2).Select(f => new FrameRemovalMove(f.RelativePath[(sequence.Folder.Length + 1)..],
fx.Paths.ToRelative(Path.Combine(entry, "originals", Path.GetFileName(f.RelativePath))))).ToList();
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(entry), [frames[0].Id, frames[1].Id], moves));
File.Move(fx.Paths.FromRelative(frames[0].RelativePath), fx.Paths.FromRelative(moves[0].To));
// Any writer of the sequence reads its sidecar under the lease first.
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.FileMissing(folder.RemovalJournalPath);
Assert.FileMissing(record);
Assert.False(Directory.Exists(entry));
Assert.Count(3, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
}
[Test]
public async Task ErasingTheCardKeepsTheCardCopyOfADeletedPhoto()
{
var fx = await FixtureAsync();
var card = Context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 4);
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592), fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null), Guid.NewGuid(), 0,
_ => ValueTask.CompletedTask, null, fx.Ct);
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions.Single().SessionKey))!.Value;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(id))!, frames[2]);
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
CardEraser.Execute(erase);
// The deleted photo has no frame any more: nothing in the archive vouches for its card copy, which stays.
Assert.Equal(3, erase.ToDelete.Count());
var left = Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetFileName(f)).ToList();
Assert.SequenceEqual([frames[2].OriginalFileName!], left);
}
[Test]
public async Task EmptyingErasesExactlyThePlannedEntries()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var (other, _) = await UploadedAsync(fx, 2, "upload-2", 200);
var photos = await DeleteAsync(fx, sequence, frames[1]);
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([other.Id], null, fx.Ct), null, null, fx.Ct);
var engine = await fx.StartEngineAsync();
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
Assert.Count(2, plan.Entries);
Assert.Equal(3, plan.Photos);
Assert.True(plan.Bytes > 0);
Assert.Equal(plan.Fingerprint, (await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct)).Fingerprint, "the plan reads only");
Assert.SequenceEqual(["nope"], (await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [photos.Entry.Name, "nope"], fx.Ct)).Unknown);
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
Assert.Empty(Directory.EnumerateFileSystemEntries(FrameTrash.Directory(fx.Paths)));
Assert.Empty(Directory.EnumerateFiles(SequenceTrash.TrashDirectory(fx.Paths), "*", SearchOption.AllDirectories));
Assert.Contains(Chronos.Core.Localization.Loc.L("Archive.Log.TrashEmptied", 2, Housekeeping.CleanupSteps.Megabytes(plan.Bytes)), done.LogTail);
// Tombstones outlive the trash: the deleted photo is still not offered back.
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[1].Sha256!));
}
[Test]
public async Task AnEmptyingPlannedBeforeTheTrashChangedErasesNothing()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var first = await DeleteAsync(fx, sequence, frames[1]);
var second = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[2]);
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var engine = await fx.StartEngineAsync();
// A planned entry came back (restored) before the job ran.
await fx.Service.RestoreEntryAsync(fx.Archive.Id, first.Entry.Name, null, fx.Ct);
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Archive.Error.TrashChanged"), failed.Error);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == second.Entry.Name);
// A planned entry whose folder changed since makes the plan stale too.
var again = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var folder = Path.Combine(FrameTrash.Directory(fx.Paths), second.Entry.Name[FrameTrash.EntryPrefix.Length..]);
File.WriteAllBytes(Path.Combine(folder, "added.bin"), [1, 2, 3]);
Assert.NotEqual(again.Fingerprint, (await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct)).Fingerprint);
var stale = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, again), cancellationToken: fx.Ct);
await fx.WaitForAsync(stale.Id, JobState.Failed);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == second.Entry.Name);
Assert.FileExists(Path.Combine(folder, "added.bin"));
}
[Test]
public async Task AReplacedEntryIsErasedOnlyWhileItsReplacementMatches()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 2);
var replacement = frames[1];
var good = WriteReplaced(fx, sequence, "20261027100000", replacement.Sha256!);
// A replacement no frame of the sequence holds any more, and whose deletion the sidecar records (its conversion deleted
// and erased): the dry run says so, and the emptying confirmed with it erases the originals (owner review 2, R7).
var deleted = new string('a', 64);
var gone = WriteReplaced(fx, sequence, "20261027110000", deleted);
var sidecar = fx.Paths.Sequence(sequence.Folder).ManifestPath;
var manifest = ManifestStore.ReadSequence(sidecar);
ManifestStore.WriteSequence(sidecar, manifest with
{
Removed = [.. (manifest.Removed ?? []), new RemovedPhoto { Sha256 = deleted, Reason = RemovalReason.Deleted, At = DateTimeOffset.UtcNow }],
});
// One that nothing in the archive says was deleted stays: the sidecar holds no frame of it and no tombstone of it, as a
// sidecar rebuilt from the photos shows a conversion it never knew.
var untraced = WriteReplaced(fx, sequence, "20261027113000", new string('b', 64));
var engine = await fx.StartEngineAsync();
var listed = await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct);
Assert.True(listed.All(e => e.Kind == TrashKind.Replaced));
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
Assert.True(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + gone).ConversionDeleted);
Assert.False(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + good).ConversionDeleted);
Assert.False(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + untraced).ConversionDeleted, "the archive does not say it was deleted");
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
var kept = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains(Chronos.Core.Localization.Loc.L("Archive.Log.TrashKeptUnknown", sequence.DisplayTitle), kept.LogTail);
Assert.Equal(FrameTrash.EntryPrefix + untraced, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name, "the others are erased");
// A replacement whose file no longer hashes to it keeps its originals.
var other = WriteReplaced(fx, sequence, "20261027120000", replacement.Sha256!);
var file = fx.Paths.FromRelative(replacement.RelativePath);
var bytes = File.ReadAllBytes(file);
bytes[^10] ^= 0x01;
File.WriteAllBytes(file, bytes);
var again = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [FrameTrash.EntryPrefix + other], fx.Ct);
var second = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, again), cancellationToken: fx.Ct);
await fx.WaitForAsync(second.Id, JobState.Failed);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == FrameTrash.EntryPrefix + other);
}
[Test]
public async Task CompanionsGoWithTheirPhotoAndFollowItsNameBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var second = fx.Paths.FromRelative(frames[1].RelativePath);
var third = fx.Paths.FromRelative(frames[2].RelativePath);
// An upper-case extension (listed on Linux too), and one named after the whole file name.
var upper = Path.ChangeExtension(second, ".XMP");
var whole = third + ".xmp";
File.WriteAllText(upper, "<rating>2</rating>");
File.WriteAllText(whole, "<rating>3</rating>");
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
Assert.FileMissing(upper);
Assert.FileMissing(whole);
Assert.Equal((2, 4), (deleted.Entry.Frames, deleted.Entry.Files));
var entryFolder = EntryFolder(fx, deleted.Entry);
Assert.Equal("<rating>2</rating>", File.ReadAllText(Path.Combine(entryFolder, "originals", Path.GetFileName(upper))));
Assert.Equal("<rating>3</rating>", File.ReadAllText(Path.Combine(entryFolder, "originals", Path.GetFileName(whole))));
// A file took the second photo's name since: it comes back as _2, and its XMP follows that name.
File.WriteAllBytes(second, Photos.Jpeg(Start, 999));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(2, restored.Photos);
Assert.Empty(restored.Kept);
var now = await fx.Db.Frames.ListBySequenceAsync(sequence.Id);
Assert.SequenceEqual(frames.Select(f => f.Id), now.Select(f => f.Id));
var renamed = fx.Paths.FromRelative(now[1].RelativePath);
Assert.Equal($"{Path.GetFileNameWithoutExtension(second)}_2{Path.GetExtension(second)}", Path.GetFileName(renamed));
Assert.Equal(frames[1].Sha256, Sha(renamed));
Assert.Equal("<rating>2</rating>", File.ReadAllText(Path.ChangeExtension(renamed, ".XMP")));
Assert.Equal(frames[2].RelativePath, now[2].RelativePath);
Assert.Equal("<rating>3</rating>", File.ReadAllText(whole));
Assert.False(Directory.Exists(entryFolder));
}
[Test]
public async Task APhotoRestoredIntoANewSequenceTakesANameItsCompanionCanFollow()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var name = fx.Paths.FromRelative(frames[1].RelativePath);
File.WriteAllText(Path.ChangeExtension(name, ".xmp"), "first");
var first = await DeleteAsync(fx, sequence, frames[1]);
// Another photo filed under the same name since, with an XMP of its own, deleted in turn.
var (path, size, sha) = fx.Stage(Start.AddSeconds(10), 300);
var again = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, path, frames[1].OriginalFileName!, size, sha, null, null), fx.Ct);
Assert.Equal(frames[1].RelativePath, again.RelativePath);
File.WriteAllText(Path.ChangeExtension(name, ".xmp"), "second");
var row = (await fx.Db.Sequences.GetAsync(sequence.Id))!;
var second = await DeleteAsync(fx, row, (await fx.Db.Frames.GetAsync(again.FrameId))!);
await DeleteSequenceAsync(fx, sequence.Id);
var one = await fx.Service.RestoreEntryAsync(fx.Archive.Id, first.Entry.Name, null, fx.Ct);
var two = await fx.Service.RestoreEntryAsync(fx.Archive.Id, second.Entry.Name, null, fx.Ct);
Assert.Equal(one.Sequence.Id, two.Sequence.Id);
var back = await fx.Db.Frames.ListBySequenceAsync(one.Sequence.Id);
Assert.Count(2, back);
foreach (var (frame, xmp) in back.OrderBy(f => f.RelativePath, StringComparer.Ordinal).Zip(["first", "second"]))
{
var file = fx.Paths.FromRelative(frame.RelativePath);
Assert.Equal(frame.Sha256, Sha(file));
Assert.Equal(xmp, File.ReadAllText(Path.ChangeExtension(file, ".xmp")));
}
Assert.DoesNotContain(TrashKind.Photos, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Kind));
}
[Test]
public async Task TwoRestoresOfOneDeletedSequenceAtOnceLoseNothing()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var first = await DeleteAsync(fx, sequence, frames[1]);
var second = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[2]);
await DeleteSequenceAsync(fx, sequence.Id);
var restores = await Task.WhenAll(
Task.Run(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, first.Entry.Name, null, fx.Ct)),
Task.Run(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, second.Entry.Name, null, fx.Ct)));
Assert.Equal(restores[0].Sequence.Id, restores[1].Sequence.Id);
var back = await fx.Db.Frames.ListBySequenceAsync(restores[0].Sequence.Id);
Assert.SequenceEqual([frames[1].Sha256, frames[2].Sha256], back.Select(f => f.Sha256));
foreach (var frame in back) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.False(Directory.Exists(EntryFolder(fx, first.Entry)));
Assert.False(Directory.Exists(EntryFolder(fx, second.Entry)));
}
[Test]
public async Task AnOriginalAStoppedRestoreLeftStagedSurvivesMaintenanceAndComesBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
await DeleteSequenceAsync(fx, sequence.Id);
var entryFolder = EntryFolder(fx, deleted.Entry);
// A restore into a new sequence stopped (the service, or a crash) with an original staged in its entry, days ago.
var name = Path.GetFileName(frames[2].RelativePath);
var staged = Path.Combine(entryFolder, ".restoring", "originals", name);
Directory.CreateDirectory(Path.GetDirectoryName(staged)!);
File.Move(Path.Combine(entryFolder, "originals", name), staged);
File.SetLastWriteTimeUtc(staged, DateTime.UtcNow.AddDays(-3));
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(frames[2].Sha256, Sha(staged));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(2, restored.Photos);
Assert.Empty(restored.Kept);
var back = await fx.Db.Frames.ListBySequenceAsync(restored.Sequence.Id);
Assert.SequenceEqual([frames[1].Sha256, frames[2].Sha256], back.Select(f => f.Sha256));
foreach (var frame in back) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.False(Directory.Exists(entryFolder));
Assert.False(Directory.Exists(fx.Paths.StagingDirectory) && Directory.EnumerateDirectories(fx.Paths.StagingDirectory, "trash-*").Any());
}
[Test]
public async Task ARestoreIntoANewSequenceThatFailsPutsTheOriginalBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
await DeleteSequenceAsync(fx, sequence.Id);
var entryFolder = EntryFolder(fx, deleted.Entry);
var original = Path.Combine(entryFolder, "originals", Path.GetFileName(frames[1].RelativePath));
await FailAsync(fx, "sequences", "INSERT");
// The ingest fails after the original left the entry, with no file error: it goes back all the same.
await Assert.ThrowsAsync<DatabaseException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct));
Assert.Equal(frames[1].Sha256, Sha(original));
Assert.False(Directory.Exists(Path.Combine(entryFolder, ".restoring")));
var listed = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal((1, deleted.Entry.Bytes), (listed.Frames, listed.Bytes));
await HealAsync(fx, "sequences");
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
Assert.Equal(frames[1].Sha256, (await fx.Db.Frames.ListBySequenceAsync(restored.Sequence.Id)).Single().Sha256);
Assert.False(Directory.Exists(entryFolder));
}
[Test]
public async Task ARestoreACrashInterruptedBeforeTheSidecarIsUndoneByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
// The journal was written and one of two originals moved back when the process died; the sidecar lists neither.
var journal = RestoreJournal(fx, deleted.Entry);
journal.Write(folder);
var moved = journal.Photos[0].Moves[0];
File.Move(fx.Paths.FromRelative(moved.To), folder.File(moved.From));
// Any writer of the sequence reads its sidecar under the lease first.
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
Assert.FileMissing(folder.RestoreJournalPath);
Assert.FileMissing(folder.File(moved.From));
Assert.Equal(frames[1].Sha256, Sha(fx.Paths.FromRelative(moved.To)));
Assert.Count(2, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
Assert.Equal(2, Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name).Frames);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(2, restored.Photos);
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
}
[Test]
public async Task ARestoreACrashInterruptedAfterTheSidecarIsCompletedByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
var entryFolder = EntryFolder(fx, deleted.Entry);
// Every file moved back and the sidecar was written when the process died; the trash record still lists the photos.
var journal = RestoreJournal(fx, deleted.Entry);
journal.Write(folder);
foreach (var move in journal.Photos.SelectMany(p => p.Moves)) File.Move(fx.Paths.FromRelative(move.To), folder.File(move.From));
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
var back = SequenceFrames.Insert(manifest, [.. FrameTrash.ReadEntry(fx.Paths, deleted.Entry.Name)!.Photos.Select(p => p.Frame)]);
ManifestStore.WriteSequence(folder.ManifestPath, back with { Removed = null });
// The journal makes a scan read the folder under the lease, which settles the restore and indexes its frames.
await fx.ScanAsync();
Assert.FileMissing(folder.RestoreJournalPath);
Assert.DoesNotContain(deleted.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
Assert.False(Directory.Exists(entryFolder));
Assert.FileMissing(FrameTrash.Location(fx.Paths, deleted.Entry.Name).Record);
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
}
[Test]
public async Task ARestoreCancelledBeforeItsFirstMoveLeavesEverythingAsItWas()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
using var cancel = CancellationTokenSource.CreateLinkedTokenSource(fx.Ct);
// Another process holds the sequence: the restore waits for it, and is cancelled meanwhile.
using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, $"seq-{sequence.Id:N}", "other")))
{
var restore = fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, cancel.Token);
cancel.CancelAfter(TimeSpan.FromMilliseconds(200));
await Assert.ThrowsAsync<OperationCanceledException>(() => restore);
}
Assert.FileMissing(folder.RestoreJournalPath);
Assert.FileMissing(fx.Paths.FromRelative(frames[1].RelativePath));
Assert.Equal(frames[1].Sha256, Sha(Path.Combine(EntryFolder(fx, deleted.Entry), "originals", Path.GetFileName(frames[1].RelativePath))));
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal(1, (await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct)).Photos);
}
[Test]
public async Task APhotoWhoseCompanionCannotFollowStaysWholeInTheTrash()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var photo = fx.Paths.FromRelative(frames[1].RelativePath);
File.WriteAllText(Path.ChangeExtension(photo, ".xmp"), "edits");
var deleted = await DeleteAsync(fx, sequence, frames[1]);
// Something that is no file holds the companion's place: the original moves back in, the XMP cannot follow it.
Directory.CreateDirectory(Path.ChangeExtension(photo, ".xmp"));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(0, restored.Photos);
Assert.SequenceEqual([new KeptPhoto(frames[1].OriginalFileName!, KeptReason.NotMoved)], restored.Kept);
Assert.FileMissing(photo);
Assert.FileMissing(folder.RestoreJournalPath);
var entryFolder = EntryFolder(fx, deleted.Entry);
Assert.Equal(frames[1].Sha256, Sha(Path.Combine(entryFolder, "originals", Path.GetFileName(photo))));
Assert.Equal("edits", File.ReadAllText(Path.Combine(entryFolder, "originals", Path.GetFileName(Path.ChangeExtension(photo, ".xmp")))));
Assert.Count(2, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[1].Sha256!));
var listed = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal((1, 2), (listed.Frames, listed.Files));
}
[Test]
public async Task AnInterruptedDeletionIsNeitherListedNorEmptiedAndAScanUndoesIt()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
// As a deletion leaves it when the process dies after its first move: journal, record under its pending name, one file moved.
var name = $"{sequence.Id:N}-20261027120000";
var entry = Path.Combine(FrameTrash.Directory(fx.Paths), name);
var record = entry + ".json";
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath).Frames.Take(2).ToList();
var files = sidecar.Select(f => new TrashedFile(f.File, "originals/" + Path.GetFileName(f.File), f.Size, f.Sha256)).ToList();
Directory.CreateDirectory(Path.Combine(entry, "originals"));
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), JsonSerializer.Serialize(new PhotoTrashRecord(TrashKind.Photos, sequence.Id, sequence.DisplayTitle,
sequence.Folder, [.. sidecar.Zip(files, (f, t) => new TrashedPhoto(f, [t], null))], files.Sum(f => f.Size), DateTimeOffset.UtcNow, "alberto"),
TrashJson.Default.PhotoTrashRecord));
var moves = files.Select(f => new FrameRemovalMove(f.From, fx.Paths.ToRelative(Path.Combine(entry, f.Path)))).ToList();
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(entry), [.. sidecar.Select(f => f.Id)], moves));
File.Move(folder.File(moves[0].From), fx.Paths.FromRelative(moves[0].To));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
Assert.Empty((await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct)).Entries);
await Assert.ThrowsAsync<DirectoryNotFoundException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, FrameTrash.EntryPrefix + name, null, fx.Ct));
// The journal makes a scan read the folder under the lease, which undoes the deletion.
await fx.ScanAsync();
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.FileMissing(folder.RemovalJournalPath);
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
Assert.False(Directory.Exists(entry));
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task ADeletionWhoseSidecarWasWrittenIsListedByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
var (entry, record) = FrameTrash.Location(fx.Paths, deleted.Entry.Name);
// As a deletion leaves it when the process dies after its sidecar was written: the record not yet under its own name.
File.Move(record, FrameRemovalJournal.PendingRecord(record));
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(entry), [frames[1].Id], []));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
Assert.FileMissing(folder.RemovalJournalPath);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
}
[Test]
[WindowsOnly]
public async Task ADeletionWhoseSidecarCannotBeWrittenPutsEveryPhotoBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var plan = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [frames[0].Id, frames[2].Id], sequence.FramesVersion, fx.Ct);
// Another program holds the sidecar open: it cannot be replaced.
Exception error;
using (new FileStream(folder.ManifestPath, FileMode.Open, FileAccess.Read, FileShare.Read))
error = await Assert.ThrowsAsync<Exception>(() => fx.Service.FrameTrash.DeleteAsync(plan, "alberto", null, fx.Ct));
Assert.True(error is IOException or UnauthorizedAccessException, error.ToString());
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.FileMissing(folder.RemovalJournalPath);
Assert.Empty(Directory.EnumerateFileSystemEntries(FrameTrash.Directory(fx.Paths)));
Assert.Count(3, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
}
[Test]
public async Task APhotoNeverHashedIsHashedAsItGoesAndLeavesItsTombstone()
{
var fx = await FixtureAsync();
var (sequence, frames) = await AdoptedAsync(fx, "Shoot", 4);
Assert.True(frames.All(f => f.Sha256 is null), "adopted, not verified");
var sha = Sha(fx.Paths.FromRelative(frames[1].RelativePath));
var deleted = await DeleteAsync(fx, sequence, frames[1]);
var folder = fx.Paths.Sequence(sequence.Folder);
Assert.SequenceEqual([sha], Assert.NotNull(ManifestStore.ReadSequence(folder.ManifestPath).Removed).Select(r => r.Sha256));
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, sha));
Assert.Equal(sha, Sha(Path.Combine(EntryFolder(fx, deleted.Entry), "originals", Path.GetFileName(frames[1].RelativePath))));
// One whose original is gone as well has no content left to name: it goes without a file and without a tombstone,
// and its record stays in the trash, listed until Empty erases it (nothing can bring it back).
var third = Sha(fx.Paths.FromRelative(frames[2].RelativePath));
File.Delete(fx.Paths.FromRelative(frames[3].RelativePath));
var gone = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[2], frames[3]);
Assert.Equal((2, 1), (gone.Entry.Frames, gone.Entry.Files));
Assert.SequenceEqual(new[] { sha, third }.Order(), Assert.NotNull(ManifestStore.ReadSequence(folder.ManifestPath).Removed).Select(r => r.Sha256).Order());
Assert.SequenceEqual([frames[0].Id], ManifestStore.ReadSequence(folder.ManifestPath).Frames.Select(f => f.Id));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, gone.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
Assert.SequenceEqual([KeptReason.NoFile], restored.Kept.Select(k => k.Reason));
var left = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == gone.Entry.Name);
Assert.Equal((1, 0), (left.Frames, left.Files));
await Assert.ThrowsAsync<TrashMismatchException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, gone.Entry.Name, null, fx.Ct));
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [gone.Entry.Name], fx.Ct);
var items = plan.Entries.Select(e => new EmptyTrashItem(e.Name, SequenceTrash.EntryFingerprint(e))).ToList();
var emptied = await fx.Service.Trash.EmptyAsync(fx.Archive, new EmptyTrashParameters(items, plan.Fingerprint), 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(1, emptied.Erased);
Assert.DoesNotContain(gone.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
}
[Test]
[WindowsOnly]
public async Task APhotoNeverHashedThatCannotBeReadWaitsUntilItCan()
{
var fx = await FixtureAsync();
var (sequence, frames) = await AdoptedAsync(fx, "Shoot", 3);
var original = fx.Paths.FromRelative(frames[1].RelativePath);
var sha = Sha(original);
// Another program holds it: it could not be hashed for its tombstone, so the plan refuses it and says which.
using (new FileStream(original, FileMode.Open, FileAccess.Read, FileShare.None))
{
var refused = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [frames[1].Id], sequence.FramesVersion, fx.Ct);
Assert.SequenceEqual([FrameDeleteBlock.Unhashable], refused.Blocked);
Assert.SequenceEqual([frames[1].Id], refused.Unhashable.Select(f => f.Id));
await Assert.ThrowsAsync<StaleDeletePlanException>(() => fx.Service.FrameTrash.DeleteAsync(refused, null, null, fx.Ct));
}
Assert.Equal(sha, Sha(original));
await DeleteAsync(fx, sequence, frames[1]);
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, sha));
}
[Test]
public async Task AFrameThatStaysHashedToFindATwinKeepsItsHashAndOnlyLikelyTwinsAreReadWhole()
{
var fx = await FixtureAsync();
var shoot = Path.Combine(fx.Archive.RootPath, "Raw");
Directory.CreateDirectory(shoot);
// Originals of one size, as uncompressed raw files are: only one whose first and last 64 KiB match a deleted photo's
// is read whole.
const int Size = 300_000;
static byte[] Padded(int seed)
{
var bytes = new byte[Size];
Photos.Jpeg(Start, seed).CopyTo(bytes, 0);
return bytes;
}
var deleted = Padded(1);
var near = (byte[])deleted.Clone();
near[Size / 2] ^= 0xFF;
File.WriteAllBytes(Path.Combine(shoot, "IMG_0001.JPG"), Padded(2));
File.WriteAllBytes(Path.Combine(shoot, "IMG_0002.JPG"), deleted);
File.WriteAllBytes(Path.Combine(shoot, "IMG_0003.JPG"), deleted);
File.WriteAllBytes(Path.Combine(shoot, "IMG_0004.JPG"), near);
await fx.ScanAsync();
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Raw").Id;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.True(frames.All(f => f.Sha256 is null), "adopted, not verified");
await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(id))!, frames.Single(f => f.RelativePath.EndsWith("IMG_0002.JPG", StringComparison.Ordinal)));
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("Raw").ManifestPath);
Assert.Null(sidecar.Removed, "its twin stays");
string? InSidecar(string name) => sidecar.Frames.Single(f => f.File.EndsWith(name, StringComparison.Ordinal)).Sha256;
Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(deleted)), InSidecar("IMG_0003.JPG"));
Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(near)), InSidecar("IMG_0004.JPG"));
Assert.Null(InSidecar("IMG_0001.JPG"), "its first 64 KiB differ: never read whole");
// The index holds the hashes too, so a later deletion does not read those files again.
var now = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.Equal(InSidecar("IMG_0003.JPG"), now.Single(f => f.RelativePath.EndsWith("IMG_0003.JPG", StringComparison.Ordinal)).Sha256);
Assert.Null(now.Single(f => f.RelativePath.EndsWith("IMG_0001.JPG", StringComparison.Ordinal)).Sha256);
}
[Test]
public async Task ADuplicateThatStaysNeverHashedKeepsItsContentOutOfTheTombstones()
{
var fx = await FixtureAsync();
var shoot = Path.Combine(fx.Archive.RootPath, "Twins");
var files = Photos.WriteSeries(shoot, Start, 3);
File.Copy(files[1], Path.Combine(shoot, "IMG_0009.JPG"));
await fx.ScanAsync();
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Twins").Id;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
var twin = frames.Single(f => f.RelativePath.EndsWith("IMG_0002.JPG", StringComparison.Ordinal));
await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(id))!, twin);
Assert.Null(ManifestStore.ReadSequence(fx.Paths.Sequence("Twins").ManifestPath).Removed);
Assert.False(await fx.Db.FrameRemovals.IsRemovedAsync(id, Sha(Path.Combine(shoot, "IMG_0009.JPG"))));
}
[Test]
public async Task ARestoreBringsBackAPhotoWhoseOnlyTwinInTheSequenceIsMissing()
{
var fx = await FixtureAsync();
var shoot = Path.Combine(fx.Archive.RootPath, "Twins");
var files = Photos.WriteSeries(shoot, Start, 3);
File.Copy(files[1], Path.Combine(shoot, "IMG_0009.JPG"));
await fx.ScanAsync();
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Twins").Id;
await fx.Service.Verifier.VerifyAsync(fx.Archive, [id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
var twins = (await fx.Db.Frames.ListBySequenceAsync(id)).Where(f => f.Sha256 == Sha(files[1])).ToList();
Assert.Count(2, twins);
var deleted = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(id))!, twins[1]);
Assert.False(await fx.Db.FrameRemovals.IsRemovedAsync(id, twins[1].Sha256!), "its twin holds the content");
// The twin that stayed loses its file since; a scan flags it missing.
File.Delete(fx.Paths.FromRelative(twins[0].RelativePath));
await fx.ScanAsync();
Assert.True((await fx.Db.Frames.GetAsync(twins[0].Id))!.Flags.HasFlag(FrameFlags.Missing));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
Assert.Empty(restored.Kept);
var back = Assert.NotNull(await fx.Db.Frames.GetAsync(twins[1].Id));
Assert.Equal(twins[1].Sha256, Sha(fx.Paths.FromRelative(back.RelativePath)));
Assert.False(back.Flags.HasFlag(FrameFlags.Missing));
Assert.False(Directory.Exists(EntryFolder(fx, deleted.Entry)));
}
[Test]
public async Task ASidecarRebuiltFromThePhotosKeepsTheTombstones()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
await DeleteAsync(fx, sequence, frames[0]);
var folder = fx.Paths.Sequence(sequence.Folder);
File.WriteAllText(folder.ManifestPath, "{ \"damaged\": ");
await fx.ScanAsync();
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[0].Sha256!));
Assert.SequenceEqual([frames[0].Sha256], Assert.NotNull(ManifestStore.ReadSequence(folder.ManifestPath).Removed).Select(r => r.Sha256));
// A sequence restored from the trash without its sidecar takes them back from its record.
await DeleteSequenceAsync(fx, sequence.Id);
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(e => e.Kind == TrashKind.Sequence);
File.Delete(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), entry.Name, ".chronos", "sequence.json"));
await fx.Service.RestoreAsync(fx.Archive.Id, entry.Name, fx.Ct);
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[0].Sha256!));
}
[Test]
public async Task ADeletionStandsWhenTheIndexCannotFollowAndTheNextScanResyncsIt()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
await FailAsync(fx, "frame_removals", "INSERT");
var result = await DeleteAsync(fx, sequence, frames[1]);
Assert.FileMissing(fx.Paths.FromRelative(frames[1].RelativePath));
Assert.Equal((sequence.FramesVersion + 1, 2), (result.Sequence.FramesVersion, result.Sequence.FrameCount));
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == result.Entry.Name);
Assert.Equal(sequence.FramesVersion, (await fx.Db.Sequences.GetAsync(sequence.Id))!.FramesVersion);
await HealAsync(fx, "frame_removals");
await fx.ScanAsync();
Assert.Equal(result.Sequence.FramesVersion, (await fx.Db.Sequences.GetAsync(sequence.Id))!.FramesVersion);
Assert.SequenceEqual([frames[0].Id, frames[2].Id], (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[1].Sha256!));
}
[Test]
public async Task ThumbnailsFollowTheirFramesThroughADeletionAndARestore()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
async Task<ImagePassResult> ThumbnailsAsync() =>
await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(sequence.Id))!, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(5, (await ThumbnailsAsync()).Built);
var deleted = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[1]);
var after = await ThumbnailsAsync();
Assert.Equal((0, 4), (after.Built, after.Reused));
await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
var again = await ThumbnailsAsync();
Assert.Equal((1, 4), (again.Built, again.Reused));
}
[Test]
public async Task AnEntryRestoredAndDeletedAgainWhileTheTrashEmptiesIsKept()
{
var fx = await FixtureAsync();
var (a, _) = await UploadedAsync(fx, 2);
var (b, _) = await UploadedAsync(fx, 2, "upload-2", 200);
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([a.Id, b.Id], null, fx.Ct), null, null, fx.Ct);
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var items = plan.Entries.OrderBy(e => e.Name, StringComparer.Ordinal).Select(e => new EmptyTrashItem(e.Name, SequenceTrash.EntryFingerprint(e))).ToList();
var later = items[1].Name;
// Another host sharing the archive restores the second entry and deletes its sequence again once the first is erased.
var result = await fx.Service.Trash.EmptyAsync(fx.Archive, new EmptyTrashParameters(items, plan.Fingerprint), 0, async (done, _) =>
{
if (done != 1) return;
var back = await fx.Service.RestoreAsync(fx.Archive.Id, later, fx.Ct);
await DeleteSequenceAsync(fx, back.Id);
}, fx.Ct);
Assert.Equal(1, result.Erased);
Assert.Count(1, result.Refused);
var kept = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == later);
Assert.True(kept.Files > 0);
Assert.FileExists(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), later, ".chronos", "sequence.json"));
}
[Test]
public async Task AResumedEmptyingKeepsAnEntryThatChangedSinceThePlan()
{
var fx = await FixtureAsync();
var (a, _) = await UploadedAsync(fx, 2);
var (b, _) = await UploadedAsync(fx, 2, "upload-2", 200);
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([a.Id, b.Id], null, fx.Ct), null, null, fx.Ct);
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var names = plan.Entries.Select(e => e.Name).Order(StringComparer.Ordinal).ToList();
File.WriteAllBytes(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), names[1], "added.bin"), [1, 2, 3]);
var engine = await fx.StartEngineAsync();
// An earlier run erased the first entry and stopped; the second changed since the plan.
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.EmptyTrash(fx.Archive, plan), "done:1");
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.Plural("Archive.Error.TrashKept", 1), failed.Error);
var listed = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name).ToList();
Assert.Contains(names[0], listed);
Assert.Contains(names[1], listed);
Assert.FileExists(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), names[1], "added.bin"));
}
[Test]
public async Task ASequenceDeletedAfterAnInterruptedPhotoDeletionTakesNoHalfEntryAlong()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
// As a photo deletion leaves it when the process dies after its first move: journal, pending record, one file moved.
var name = $"{sequence.Id:N}-20261027120000";
var entry = Path.Combine(FrameTrash.Directory(fx.Paths), name);
var record = entry + ".json";
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath).Frames.Take(2).ToList();
var files = sidecar.Select(f => new TrashedFile(f.File, "originals/" + Path.GetFileName(f.File), f.Size, f.Sha256)).ToList();
Directory.CreateDirectory(Path.Combine(entry, "originals"));
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), JsonSerializer.Serialize(new PhotoTrashRecord(TrashKind.Photos, sequence.Id, sequence.DisplayTitle,
sequence.Folder, [.. sidecar.Zip(files, (f, t) => new TrashedPhoto(f, [t], null))], files.Sum(f => f.Size), DateTimeOffset.UtcNow, "alberto"),
TrashJson.Default.PhotoTrashRecord));
var moves = files.Select(f => new FrameRemovalMove(f.From, fx.Paths.ToRelative(Path.Combine(entry, f.Path)))).ToList();
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(entry), [.. sidecar.Select(f => f.Id)], moves));
File.Move(folder.File(moves[0].From), fx.Paths.FromRelative(moves[0].To));
await DeleteSequenceAsync(fx, sequence.Id);
// The deletion was undone first: every photo went with its sequence, and neither its journal nor half an entry is left.
var deleted = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Kind == TrashKind.Sequence);
var trashed = fx.Paths.Sequence(".chronos/trash/" + deleted.Name);
Assert.FileMissing(trashed.RemovalJournalPath);
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(trashed.File(frame.RelativePath[(sequence.Folder.Length + 1)..])));
Assert.False(Directory.Exists(entry));
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
Assert.DoesNotContain(TrashKind.Photos, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Kind));
}
[Test]
public async Task ARestoreWhoseEntryIsBusyWaitsAndNothingGoesOnOverIt()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
var other = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[3]);
// A restore died after moving one original back, before the sidecar; the entry is being emptied meanwhile.
var journal = RestoreJournal(fx, deleted.Entry);
journal.Write(folder);
var moved = journal.Photos[0].Moves[0];
File.Move(fx.Paths.FromRelative(moved.To), folder.File(moved.From));
using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, SequenceTrash.EntryLease(deleted.Entry.Name), "other")))
{
// Another writer leaves the journal for later: it never moves a file into an entry being erased.
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
Assert.FileExists(folder.RestoreJournalPath);
Assert.FileExists(folder.File(moved.From));
Assert.FileMissing(fx.Paths.FromRelative(moved.To));
// A scan does not adopt the photo that restore moved in.
File.SetLastWriteTimeUtc(folder.File(moved.From), DateTime.UtcNow);
await fx.ScanAsync();
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
// Neither a restore of another entry into the sequence nor the sequence's deletion goes on over it.
await Assert.ThrowsAsync<LeaseUnavailableException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, other.Entry.Name, null, fx.Ct));
await Assert.ThrowsAsync<LeaseUnavailableException>(() => DeleteSequenceAsync(fx, sequence.Id));
Assert.FileExists(folder.RestoreJournalPath);
Assert.FileExists(folder.ManifestPath);
}
// Once the entry is free, the next writer settles it: the original goes back into its entry, and both entries come back.
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
Assert.FileMissing(folder.RestoreJournalPath);
Assert.Equal(frames[1].Sha256, Sha(fx.Paths.FromRelative(moved.To)));
Assert.FileMissing(folder.File(moved.From));
Assert.Equal(2, (await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct)).Photos);
Assert.Equal(1, (await fx.Service.RestoreEntryAsync(fx.Archive.Id, other.Entry.Name, null, fx.Ct)).Photos);
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
}
/// <summary>The files of a sequence as its sidecar names them, after checking that the index names the same ones and that each holds its photo.</summary>
private static async Task<List<string>> FilesAsync(ArchiveFixture fx, Guid id)
{
var sequence = Assert.NotNull(await fx.Db.Sequences.GetAsync(id));
var folder = fx.Paths.Sequence(sequence.Folder);
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
var rows = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, folder.ArchiveRelative(f.File))), rows.Select(r => (r.Id, r.RelativePath)));
foreach (var frame in sidecar.Frames) Assert.Equal(frame.Sha256, Sha(folder.File(frame.File)));
return [.. sidecar.Frames.Select(f => f.File)];
}
[Test]
public async Task PhotosDeletedBeforeAReorganizeComeBackBesideTheNamesItGave()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var numbered = new RenameTemplates(File: "{n:000}");
await fx.Service.ApplyRenameAsync(await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [sequence.Id], numbered, fx.Ct), null, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
File.WriteAllText(folder.File("originals/002.xmp"), "second");
await fx.ScanAsync();
// The record keeps the names the photo and its companion had: 002.JPG and 002.xmp.
var deleted = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[1]);
Assert.SequenceEqual(["originals/002.JPG", "originals/002.xmp"], FrameTrash.ReadEntry(fx.Paths, deleted.Entry.Name)!.Photos.Single().Files.Select(f => f.From));
// Moved and renamed since: the photos that stay take 001-003, the deleted photo's name included.
var moved = new RenameTemplates(Folder: "moved/{date}", File: "{n:000}");
await fx.Service.ApplyRenameAsync(await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [sequence.Id], moved, fx.Ct), null, fx.Ct);
Assert.SequenceEqual(["originals/001.JPG", "originals/002.JPG", "originals/003.JPG"], await FilesAsync(fx, sequence.Id));
// Restored into the sequence where it is now, under its own name made free: its companion follows that name.
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id))!;
Assert.Equal("moved/2026-09-28", row.Folder);
Assert.SequenceEqual(["originals/001.JPG", "originals/002_2.JPG", "originals/002.JPG", "originals/003.JPG"], await FilesAsync(fx, sequence.Id));
var now = fx.Paths.Sequence(row.Folder);
Assert.Equal(frames[1].Sha256, Sha(now.File("originals/002_2.JPG")));
Assert.Equal("second", File.ReadAllText(now.File("originals/002_2.xmp")));
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.DoesNotContain(TrashKind.Photos, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Kind));
}
[Test]
public async Task AReorganizeWaitsForARestoreItsEntryHoldsThenSettlesItFirst()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
// A restore died after moving the original back, before the sidecar; the entry is being emptied meanwhile.
var journal = RestoreJournal(fx, deleted.Entry);
journal.Write(folder);
var back = journal.Photos[0].Moves[0];
File.Move(fx.Paths.FromRelative(back.To), folder.File(back.From));
var templates = new RenameTemplates(Folder: "moved/{date}", File: "{n:000}");
var plan = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [sequence.Id], templates, fx.Ct);
Assert.Equal(2, plan.Steps.Single().FileChanges);
var before = await FilesAsync(fx, sequence.Id);
using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, SequenceTrash.EntryLease(deleted.Entry.Name), "other")))
{
// Neither the folder nor a file name changes under that restore.
await Assert.ThrowsAsync<LeaseUnavailableException>(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct));
Assert.Equal(sequence.Folder, (await fx.Db.Sequences.GetAsync(sequence.Id))!.Folder);
Assert.SequenceEqual(before, await FilesAsync(fx, sequence.Id));
Assert.FileExists(folder.RestoreJournalPath);
Assert.FileExists(folder.File(back.From));
}
// Once the entry is free the rename resumes: the restore is settled first (the original goes back into its entry),
// then the folder moves and the files are renamed; the photo then comes back beside them.
Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct));
Assert.Equal(frames[1].Sha256, Sha(fx.Paths.FromRelative(back.To)));
var row = (await fx.Db.Sequences.GetAsync(sequence.Id))!;
Assert.Equal("moved/2026-09-28", row.Folder);
Assert.FileMissing(fx.Paths.Sequence(row.Folder).RestoreJournalPath);
Assert.SequenceEqual(["originals/001.JPG", "originals/002.JPG"], await FilesAsync(fx, sequence.Id));
Assert.Equal(1, (await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct)).Photos);
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.Count(3, await FilesAsync(fx, sequence.Id));
}
[Test]
public async Task ARestoreIntoANewSequenceStoppedBetweenPhotosKeepsItsRecordTrue()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var first = await DeleteAsync(fx, sequence, frames[1]);
var second = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[2], frames[3]);
await DeleteSequenceAsync(fx, sequence.Id);
var made = await fx.Service.RestoreEntryAsync(fx.Archive.Id, first.Entry.Name, null, fx.Ct);
Assert.True(made.NewSequence);
var entryFolder = EntryFolder(fx, second.Entry);
var staged = Path.Combine(entryFolder, ".restoring", "originals", Path.GetFileName(frames[2].RelativePath));
using var cancel = CancellationTokenSource.CreateLinkedTokenSource(fx.Ct);
// The new sequence is held while the first photo waits to be filed into it, and the restore is cancelled meanwhile.
Task restore;
using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, $"seq-{made.Sequence.Id:N}", "other")))
{
restore = fx.Service.RestoreEntryAsync(fx.Archive.Id, second.Entry.Name, null, cancel.Token);
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(20));
while (!File.Exists(staged))
{
Assert.True(DateTime.UtcNow < until, "the first photo was never staged");
await Task.Delay(10, fx.Ct);
}
cancel.Cancel();
}
await Assert.ThrowsAsync<OperationCanceledException>(() => restore);
// The first photo is filed, the second stays, and the record lists exactly what is left.
Assert.SequenceEqual([frames[1].Sha256, frames[2].Sha256], (await fx.Db.Frames.ListBySequenceAsync(made.Sequence.Id)).Select(f => f.Sha256));
var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, second.Entry.Name));
Assert.SequenceEqual([frames[3].Id], record.Photos.Select(p => p.Frame.Id));
Assert.Equal(frames[3].Size, record.Bytes);
Assert.Equal(frames[3].Sha256, Sha(Path.Combine(entryFolder, "originals", Path.GetFileName(frames[3].RelativePath))));
Assert.False(Directory.Exists(Path.Combine(entryFolder, ".restoring")));
var rest = await fx.Service.RestoreEntryAsync(fx.Archive.Id, second.Entry.Name, null, fx.Ct);
Assert.Equal((1, made.Sequence.Id), (rest.Photos, rest.Sequence.Id));
Assert.Empty(rest.Kept);
Assert.False(Directory.Exists(entryFolder));
}
[Test]
public async Task AnIngestThatRefusesRemovedContentChecksTheTombstonesUnderTheLease()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
await DeleteAsync(fx, sequence, frames[1]);
// The same photo staged again (an upload whose completion checked the index before the deletion landed).
var (path, size, sha) = fx.Stage(Start.AddSeconds(10), 101);
Assert.Equal(frames[1].Sha256, sha);
var request = new IngestRequest(fx.Archive.Id, sequence.Id, null, null, path, frames[1].OriginalFileName!, size, sha, null, null, RefuseRemoved: true);
var refused = await Assert.ThrowsAsync<PhotoRemovedException>(() => fx.Service.Ingest.CommitAsync(request, fx.Ct));
Assert.Equal(sequence.Id, refused.SequenceId);
Assert.FileExists(path);
Assert.Count(2, ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath).Frames);
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, sha));
// Asked back, it is filed and its tombstone goes.
var back = await fx.Service.Ingest.CommitAsync(request with { RefuseRemoved = false }, fx.Ct);
Assert.False(back.Duplicate);
Assert.False(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, sha));
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
}
/// <summary>An entry of originals a conversion replaced (a later phase), its one photo replaced by <paramref name="replacement"/>.</summary>
private static string WriteReplaced(ArchiveFixture fx, Sequence sequence, string stamp, string replacement)
{
var name = $"{sequence.Id:N}-{stamp}";
var folder = Path.Combine(FrameTrash.Directory(fx.Paths), name);
Directory.CreateDirectory(Path.Combine(folder, "originals"));
var bytes = Photos.Jpeg(Start, 999);
File.WriteAllBytes(Path.Combine(folder, "originals", "IMG_0001.CR2"), bytes);
var sha = Convert.ToHexStringLower(SHA256.HashData(bytes));
var frame = new SidecarFrame { Id = Guid.CreateVersion7(), File = "originals/IMG_0001.CR2", Size = bytes.Length, Sha256 = sha };
var record = new PhotoTrashRecord(TrashKind.Replaced, sequence.Id, sequence.DisplayTitle, sequence.Folder,
[new TrashedPhoto(frame, [new TrashedFile(frame.File, "originals/IMG_0001.CR2", bytes.Length, sha)], replacement)], bytes.Length,
DateTimeOffset.UtcNow, "converter");
File.WriteAllText(folder + ".json", JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord));
return name;
}
}
public sealed class SqliteFrameTrashTests(TestContext context) : FrameTrashTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresFrameTrashTests(TestContext context) : FrameTrashTests(context)
{
protected override Backend Backend => Backend.Postgres;
}