Files
Chronos/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs
T
Alby96 d0bbd77c92 fix(archive): settle conversions by content and keep their aliases through trash, undo, sync and ingest
Owner review 2, phase 4b, fix round 1 (trash, undo, recovery, sync and ingest).

- Replace journal: records a phase (moving, written, undoing) and the name each swapped
  frame is filed under. Recovery decides "sidecar written" by what the sidecar lists, not
  by the frames version any writer moves; a failed rollback stays undoing and its failure
  now stops the writer (as a photo deletion's recovery does). The ingest settles restores
  and replacements first, so an upload can no longer complete a swap that never happened.
- Emptying a Replaced entry re-verifies each replacement where it is now: in the sequence
  through later conversions (a second conversion no longer strands the first originals),
  or, when the conversion was deleted, its copy in the trash. The dry run marks such
  entries "conversion deleted" (fingerprinted, shown on the desktop and the panel), and
  the confirmed emptying may then erase them instead of keeping them forever.
- Undo of a conversion takes exactly the entries its job made (the job id is recorded on
  the entry), never entries found by time window.
- Project positions translated by photo use the sidecar's photo of every frame still
  there, so positions on a converted frame stay put when another frame went meanwhile.
- Sync aliases are equivalence classes: an original converted on both sides is one photo;
  link-by-content, proxy counting and proxy upload follow aliases; a download into a
  frame missing here swaps the other encoding into that frame instead of adding a frame.
- The ingest answers a converted photo's original (a card or folder imported again, a
  download) as a duplicate of its conversion and keeps the alias tombstone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 3341a2f845753fc4c5bcba51a3b645f126c95eaf)
2026-10-27 22:13:36 +01:00

1310 lines
78 KiB
C#

using System.Security.Cryptography;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Import;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Repositories;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>Deleting photos of a sequence into the archive's trash, restoring them and emptying the trash (owner review 2, R6).</summary>
[Category("archive")]
public abstract class FrameTrashTests(TestContext context) : DatabaseTest(context)
{
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
/// <summary>A sequence of <paramref name="count"/> uploaded photos (hashed, under <c>originals/</c>), ten seconds apart.</summary>
private static async Task<(Sequence Sequence, List<Frame> Frames)> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1", int seed = 100)
{
Guid? id = null;
for (var i = 0; i < count; i++)
{
var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i), seed + i);
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG",
size, sha, null, null), fx.Ct);
id = result.SequenceId;
}
return ((await fx.Db.Sequences.GetAsync(id!.Value))!, [.. await fx.Db.Frames.ListBySequenceAsync(id.Value)]);
}
private static async Task<FrameDeleteResult> DeleteAsync(ArchiveFixture fx, Sequence sequence, params Frame[] frames)
{
var plan = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [.. frames.Select(f => f.Id)], sequence.FramesVersion, fx.Ct);
Assert.True(plan.CanDelete, string.Join(", ", plan.Blocked));
return await fx.Service.FrameTrash.DeleteAsync(plan, "alberto", null, fx.Ct);
}
private static Job Running(JobKind kind, IReadOnlyList<Guid> targets) => new()
{
Id = Guid.CreateVersion7(),
Kind = kind,
TargetIds = targets,
State = JobState.Running,
NodeId = "test#0",
Created = DateTimeOffset.UtcNow,
Started = DateTimeOffset.UtcNow,
};
private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
private static string EntryFolder(ArchiveFixture fx, TrashEntry entry) => FrameTrash.Location(fx.Paths, entry.Name).Folder;
private static async Task DeleteSequenceAsync(ArchiveFixture fx, Guid id) =>
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([id], null, fx.Ct), null, null, fx.Ct);
/// <summary>A sequence a scan adopted in place from photos written into <paramref name="folder"/>: its frames are not hashed.</summary>
private static async Task<(Sequence Sequence, List<Frame> Frames)> AdoptedAsync(ArchiveFixture fx, string folder, int count, int seed = 1)
{
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, folder), Start, count, seed);
await fx.ScanAsync();
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == folder).Id;
return ((await fx.Db.Sequences.GetAsync(id))!, [.. await fx.Db.Frames.ListBySequenceAsync(id)]);
}
/// <summary>The journal a restore of <paramref name="entry"/> into its sequence writes before its first move (names free: as they were).</summary>
private static FrameRestoreJournal RestoreJournal(ArchiveFixture fx, TrashEntry entry)
{
var (folder, record) = FrameTrash.Location(fx.Paths, entry.Name);
var photos = FrameTrash.ReadEntry(fx.Paths, entry.Name)!.Photos;
return new FrameRestoreJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(folder),
[.. photos.Select(p => new RestoredPhoto(p.Frame.Id, [.. p.Files.Select(f => new FrameRemovalMove(f.From, fx.Paths.ToRelative(Path.Combine(folder, f.Path))))]))]);
}
/// <summary>Makes every <paramref name="operation"/> on <paramref name="table"/> fail, as a lost database connection would.</summary>
private async Task FailAsync(ArchiveFixture fx, string table, string operation)
{
await using var session = await fx.Db.Database.OpenAsync(fx.Ct);
await session.ExecuteScriptAsync(ForDialect(
$"CREATE TRIGGER fail_{table} BEFORE {operation} ON {table} BEGIN SELECT RAISE(ABORT, 'injected'); END;",
$"CREATE FUNCTION fail_{table}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected'; END $$; " +
$"CREATE TRIGGER fail_{table} BEFORE {operation} ON {table} FOR EACH ROW EXECUTE FUNCTION fail_{table}();"), fx.Ct);
}
private async Task HealAsync(ArchiveFixture fx, string table)
{
await using var session = await fx.Db.Database.OpenAsync(fx.Ct);
await session.ExecuteScriptAsync(ForDialect($"DROP TRIGGER fail_{table};", $"DROP TRIGGER fail_{table} ON {table};"), fx.Ct);
}
[Test]
public async Task TheDryRunCountsThePhotosAndChangesNothing()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var plan = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [frames[1].Id, frames[3].Id], sequence.FramesVersion, fx.Ct);
Assert.True(plan.CanDelete);
Assert.SequenceEqual([frames[1].Id, frames[3].Id], plan.Frames.Select(f => f.Id));
Assert.Equal(frames[1].Size + frames[3].Size, plan.Bytes);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.False(Directory.Exists(FrameTrash.Directory(fx.Paths)), "a dry run moves nothing");
foreach (var frame in frames) Assert.FileExists(fx.Paths.FromRelative(frame.RelativePath));
}
[Test]
public async Task APlanSaysWhyPhotosCannotGo()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var trash = fx.Service.FrameTrash;
Assert.SequenceEqual([FrameDeleteBlock.WouldEmpty], (await trash.PlanDeleteAsync(sequence.Id, [.. frames.Select(f => f.Id)], null, fx.Ct)).Blocked);
Assert.SequenceEqual([FrameDeleteBlock.FramesChanged], (await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id], sequence.FramesVersion - 1, fx.Ct)).Blocked);
Assert.SequenceEqual([FrameDeleteBlock.FramesChanged], (await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id, Guid.CreateVersion7()], null, fx.Ct)).Blocked);
await Assert.ThrowsAsync<ArgumentException>(() => trash.PlanDeleteAsync(Guid.CreateVersion7(), [frames[0].Id], null, fx.Ct));
// A job on the sequence, or on its whole archive, holds it.
await fx.Db.Jobs.EnqueueAsync(Running(JobKind.Render, [sequence.Id]));
var busy = await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id], null, fx.Ct);
Assert.SequenceEqual([FrameDeleteBlock.Busy], busy.Blocked);
Assert.False(busy.CanDelete);
await Assert.ThrowsAsync<StaleDeletePlanException>(() => trash.DeleteAsync(busy, null, null, fx.Ct));
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task DeletingMovesThePhotosToTheTrashAndTombstonesThem()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
var folder = fx.Paths.Sequence(sequence.Folder);
// Proxies are named by frame index: they follow their frames.
Directory.CreateDirectory(folder.Proxies);
for (var i = 0; i < frames.Count; i++) File.WriteAllBytes(folder.Proxy(i), [(byte)i]);
await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt, SequenceStatus.None);
var result = await DeleteAsync(fx, sequence, frames[1], frames[3]);
foreach (var gone in new[] { frames[1], frames[3] }) Assert.FileMissing(fx.Paths.FromRelative(gone.RelativePath));
var entry = result.Entry;
Assert.Equal(TrashKind.Photos, entry.Kind);
Assert.True(FrameTrash.IsPhotoEntry(entry.Name));
Assert.Equal(2, entry.Frames);
Assert.Equal(frames[1].Size + frames[3].Size, entry.Bytes);
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), entry.Name[FrameTrash.EntryPrefix.Length..]);
Assert.Equal(frames[1].Sha256, Sha(Path.Combine(entryFolder, "originals", Path.GetFileName(frames[1].RelativePath))));
Assert.FileMissing(folder.RemovalJournalPath);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
Assert.SequenceEqual([frames[0].Id, frames[2].Id, frames[4].Id], manifest.Frames.Select(f => f.Id));
Assert.SequenceEqual([0, 1, 2], manifest.Frames.Select(f => f.I));
Assert.Equal(sequence.FramesVersion + 1, manifest.Versions.Frames);
Assert.SequenceEqual([frames[1].Sha256, frames[3].Sha256], Assert.NotNull(manifest.Removed).Select(r => r.Sha256));
Assert.True(manifest.Removed.All(r => r.Reason == RemovalReason.Deleted));
var row = result.Sequence;
Assert.Equal(3, row.FrameCount);
Assert.Equal(manifest.Versions.Frames, row.FramesVersion);
Assert.Equal(SequenceStatus.None, row.Status & (SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt));
Assert.SequenceEqual([frames[0].Id, frames[2].Id, frames[4].Id], (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.SequenceEqual([frames[1].Sha256, frames[3].Sha256], (await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id)).Select(r => r.Sha256).Order());
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[3].Sha256!.ToUpperInvariant()));
Assert.SequenceEqual<byte>([0, 2, 4], Enumerable.Range(0, 3).Select(i => File.ReadAllBytes(folder.Proxy(i))[0]));
Assert.FileMissing(folder.Proxy(3));
Assert.FileMissing(folder.Proxy(4));
var listed = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == entry.Name);
Assert.Equal(sequence.DisplayTitle, listed.Title);
Assert.Equal("alberto", listed.DeletedBy);
}
[Test]
public async Task PositionsOnDeletedPhotosWaitAndComeBackWithThem()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
var folder = fx.Paths.Sequence(sequence.Folder);
var project = new ProjectSettings { ExcludedFrames = [1, 4], HeroFrame = 3 }.ToJson();
var version = (await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = project, HeroFrameId = frames[3].Id }, null, fx.Ct))
.Sequence!.SettingsVersion;
var deleted = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[1], frames[3]);
var row = deleted.Sequence;
var parked = ProjectSettings.FromJson(row.SettingsJson!);
Assert.SequenceEqual([2], parked.ExcludedFrames, "the exclusion of the fifth photo follows it");
Assert.Null(parked.HeroFrame);
Assert.Count(2, Assert.NotNull(parked.PendingFrames));
Assert.True(parked.HeroPending, "the hero waits for its photo");
Assert.Null(row.HeroFrameId);
Assert.Equal(version, row.SettingsVersion);
Assert.True(File.Exists(folder.ProjectPath), "project.json mirrors the index");
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(TrashKind.Photos, restored.Kind);
Assert.Equal(2, restored.Photos);
Assert.Empty(restored.Kept);
var back = (await fx.Db.Sequences.GetAsync(sequence.Id))!;
var placed = ProjectSettings.FromJson(back.SettingsJson!);
Assert.SequenceEqual([1, 4], placed.ExcludedFrames);
Assert.Equal(3, placed.HeroFrame);
Assert.Null(placed.PendingFrames);
Assert.Equal(frames[3].Id, back.HeroFrameId);
Assert.Equal(version, back.SettingsVersion);
}
[Test]
public async Task ARestorePutsThePhotosBackWithTheirIdsAndDropsTheirTombstones()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var folder = fx.Paths.Sequence(sequence.Folder);
Directory.CreateDirectory(folder.Proxies);
for (var i = 0; i < frames.Count; i++) File.WriteAllBytes(folder.Proxy(i), [(byte)i]);
var deleted = await DeleteAsync(fx, sequence, frames[0], frames[2]);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.False(restored.NewSequence);
Assert.Equal(sequence.Id, restored.Sequence.Id);
var now = await fx.Db.Frames.ListBySequenceAsync(sequence.Id);
Assert.SequenceEqual(frames.Select(f => f.Id), now.Select(f => f.Id));
Assert.SequenceEqual(frames.Select(f => f.RelativePath), now.Select(f => f.RelativePath));
Assert.SequenceEqual(frames.Select(f => f.Sha256), now.Select(f => f.Sha256));
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.Null(ManifestStore.ReadSequence(folder.ManifestPath).Removed);
Assert.Empty(await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id));
Assert.Equal(sequence.FramesVersion + 2, restored.Sequence.FramesVersion);
Assert.DoesNotContain(deleted.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
Assert.False(Directory.Exists(Path.Combine(FrameTrash.Directory(fx.Paths), deleted.Entry.Name[FrameTrash.EntryPrefix.Length..])));
// The proxies of the frames that stayed are back at their index; those of the photos that went are made again.
Assert.SequenceEqual<byte>([1], File.ReadAllBytes(folder.Proxy(1)));
Assert.SequenceEqual<byte>([3], File.ReadAllBytes(folder.Proxy(3)));
Assert.FileMissing(folder.Proxy(0));
}
[Test]
public async Task ARestoreKeepsAPhotoWhoseFileNoLongerMatchesItsRecord()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), deleted.Entry.Name[FrameTrash.EntryPrefix.Length..]);
var damaged = Path.Combine(entryFolder, "originals", Path.GetFileName(frames[2].RelativePath));
var bytes = File.ReadAllBytes(damaged);
bytes[^10] ^= 0x01;
File.WriteAllBytes(damaged, bytes);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
Assert.SequenceEqual([new KeptPhoto(frames[2].OriginalFileName!, KeptReason.Mismatch)], restored.Kept);
Assert.SequenceEqual([frames[0].Id, frames[1].Id, frames[3].Id], (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.SequenceEqual([frames[2].Sha256], (await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id)).Select(r => r.Sha256));
var left = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal(1, left.Frames);
Assert.FileExists(damaged);
// Nothing left that matches: the entry stays as it is.
await Assert.ThrowsAsync<TrashMismatchException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct));
Assert.FileExists(damaged);
}
[Test]
public async Task PhotosOfASequenceThatIsGoneComeBackAsANewOne()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
var whole = await fx.Service.PlanDeleteAsync([sequence.Id], null, fx.Ct);
await fx.Service.DeleteAsync(whole, null, null, fx.Ct);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.True(restored.NewSequence);
Assert.NotEqual(sequence.Id, restored.Sequence.Id);
Assert.Equal(sequence.DisplayTitle, restored.Sequence.DisplayTitle);
var frame = (await fx.Db.Frames.ListBySequenceAsync(restored.Sequence.Id)).Single();
Assert.Equal(frames[1].Sha256, frame.Sha256);
Assert.Equal(frames[1].Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
// The whole sequence is still in the trash, restorable on its own.
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Kind == TrashKind.Sequence && e.SequenceId == sequence.Id);
}
[Test]
public async Task TombstonesSurviveARebuildFromTheSidecars()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
await DeleteAsync(fx, sequence, frames[0]);
await fx.ReopenWithEmptyIndexAsync();
await fx.ScanAsync(ScanMode.Rebuild);
var removals = await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id);
Assert.SequenceEqual([frames[0].Sha256], removals.Select(r => r.Sha256));
Assert.Equal(RemovalReason.Deleted, removals[0].Reason);
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task TombstonesAreReplacedWholeAndGoWithTheirSequence()
{
var fx = await FixtureAsync();
var (sequence, _) = await UploadedAsync(fx, 1);
var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero);
var many = Enumerable.Range(0, 450).Select(i => new FrameRemoval(sequence.Id, i.ToString("x64", System.Globalization.CultureInfo.InvariantCulture),
i % 2 == 0 ? null : new string('A', 64), i % 2 == 0 ? RemovalReason.Deleted : RemovalReason.Replaced, at)).ToList();
Assert.Equal(450, await fx.Db.FrameRemovals.SyncSequenceAsync(sequence.Id, many, fx.Ct));
var stored = await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id, fx.Ct);
Assert.Count(450, stored);
Assert.Equal((new string('a', 64), RemovalReason.Replaced), (stored.Single(r => r.Sha256 == many[1].Sha256).ReplacementSha256, stored[1].Reason));
Assert.Equal(1, await fx.Db.FrameRemovals.SyncSequenceAsync(sequence.Id, [many[3], many[3]], fx.Ct));
Assert.Count(1, await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id, fx.Ct));
await fx.Db.Sequences.DeleteAsync(sequence.Id, fx.Ct);
Assert.Empty(await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id, fx.Ct));
}
[Test]
public async Task ADeletionACrashInterruptedIsUndoneByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
// The journal was written and one of two files moved when the process died; the sidecar still lists both frames.
var entry = Path.Combine(FrameTrash.Directory(fx.Paths), $"{sequence.Id:N}-20261027120000");
var record = entry + ".json";
Directory.CreateDirectory(Path.Combine(entry, "originals"));
File.WriteAllText(record, "{}");
var moves = frames.Take(2).Select(f => new FrameRemovalMove(f.RelativePath[(sequence.Folder.Length + 1)..],
fx.Paths.ToRelative(Path.Combine(entry, "originals", Path.GetFileName(f.RelativePath))))).ToList();
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(entry), [frames[0].Id, frames[1].Id], moves));
File.Move(fx.Paths.FromRelative(frames[0].RelativePath), fx.Paths.FromRelative(moves[0].To));
// Any writer of the sequence reads its sidecar under the lease first.
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.FileMissing(folder.RemovalJournalPath);
Assert.FileMissing(record);
Assert.False(Directory.Exists(entry));
Assert.Count(3, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
}
[Test]
public async Task ErasingTheCardKeepsTheCardCopyOfADeletedPhoto()
{
var fx = await FixtureAsync();
var card = Context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 4);
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592), fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null), Guid.NewGuid(), 0,
_ => ValueTask.CompletedTask, null, fx.Ct);
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions.Single().SessionKey))!.Value;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(id))!, frames[2]);
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
CardEraser.Execute(erase);
// The deleted photo has no frame any more: nothing in the archive vouches for its card copy, which stays.
Assert.Equal(3, erase.ToDelete.Count());
var left = Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetFileName(f)).ToList();
Assert.SequenceEqual([frames[2].OriginalFileName!], left);
}
[Test]
public async Task EmptyingErasesExactlyThePlannedEntries()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var (other, _) = await UploadedAsync(fx, 2, "upload-2", 200);
var photos = await DeleteAsync(fx, sequence, frames[1]);
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([other.Id], null, fx.Ct), null, null, fx.Ct);
var engine = await fx.StartEngineAsync();
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
Assert.Count(2, plan.Entries);
Assert.Equal(3, plan.Photos);
Assert.True(plan.Bytes > 0);
Assert.Equal(plan.Fingerprint, (await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct)).Fingerprint, "the plan reads only");
Assert.SequenceEqual(["nope"], (await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [photos.Entry.Name, "nope"], fx.Ct)).Unknown);
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
Assert.Empty(Directory.EnumerateFileSystemEntries(FrameTrash.Directory(fx.Paths)));
Assert.Empty(Directory.EnumerateFiles(SequenceTrash.TrashDirectory(fx.Paths), "*", SearchOption.AllDirectories));
Assert.Contains(Chronos.Core.Localization.Loc.L("Archive.Log.TrashEmptied", 2, Housekeeping.CleanupSteps.Megabytes(plan.Bytes)), done.LogTail);
// Tombstones outlive the trash: the deleted photo is still not offered back.
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[1].Sha256!));
}
[Test]
public async Task AnEmptyingPlannedBeforeTheTrashChangedErasesNothing()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var first = await DeleteAsync(fx, sequence, frames[1]);
var second = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[2]);
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var engine = await fx.StartEngineAsync();
// A planned entry came back (restored) before the job ran.
await fx.Service.RestoreEntryAsync(fx.Archive.Id, first.Entry.Name, null, fx.Ct);
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Archive.Error.TrashChanged"), failed.Error);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == second.Entry.Name);
// A planned entry whose folder changed since makes the plan stale too.
var again = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var folder = Path.Combine(FrameTrash.Directory(fx.Paths), second.Entry.Name[FrameTrash.EntryPrefix.Length..]);
File.WriteAllBytes(Path.Combine(folder, "added.bin"), [1, 2, 3]);
Assert.NotEqual(again.Fingerprint, (await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct)).Fingerprint);
var stale = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, again), cancellationToken: fx.Ct);
await fx.WaitForAsync(stale.Id, JobState.Failed);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == second.Entry.Name);
Assert.FileExists(Path.Combine(folder, "added.bin"));
}
[Test]
public async Task AReplacedEntryIsErasedOnlyWhileItsReplacementMatches()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 2);
var replacement = frames[1];
var good = WriteReplaced(fx, sequence, "20261027100000", replacement.Sha256!);
// A replacement no frame of the sequence holds any more (its conversion deleted and erased): the dry run says so, and
// the emptying confirmed with it erases the originals (owner review 2, R7).
var gone = WriteReplaced(fx, sequence, "20261027110000", new string('a', 64));
var engine = await fx.StartEngineAsync();
var listed = await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct);
Assert.True(listed.All(e => e.Kind == TrashKind.Replaced));
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
Assert.True(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + gone).ConversionDeleted);
Assert.False(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + good).ConversionDeleted);
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
// A replacement whose file no longer hashes to it keeps its originals.
var other = WriteReplaced(fx, sequence, "20261027120000", replacement.Sha256!);
var file = fx.Paths.FromRelative(replacement.RelativePath);
var bytes = File.ReadAllBytes(file);
bytes[^10] ^= 0x01;
File.WriteAllBytes(file, bytes);
var again = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [FrameTrash.EntryPrefix + other], fx.Ct);
var second = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, again), cancellationToken: fx.Ct);
await fx.WaitForAsync(second.Id, JobState.Failed);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == FrameTrash.EntryPrefix + other);
}
[Test]
public async Task CompanionsGoWithTheirPhotoAndFollowItsNameBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var second = fx.Paths.FromRelative(frames[1].RelativePath);
var third = fx.Paths.FromRelative(frames[2].RelativePath);
// An upper-case extension (listed on Linux too), and one named after the whole file name.
var upper = Path.ChangeExtension(second, ".XMP");
var whole = third + ".xmp";
File.WriteAllText(upper, "<rating>2</rating>");
File.WriteAllText(whole, "<rating>3</rating>");
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
Assert.FileMissing(upper);
Assert.FileMissing(whole);
Assert.Equal((2, 4), (deleted.Entry.Frames, deleted.Entry.Files));
var entryFolder = EntryFolder(fx, deleted.Entry);
Assert.Equal("<rating>2</rating>", File.ReadAllText(Path.Combine(entryFolder, "originals", Path.GetFileName(upper))));
Assert.Equal("<rating>3</rating>", File.ReadAllText(Path.Combine(entryFolder, "originals", Path.GetFileName(whole))));
// A file took the second photo's name since: it comes back as _2, and its XMP follows that name.
File.WriteAllBytes(second, Photos.Jpeg(Start, 999));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(2, restored.Photos);
Assert.Empty(restored.Kept);
var now = await fx.Db.Frames.ListBySequenceAsync(sequence.Id);
Assert.SequenceEqual(frames.Select(f => f.Id), now.Select(f => f.Id));
var renamed = fx.Paths.FromRelative(now[1].RelativePath);
Assert.Equal($"{Path.GetFileNameWithoutExtension(second)}_2{Path.GetExtension(second)}", Path.GetFileName(renamed));
Assert.Equal(frames[1].Sha256, Sha(renamed));
Assert.Equal("<rating>2</rating>", File.ReadAllText(Path.ChangeExtension(renamed, ".XMP")));
Assert.Equal(frames[2].RelativePath, now[2].RelativePath);
Assert.Equal("<rating>3</rating>", File.ReadAllText(whole));
Assert.False(Directory.Exists(entryFolder));
}
[Test]
public async Task APhotoRestoredIntoANewSequenceTakesANameItsCompanionCanFollow()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var name = fx.Paths.FromRelative(frames[1].RelativePath);
File.WriteAllText(Path.ChangeExtension(name, ".xmp"), "first");
var first = await DeleteAsync(fx, sequence, frames[1]);
// Another photo filed under the same name since, with an XMP of its own, deleted in turn.
var (path, size, sha) = fx.Stage(Start.AddSeconds(10), 300);
var again = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, path, frames[1].OriginalFileName!, size, sha, null, null), fx.Ct);
Assert.Equal(frames[1].RelativePath, again.RelativePath);
File.WriteAllText(Path.ChangeExtension(name, ".xmp"), "second");
var row = (await fx.Db.Sequences.GetAsync(sequence.Id))!;
var second = await DeleteAsync(fx, row, (await fx.Db.Frames.GetAsync(again.FrameId))!);
await DeleteSequenceAsync(fx, sequence.Id);
var one = await fx.Service.RestoreEntryAsync(fx.Archive.Id, first.Entry.Name, null, fx.Ct);
var two = await fx.Service.RestoreEntryAsync(fx.Archive.Id, second.Entry.Name, null, fx.Ct);
Assert.Equal(one.Sequence.Id, two.Sequence.Id);
var back = await fx.Db.Frames.ListBySequenceAsync(one.Sequence.Id);
Assert.Count(2, back);
foreach (var (frame, xmp) in back.OrderBy(f => f.RelativePath, StringComparer.Ordinal).Zip(["first", "second"]))
{
var file = fx.Paths.FromRelative(frame.RelativePath);
Assert.Equal(frame.Sha256, Sha(file));
Assert.Equal(xmp, File.ReadAllText(Path.ChangeExtension(file, ".xmp")));
}
Assert.DoesNotContain(TrashKind.Photos, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Kind));
}
[Test]
public async Task TwoRestoresOfOneDeletedSequenceAtOnceLoseNothing()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var first = await DeleteAsync(fx, sequence, frames[1]);
var second = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[2]);
await DeleteSequenceAsync(fx, sequence.Id);
var restores = await Task.WhenAll(
Task.Run(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, first.Entry.Name, null, fx.Ct)),
Task.Run(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, second.Entry.Name, null, fx.Ct)));
Assert.Equal(restores[0].Sequence.Id, restores[1].Sequence.Id);
var back = await fx.Db.Frames.ListBySequenceAsync(restores[0].Sequence.Id);
Assert.SequenceEqual([frames[1].Sha256, frames[2].Sha256], back.Select(f => f.Sha256));
foreach (var frame in back) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.False(Directory.Exists(EntryFolder(fx, first.Entry)));
Assert.False(Directory.Exists(EntryFolder(fx, second.Entry)));
}
[Test]
public async Task AnOriginalAStoppedRestoreLeftStagedSurvivesMaintenanceAndComesBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
await DeleteSequenceAsync(fx, sequence.Id);
var entryFolder = EntryFolder(fx, deleted.Entry);
// A restore into a new sequence stopped (the service, or a crash) with an original staged in its entry, days ago.
var name = Path.GetFileName(frames[2].RelativePath);
var staged = Path.Combine(entryFolder, ".restoring", "originals", name);
Directory.CreateDirectory(Path.GetDirectoryName(staged)!);
File.Move(Path.Combine(entryFolder, "originals", name), staged);
File.SetLastWriteTimeUtc(staged, DateTime.UtcNow.AddDays(-3));
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(frames[2].Sha256, Sha(staged));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(2, restored.Photos);
Assert.Empty(restored.Kept);
var back = await fx.Db.Frames.ListBySequenceAsync(restored.Sequence.Id);
Assert.SequenceEqual([frames[1].Sha256, frames[2].Sha256], back.Select(f => f.Sha256));
foreach (var frame in back) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.False(Directory.Exists(entryFolder));
Assert.False(Directory.Exists(fx.Paths.StagingDirectory) && Directory.EnumerateDirectories(fx.Paths.StagingDirectory, "trash-*").Any());
}
[Test]
public async Task ARestoreIntoANewSequenceThatFailsPutsTheOriginalBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
await DeleteSequenceAsync(fx, sequence.Id);
var entryFolder = EntryFolder(fx, deleted.Entry);
var original = Path.Combine(entryFolder, "originals", Path.GetFileName(frames[1].RelativePath));
await FailAsync(fx, "sequences", "INSERT");
// The ingest fails after the original left the entry, with no file error: it goes back all the same.
await Assert.ThrowsAsync<DatabaseException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct));
Assert.Equal(frames[1].Sha256, Sha(original));
Assert.False(Directory.Exists(Path.Combine(entryFolder, ".restoring")));
var listed = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal((1, deleted.Entry.Bytes), (listed.Frames, listed.Bytes));
await HealAsync(fx, "sequences");
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
Assert.Equal(frames[1].Sha256, (await fx.Db.Frames.ListBySequenceAsync(restored.Sequence.Id)).Single().Sha256);
Assert.False(Directory.Exists(entryFolder));
}
[Test]
public async Task ARestoreACrashInterruptedBeforeTheSidecarIsUndoneByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
// The journal was written and one of two originals moved back when the process died; the sidecar lists neither.
var journal = RestoreJournal(fx, deleted.Entry);
journal.Write(folder);
var moved = journal.Photos[0].Moves[0];
File.Move(fx.Paths.FromRelative(moved.To), folder.File(moved.From));
// Any writer of the sequence reads its sidecar under the lease first.
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
Assert.FileMissing(folder.RestoreJournalPath);
Assert.FileMissing(folder.File(moved.From));
Assert.Equal(frames[1].Sha256, Sha(fx.Paths.FromRelative(moved.To)));
Assert.Count(2, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
Assert.Equal(2, Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name).Frames);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(2, restored.Photos);
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
}
[Test]
public async Task ARestoreACrashInterruptedAfterTheSidecarIsCompletedByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
var entryFolder = EntryFolder(fx, deleted.Entry);
// Every file moved back and the sidecar was written when the process died; the trash record still lists the photos.
var journal = RestoreJournal(fx, deleted.Entry);
journal.Write(folder);
foreach (var move in journal.Photos.SelectMany(p => p.Moves)) File.Move(fx.Paths.FromRelative(move.To), folder.File(move.From));
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
var back = SequenceFrames.Insert(manifest, [.. FrameTrash.ReadEntry(fx.Paths, deleted.Entry.Name)!.Photos.Select(p => p.Frame)]);
ManifestStore.WriteSequence(folder.ManifestPath, back with { Removed = null });
// The journal makes a scan read the folder under the lease, which settles the restore and indexes its frames.
await fx.ScanAsync();
Assert.FileMissing(folder.RestoreJournalPath);
Assert.DoesNotContain(deleted.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
Assert.False(Directory.Exists(entryFolder));
Assert.FileMissing(FrameTrash.Location(fx.Paths, deleted.Entry.Name).Record);
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
}
[Test]
public async Task ARestoreCancelledBeforeItsFirstMoveLeavesEverythingAsItWas()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
using var cancel = CancellationTokenSource.CreateLinkedTokenSource(fx.Ct);
// Another process holds the sequence: the restore waits for it, and is cancelled meanwhile.
using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, $"seq-{sequence.Id:N}", "other")))
{
var restore = fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, cancel.Token);
cancel.CancelAfter(TimeSpan.FromMilliseconds(200));
await Assert.ThrowsAsync<OperationCanceledException>(() => restore);
}
Assert.FileMissing(folder.RestoreJournalPath);
Assert.FileMissing(fx.Paths.FromRelative(frames[1].RelativePath));
Assert.Equal(frames[1].Sha256, Sha(Path.Combine(EntryFolder(fx, deleted.Entry), "originals", Path.GetFileName(frames[1].RelativePath))));
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal(1, (await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct)).Photos);
}
[Test]
public async Task APhotoWhoseCompanionCannotFollowStaysWholeInTheTrash()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var photo = fx.Paths.FromRelative(frames[1].RelativePath);
File.WriteAllText(Path.ChangeExtension(photo, ".xmp"), "edits");
var deleted = await DeleteAsync(fx, sequence, frames[1]);
// Something that is no file holds the companion's place: the original moves back in, the XMP cannot follow it.
Directory.CreateDirectory(Path.ChangeExtension(photo, ".xmp"));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(0, restored.Photos);
Assert.SequenceEqual([new KeptPhoto(frames[1].OriginalFileName!, KeptReason.NotMoved)], restored.Kept);
Assert.FileMissing(photo);
Assert.FileMissing(folder.RestoreJournalPath);
var entryFolder = EntryFolder(fx, deleted.Entry);
Assert.Equal(frames[1].Sha256, Sha(Path.Combine(entryFolder, "originals", Path.GetFileName(photo))));
Assert.Equal("edits", File.ReadAllText(Path.Combine(entryFolder, "originals", Path.GetFileName(Path.ChangeExtension(photo, ".xmp")))));
Assert.Count(2, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[1].Sha256!));
var listed = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal((1, 2), (listed.Frames, listed.Files));
}
[Test]
public async Task AnInterruptedDeletionIsNeitherListedNorEmptiedAndAScanUndoesIt()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
// As a deletion leaves it when the process dies after its first move: journal, record under its pending name, one file moved.
var name = $"{sequence.Id:N}-20261027120000";
var entry = Path.Combine(FrameTrash.Directory(fx.Paths), name);
var record = entry + ".json";
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath).Frames.Take(2).ToList();
var files = sidecar.Select(f => new TrashedFile(f.File, "originals/" + Path.GetFileName(f.File), f.Size, f.Sha256)).ToList();
Directory.CreateDirectory(Path.Combine(entry, "originals"));
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), JsonSerializer.Serialize(new PhotoTrashRecord(TrashKind.Photos, sequence.Id, sequence.DisplayTitle,
sequence.Folder, [.. sidecar.Zip(files, (f, t) => new TrashedPhoto(f, [t], null))], files.Sum(f => f.Size), DateTimeOffset.UtcNow, "alberto"),
TrashJson.Default.PhotoTrashRecord));
var moves = files.Select(f => new FrameRemovalMove(f.From, fx.Paths.ToRelative(Path.Combine(entry, f.Path)))).ToList();
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(entry), [.. sidecar.Select(f => f.Id)], moves));
File.Move(folder.File(moves[0].From), fx.Paths.FromRelative(moves[0].To));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
Assert.Empty((await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct)).Entries);
await Assert.ThrowsAsync<DirectoryNotFoundException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, FrameTrash.EntryPrefix + name, null, fx.Ct));
// The journal makes a scan read the folder under the lease, which undoes the deletion.
await fx.ScanAsync();
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.FileMissing(folder.RemovalJournalPath);
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
Assert.False(Directory.Exists(entry));
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task ADeletionWhoseSidecarWasWrittenIsListedByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
var (entry, record) = FrameTrash.Location(fx.Paths, deleted.Entry.Name);
// As a deletion leaves it when the process dies after its sidecar was written: the record not yet under its own name.
File.Move(record, FrameRemovalJournal.PendingRecord(record));
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(entry), [frames[1].Id], []));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
Assert.FileMissing(folder.RemovalJournalPath);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
}
[Test]
[WindowsOnly]
public async Task ADeletionWhoseSidecarCannotBeWrittenPutsEveryPhotoBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var plan = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [frames[0].Id, frames[2].Id], sequence.FramesVersion, fx.Ct);
// Another program holds the sidecar open: it cannot be replaced.
Exception error;
using (new FileStream(folder.ManifestPath, FileMode.Open, FileAccess.Read, FileShare.Read))
error = await Assert.ThrowsAsync<Exception>(() => fx.Service.FrameTrash.DeleteAsync(plan, "alberto", null, fx.Ct));
Assert.True(error is IOException or UnauthorizedAccessException, error.ToString());
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.FileMissing(folder.RemovalJournalPath);
Assert.Empty(Directory.EnumerateFileSystemEntries(FrameTrash.Directory(fx.Paths)));
Assert.Count(3, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
}
[Test]
public async Task APhotoNeverHashedIsHashedAsItGoesAndLeavesItsTombstone()
{
var fx = await FixtureAsync();
var (sequence, frames) = await AdoptedAsync(fx, "Shoot", 4);
Assert.True(frames.All(f => f.Sha256 is null), "adopted, not verified");
var sha = Sha(fx.Paths.FromRelative(frames[1].RelativePath));
var deleted = await DeleteAsync(fx, sequence, frames[1]);
var folder = fx.Paths.Sequence(sequence.Folder);
Assert.SequenceEqual([sha], Assert.NotNull(ManifestStore.ReadSequence(folder.ManifestPath).Removed).Select(r => r.Sha256));
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, sha));
Assert.Equal(sha, Sha(Path.Combine(EntryFolder(fx, deleted.Entry), "originals", Path.GetFileName(frames[1].RelativePath))));
// One whose original is gone as well has no content left to name: it goes without a file and without a tombstone,
// and its record stays in the trash, listed until Empty erases it (nothing can bring it back).
var third = Sha(fx.Paths.FromRelative(frames[2].RelativePath));
File.Delete(fx.Paths.FromRelative(frames[3].RelativePath));
var gone = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[2], frames[3]);
Assert.Equal((2, 1), (gone.Entry.Frames, gone.Entry.Files));
Assert.SequenceEqual(new[] { sha, third }.Order(), Assert.NotNull(ManifestStore.ReadSequence(folder.ManifestPath).Removed).Select(r => r.Sha256).Order());
Assert.SequenceEqual([frames[0].Id], ManifestStore.ReadSequence(folder.ManifestPath).Frames.Select(f => f.Id));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, gone.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
Assert.SequenceEqual([KeptReason.NoFile], restored.Kept.Select(k => k.Reason));
var left = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == gone.Entry.Name);
Assert.Equal((1, 0), (left.Frames, left.Files));
await Assert.ThrowsAsync<TrashMismatchException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, gone.Entry.Name, null, fx.Ct));
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [gone.Entry.Name], fx.Ct);
var items = plan.Entries.Select(e => new EmptyTrashItem(e.Name, SequenceTrash.EntryFingerprint(e))).ToList();
var emptied = await fx.Service.Trash.EmptyAsync(fx.Archive, new EmptyTrashParameters(items, plan.Fingerprint), 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(1, emptied.Erased);
Assert.DoesNotContain(gone.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
}
[Test]
[WindowsOnly]
public async Task APhotoNeverHashedThatCannotBeReadWaitsUntilItCan()
{
var fx = await FixtureAsync();
var (sequence, frames) = await AdoptedAsync(fx, "Shoot", 3);
var original = fx.Paths.FromRelative(frames[1].RelativePath);
var sha = Sha(original);
// Another program holds it: it could not be hashed for its tombstone, so the plan refuses it and says which.
using (new FileStream(original, FileMode.Open, FileAccess.Read, FileShare.None))
{
var refused = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [frames[1].Id], sequence.FramesVersion, fx.Ct);
Assert.SequenceEqual([FrameDeleteBlock.Unhashable], refused.Blocked);
Assert.SequenceEqual([frames[1].Id], refused.Unhashable.Select(f => f.Id));
await Assert.ThrowsAsync<StaleDeletePlanException>(() => fx.Service.FrameTrash.DeleteAsync(refused, null, null, fx.Ct));
}
Assert.Equal(sha, Sha(original));
await DeleteAsync(fx, sequence, frames[1]);
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, sha));
}
[Test]
public async Task AFrameThatStaysHashedToFindATwinKeepsItsHashAndOnlyLikelyTwinsAreReadWhole()
{
var fx = await FixtureAsync();
var shoot = Path.Combine(fx.Archive.RootPath, "Raw");
Directory.CreateDirectory(shoot);
// Originals of one size, as uncompressed raw files are: only one whose first and last 64 KiB match a deleted photo's
// is read whole.
const int Size = 300_000;
static byte[] Padded(int seed)
{
var bytes = new byte[Size];
Photos.Jpeg(Start, seed).CopyTo(bytes, 0);
return bytes;
}
var deleted = Padded(1);
var near = (byte[])deleted.Clone();
near[Size / 2] ^= 0xFF;
File.WriteAllBytes(Path.Combine(shoot, "IMG_0001.JPG"), Padded(2));
File.WriteAllBytes(Path.Combine(shoot, "IMG_0002.JPG"), deleted);
File.WriteAllBytes(Path.Combine(shoot, "IMG_0003.JPG"), deleted);
File.WriteAllBytes(Path.Combine(shoot, "IMG_0004.JPG"), near);
await fx.ScanAsync();
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Raw").Id;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.True(frames.All(f => f.Sha256 is null), "adopted, not verified");
await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(id))!, frames.Single(f => f.RelativePath.EndsWith("IMG_0002.JPG", StringComparison.Ordinal)));
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("Raw").ManifestPath);
Assert.Null(sidecar.Removed, "its twin stays");
string? InSidecar(string name) => sidecar.Frames.Single(f => f.File.EndsWith(name, StringComparison.Ordinal)).Sha256;
Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(deleted)), InSidecar("IMG_0003.JPG"));
Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(near)), InSidecar("IMG_0004.JPG"));
Assert.Null(InSidecar("IMG_0001.JPG"), "its first 64 KiB differ: never read whole");
// The index holds the hashes too, so a later deletion does not read those files again.
var now = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.Equal(InSidecar("IMG_0003.JPG"), now.Single(f => f.RelativePath.EndsWith("IMG_0003.JPG", StringComparison.Ordinal)).Sha256);
Assert.Null(now.Single(f => f.RelativePath.EndsWith("IMG_0001.JPG", StringComparison.Ordinal)).Sha256);
}
[Test]
public async Task ADuplicateThatStaysNeverHashedKeepsItsContentOutOfTheTombstones()
{
var fx = await FixtureAsync();
var shoot = Path.Combine(fx.Archive.RootPath, "Twins");
var files = Photos.WriteSeries(shoot, Start, 3);
File.Copy(files[1], Path.Combine(shoot, "IMG_0009.JPG"));
await fx.ScanAsync();
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Twins").Id;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
var twin = frames.Single(f => f.RelativePath.EndsWith("IMG_0002.JPG", StringComparison.Ordinal));
await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(id))!, twin);
Assert.Null(ManifestStore.ReadSequence(fx.Paths.Sequence("Twins").ManifestPath).Removed);
Assert.False(await fx.Db.FrameRemovals.IsRemovedAsync(id, Sha(Path.Combine(shoot, "IMG_0009.JPG"))));
}
[Test]
public async Task ARestoreBringsBackAPhotoWhoseOnlyTwinInTheSequenceIsMissing()
{
var fx = await FixtureAsync();
var shoot = Path.Combine(fx.Archive.RootPath, "Twins");
var files = Photos.WriteSeries(shoot, Start, 3);
File.Copy(files[1], Path.Combine(shoot, "IMG_0009.JPG"));
await fx.ScanAsync();
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Twins").Id;
await fx.Service.Verifier.VerifyAsync(fx.Archive, [id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
var twins = (await fx.Db.Frames.ListBySequenceAsync(id)).Where(f => f.Sha256 == Sha(files[1])).ToList();
Assert.Count(2, twins);
var deleted = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(id))!, twins[1]);
Assert.False(await fx.Db.FrameRemovals.IsRemovedAsync(id, twins[1].Sha256!), "its twin holds the content");
// The twin that stayed loses its file since; a scan flags it missing.
File.Delete(fx.Paths.FromRelative(twins[0].RelativePath));
await fx.ScanAsync();
Assert.True((await fx.Db.Frames.GetAsync(twins[0].Id))!.Flags.HasFlag(FrameFlags.Missing));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
Assert.Empty(restored.Kept);
var back = Assert.NotNull(await fx.Db.Frames.GetAsync(twins[1].Id));
Assert.Equal(twins[1].Sha256, Sha(fx.Paths.FromRelative(back.RelativePath)));
Assert.False(back.Flags.HasFlag(FrameFlags.Missing));
Assert.False(Directory.Exists(EntryFolder(fx, deleted.Entry)));
}
[Test]
public async Task ASidecarRebuiltFromThePhotosKeepsTheTombstones()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
await DeleteAsync(fx, sequence, frames[0]);
var folder = fx.Paths.Sequence(sequence.Folder);
File.WriteAllText(folder.ManifestPath, "{ \"damaged\": ");
await fx.ScanAsync();
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[0].Sha256!));
Assert.SequenceEqual([frames[0].Sha256], Assert.NotNull(ManifestStore.ReadSequence(folder.ManifestPath).Removed).Select(r => r.Sha256));
// A sequence restored from the trash without its sidecar takes them back from its record.
await DeleteSequenceAsync(fx, sequence.Id);
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(e => e.Kind == TrashKind.Sequence);
File.Delete(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), entry.Name, ".chronos", "sequence.json"));
await fx.Service.RestoreAsync(fx.Archive.Id, entry.Name, fx.Ct);
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[0].Sha256!));
}
[Test]
public async Task ADeletionStandsWhenTheIndexCannotFollowAndTheNextScanResyncsIt()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
await FailAsync(fx, "frame_removals", "INSERT");
var result = await DeleteAsync(fx, sequence, frames[1]);
Assert.FileMissing(fx.Paths.FromRelative(frames[1].RelativePath));
Assert.Equal((sequence.FramesVersion + 1, 2), (result.Sequence.FramesVersion, result.Sequence.FrameCount));
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == result.Entry.Name);
Assert.Equal(sequence.FramesVersion, (await fx.Db.Sequences.GetAsync(sequence.Id))!.FramesVersion);
await HealAsync(fx, "frame_removals");
await fx.ScanAsync();
Assert.Equal(result.Sequence.FramesVersion, (await fx.Db.Sequences.GetAsync(sequence.Id))!.FramesVersion);
Assert.SequenceEqual([frames[0].Id, frames[2].Id], (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[1].Sha256!));
}
[Test]
public async Task ThumbnailsFollowTheirFramesThroughADeletionAndARestore()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
async Task<ImagePassResult> ThumbnailsAsync() =>
await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(sequence.Id))!, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(5, (await ThumbnailsAsync()).Built);
var deleted = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[1]);
var after = await ThumbnailsAsync();
Assert.Equal((0, 4), (after.Built, after.Reused));
await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
var again = await ThumbnailsAsync();
Assert.Equal((1, 4), (again.Built, again.Reused));
}
[Test]
public async Task AnEntryRestoredAndDeletedAgainWhileTheTrashEmptiesIsKept()
{
var fx = await FixtureAsync();
var (a, _) = await UploadedAsync(fx, 2);
var (b, _) = await UploadedAsync(fx, 2, "upload-2", 200);
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([a.Id, b.Id], null, fx.Ct), null, null, fx.Ct);
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var items = plan.Entries.OrderBy(e => e.Name, StringComparer.Ordinal).Select(e => new EmptyTrashItem(e.Name, SequenceTrash.EntryFingerprint(e))).ToList();
var later = items[1].Name;
// Another host sharing the archive restores the second entry and deletes its sequence again once the first is erased.
var result = await fx.Service.Trash.EmptyAsync(fx.Archive, new EmptyTrashParameters(items, plan.Fingerprint), 0, async (done, _) =>
{
if (done != 1) return;
var back = await fx.Service.RestoreAsync(fx.Archive.Id, later, fx.Ct);
await DeleteSequenceAsync(fx, back.Id);
}, fx.Ct);
Assert.Equal(1, result.Erased);
Assert.Count(1, result.Refused);
var kept = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == later);
Assert.True(kept.Files > 0);
Assert.FileExists(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), later, ".chronos", "sequence.json"));
}
[Test]
public async Task AResumedEmptyingKeepsAnEntryThatChangedSinceThePlan()
{
var fx = await FixtureAsync();
var (a, _) = await UploadedAsync(fx, 2);
var (b, _) = await UploadedAsync(fx, 2, "upload-2", 200);
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([a.Id, b.Id], null, fx.Ct), null, null, fx.Ct);
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var names = plan.Entries.Select(e => e.Name).Order(StringComparer.Ordinal).ToList();
File.WriteAllBytes(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), names[1], "added.bin"), [1, 2, 3]);
var engine = await fx.StartEngineAsync();
// An earlier run erased the first entry and stopped; the second changed since the plan.
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.EmptyTrash(fx.Archive, plan), "done:1");
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.Plural("Archive.Error.TrashKept", 1), failed.Error);
var listed = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name).ToList();
Assert.Contains(names[0], listed);
Assert.Contains(names[1], listed);
Assert.FileExists(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), names[1], "added.bin"));
}
[Test]
public async Task ASequenceDeletedAfterAnInterruptedPhotoDeletionTakesNoHalfEntryAlong()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
// As a photo deletion leaves it when the process dies after its first move: journal, pending record, one file moved.
var name = $"{sequence.Id:N}-20261027120000";
var entry = Path.Combine(FrameTrash.Directory(fx.Paths), name);
var record = entry + ".json";
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath).Frames.Take(2).ToList();
var files = sidecar.Select(f => new TrashedFile(f.File, "originals/" + Path.GetFileName(f.File), f.Size, f.Sha256)).ToList();
Directory.CreateDirectory(Path.Combine(entry, "originals"));
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), JsonSerializer.Serialize(new PhotoTrashRecord(TrashKind.Photos, sequence.Id, sequence.DisplayTitle,
sequence.Folder, [.. sidecar.Zip(files, (f, t) => new TrashedPhoto(f, [t], null))], files.Sum(f => f.Size), DateTimeOffset.UtcNow, "alberto"),
TrashJson.Default.PhotoTrashRecord));
var moves = files.Select(f => new FrameRemovalMove(f.From, fx.Paths.ToRelative(Path.Combine(entry, f.Path)))).ToList();
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(entry), [.. sidecar.Select(f => f.Id)], moves));
File.Move(folder.File(moves[0].From), fx.Paths.FromRelative(moves[0].To));
await DeleteSequenceAsync(fx, sequence.Id);
// The deletion was undone first: every photo went with its sequence, and neither its journal nor half an entry is left.
var deleted = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Kind == TrashKind.Sequence);
var trashed = fx.Paths.Sequence(".chronos/trash/" + deleted.Name);
Assert.FileMissing(trashed.RemovalJournalPath);
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(trashed.File(frame.RelativePath[(sequence.Folder.Length + 1)..])));
Assert.False(Directory.Exists(entry));
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
Assert.DoesNotContain(TrashKind.Photos, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Kind));
}
[Test]
public async Task ARestoreWhoseEntryIsBusyWaitsAndNothingGoesOnOverIt()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
var other = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[3]);
// A restore died after moving one original back, before the sidecar; the entry is being emptied meanwhile.
var journal = RestoreJournal(fx, deleted.Entry);
journal.Write(folder);
var moved = journal.Photos[0].Moves[0];
File.Move(fx.Paths.FromRelative(moved.To), folder.File(moved.From));
using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, SequenceTrash.EntryLease(deleted.Entry.Name), "other")))
{
// Another writer leaves the journal for later: it never moves a file into an entry being erased.
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
Assert.FileExists(folder.RestoreJournalPath);
Assert.FileExists(folder.File(moved.From));
Assert.FileMissing(fx.Paths.FromRelative(moved.To));
// A scan does not adopt the photo that restore moved in.
File.SetLastWriteTimeUtc(folder.File(moved.From), DateTime.UtcNow);
await fx.ScanAsync();
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
// Neither a restore of another entry into the sequence nor the sequence's deletion goes on over it.
await Assert.ThrowsAsync<LeaseUnavailableException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, other.Entry.Name, null, fx.Ct));
await Assert.ThrowsAsync<LeaseUnavailableException>(() => DeleteSequenceAsync(fx, sequence.Id));
Assert.FileExists(folder.RestoreJournalPath);
Assert.FileExists(folder.ManifestPath);
}
// Once the entry is free, the next writer settles it: the original goes back into its entry, and both entries come back.
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
Assert.FileMissing(folder.RestoreJournalPath);
Assert.Equal(frames[1].Sha256, Sha(fx.Paths.FromRelative(moved.To)));
Assert.FileMissing(folder.File(moved.From));
Assert.Equal(2, (await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct)).Photos);
Assert.Equal(1, (await fx.Service.RestoreEntryAsync(fx.Archive.Id, other.Entry.Name, null, fx.Ct)).Photos);
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
}
/// <summary>The files of a sequence as its sidecar names them, after checking that the index names the same ones and that each holds its photo.</summary>
private static async Task<List<string>> FilesAsync(ArchiveFixture fx, Guid id)
{
var sequence = Assert.NotNull(await fx.Db.Sequences.GetAsync(id));
var folder = fx.Paths.Sequence(sequence.Folder);
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
var rows = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, folder.ArchiveRelative(f.File))), rows.Select(r => (r.Id, r.RelativePath)));
foreach (var frame in sidecar.Frames) Assert.Equal(frame.Sha256, Sha(folder.File(frame.File)));
return [.. sidecar.Frames.Select(f => f.File)];
}
[Test]
public async Task PhotosDeletedBeforeAReorganizeComeBackBesideTheNamesItGave()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var numbered = new RenameTemplates(File: "{n:000}");
await fx.Service.ApplyRenameAsync(await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [sequence.Id], numbered, fx.Ct), null, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
File.WriteAllText(folder.File("originals/002.xmp"), "second");
await fx.ScanAsync();
// The record keeps the names the photo and its companion had: 002.JPG and 002.xmp.
var deleted = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[1]);
Assert.SequenceEqual(["originals/002.JPG", "originals/002.xmp"], FrameTrash.ReadEntry(fx.Paths, deleted.Entry.Name)!.Photos.Single().Files.Select(f => f.From));
// Moved and renamed since: the photos that stay take 001-003, the deleted photo's name included.
var moved = new RenameTemplates(Folder: "moved/{date}", File: "{n:000}");
await fx.Service.ApplyRenameAsync(await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [sequence.Id], moved, fx.Ct), null, fx.Ct);
Assert.SequenceEqual(["originals/001.JPG", "originals/002.JPG", "originals/003.JPG"], await FilesAsync(fx, sequence.Id));
// Restored into the sequence where it is now, under its own name made free: its companion follows that name.
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id))!;
Assert.Equal("moved/2026-09-28", row.Folder);
Assert.SequenceEqual(["originals/001.JPG", "originals/002_2.JPG", "originals/002.JPG", "originals/003.JPG"], await FilesAsync(fx, sequence.Id));
var now = fx.Paths.Sequence(row.Folder);
Assert.Equal(frames[1].Sha256, Sha(now.File("originals/002_2.JPG")));
Assert.Equal("second", File.ReadAllText(now.File("originals/002_2.xmp")));
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.DoesNotContain(TrashKind.Photos, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Kind));
}
[Test]
public async Task AReorganizeWaitsForARestoreItsEntryHoldsThenSettlesItFirst()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
// A restore died after moving the original back, before the sidecar; the entry is being emptied meanwhile.
var journal = RestoreJournal(fx, deleted.Entry);
journal.Write(folder);
var back = journal.Photos[0].Moves[0];
File.Move(fx.Paths.FromRelative(back.To), folder.File(back.From));
var templates = new RenameTemplates(Folder: "moved/{date}", File: "{n:000}");
var plan = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [sequence.Id], templates, fx.Ct);
Assert.Equal(2, plan.Steps.Single().FileChanges);
var before = await FilesAsync(fx, sequence.Id);
using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, SequenceTrash.EntryLease(deleted.Entry.Name), "other")))
{
// Neither the folder nor a file name changes under that restore.
await Assert.ThrowsAsync<LeaseUnavailableException>(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct));
Assert.Equal(sequence.Folder, (await fx.Db.Sequences.GetAsync(sequence.Id))!.Folder);
Assert.SequenceEqual(before, await FilesAsync(fx, sequence.Id));
Assert.FileExists(folder.RestoreJournalPath);
Assert.FileExists(folder.File(back.From));
}
// Once the entry is free the rename resumes: the restore is settled first (the original goes back into its entry),
// then the folder moves and the files are renamed; the photo then comes back beside them.
Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct));
Assert.Equal(frames[1].Sha256, Sha(fx.Paths.FromRelative(back.To)));
var row = (await fx.Db.Sequences.GetAsync(sequence.Id))!;
Assert.Equal("moved/2026-09-28", row.Folder);
Assert.FileMissing(fx.Paths.Sequence(row.Folder).RestoreJournalPath);
Assert.SequenceEqual(["originals/001.JPG", "originals/002.JPG"], await FilesAsync(fx, sequence.Id));
Assert.Equal(1, (await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct)).Photos);
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.Count(3, await FilesAsync(fx, sequence.Id));
}
[Test]
public async Task ARestoreIntoANewSequenceStoppedBetweenPhotosKeepsItsRecordTrue()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var first = await DeleteAsync(fx, sequence, frames[1]);
var second = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[2], frames[3]);
await DeleteSequenceAsync(fx, sequence.Id);
var made = await fx.Service.RestoreEntryAsync(fx.Archive.Id, first.Entry.Name, null, fx.Ct);
Assert.True(made.NewSequence);
var entryFolder = EntryFolder(fx, second.Entry);
var staged = Path.Combine(entryFolder, ".restoring", "originals", Path.GetFileName(frames[2].RelativePath));
using var cancel = CancellationTokenSource.CreateLinkedTokenSource(fx.Ct);
// The new sequence is held while the first photo waits to be filed into it, and the restore is cancelled meanwhile.
Task restore;
using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, $"seq-{made.Sequence.Id:N}", "other")))
{
restore = fx.Service.RestoreEntryAsync(fx.Archive.Id, second.Entry.Name, null, cancel.Token);
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(20));
while (!File.Exists(staged))
{
Assert.True(DateTime.UtcNow < until, "the first photo was never staged");
await Task.Delay(10, fx.Ct);
}
cancel.Cancel();
}
await Assert.ThrowsAsync<OperationCanceledException>(() => restore);
// The first photo is filed, the second stays, and the record lists exactly what is left.
Assert.SequenceEqual([frames[1].Sha256, frames[2].Sha256], (await fx.Db.Frames.ListBySequenceAsync(made.Sequence.Id)).Select(f => f.Sha256));
var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, second.Entry.Name));
Assert.SequenceEqual([frames[3].Id], record.Photos.Select(p => p.Frame.Id));
Assert.Equal(frames[3].Size, record.Bytes);
Assert.Equal(frames[3].Sha256, Sha(Path.Combine(entryFolder, "originals", Path.GetFileName(frames[3].RelativePath))));
Assert.False(Directory.Exists(Path.Combine(entryFolder, ".restoring")));
var rest = await fx.Service.RestoreEntryAsync(fx.Archive.Id, second.Entry.Name, null, fx.Ct);
Assert.Equal((1, made.Sequence.Id), (rest.Photos, rest.Sequence.Id));
Assert.Empty(rest.Kept);
Assert.False(Directory.Exists(entryFolder));
}
[Test]
public async Task AnIngestThatRefusesRemovedContentChecksTheTombstonesUnderTheLease()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
await DeleteAsync(fx, sequence, frames[1]);
// The same photo staged again (an upload whose completion checked the index before the deletion landed).
var (path, size, sha) = fx.Stage(Start.AddSeconds(10), 101);
Assert.Equal(frames[1].Sha256, sha);
var request = new IngestRequest(fx.Archive.Id, sequence.Id, null, null, path, frames[1].OriginalFileName!, size, sha, null, null, RefuseRemoved: true);
var refused = await Assert.ThrowsAsync<PhotoRemovedException>(() => fx.Service.Ingest.CommitAsync(request, fx.Ct));
Assert.Equal(sequence.Id, refused.SequenceId);
Assert.FileExists(path);
Assert.Count(2, ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath).Frames);
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, sha));
// Asked back, it is filed and its tombstone goes.
var back = await fx.Service.Ingest.CommitAsync(request with { RefuseRemoved = false }, fx.Ct);
Assert.False(back.Duplicate);
Assert.False(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, sha));
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
}
/// <summary>An entry of originals a conversion replaced (a later phase), its one photo replaced by <paramref name="replacement"/>.</summary>
private static string WriteReplaced(ArchiveFixture fx, Sequence sequence, string stamp, string replacement)
{
var name = $"{sequence.Id:N}-{stamp}";
var folder = Path.Combine(FrameTrash.Directory(fx.Paths), name);
Directory.CreateDirectory(Path.Combine(folder, "originals"));
var bytes = Photos.Jpeg(Start, 999);
File.WriteAllBytes(Path.Combine(folder, "originals", "IMG_0001.CR2"), bytes);
var sha = Convert.ToHexStringLower(SHA256.HashData(bytes));
var frame = new SidecarFrame { Id = Guid.CreateVersion7(), File = "originals/IMG_0001.CR2", Size = bytes.Length, Sha256 = sha };
var record = new PhotoTrashRecord(TrashKind.Replaced, sequence.Id, sequence.DisplayTitle, sequence.Folder,
[new TrashedPhoto(frame, [new TrashedFile(frame.File, "originals/IMG_0001.CR2", bytes.Length, sha)], replacement)], bytes.Length,
DateTimeOffset.UtcNow, "converter");
File.WriteAllText(folder + ".json", JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord));
return name;
}
}
public sealed class SqliteFrameTrashTests(TestContext context) : FrameTrashTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresFrameTrashTests(TestContext context) : FrameTrashTests(context)
{
protected override Backend Backend => Backend.Postgres;
}