Files
Chronos/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs
T
Alby96 eb607d47aa fix(archive): crash-safe photo restore, hidden half deletions, tombstones for every photo
The review of owner review 2 phase 3 found that the photo trash could destroy or
strand originals outside the Empty job and let deleted photos come back.

- A restore into a new sequence stages each original inside its own entry
  (<entry>/.restoring), never in a staging folder other restores share or
  maintenance sweeps; any failure moves it back, the commit runs to its end once
  the file has left, a cancel stops between photos with the record kept true,
  and the next restore of the entry puts back what a crash left staged. Nothing
  erases a staging folder any more.
- A restore into an existing sequence writes .chronos/restore.pending.json before
  its first move; SequenceWriter.ReadLocked settles it (files back into the entry
  before the sidecar was written, record rewritten after), cancellation is
  checked once before the journal, and a companion that cannot follow puts its
  photo back whole. Only an intact frame keeps a trashed copy out.
- A deletion's record keeps a .pending name until its sidecar is written, so the
  trash neither lists, restores nor empties half an entry; a failed sidecar
  write undoes the moves, and scans settle any of the three journals.
- Photos never hashed are hashed as they go and get their tombstone; a photo
  with neither hash nor original blocks the plan (FrameDeleteBlock.Unhashable).
  Sidecars rebuilt from the photos, and sequences restored without one, keep
  their tombstones; the index drops a tombstone whose content a frame holds.
- Emptying re-reads and re-measures each entry under its lease before erasing.
- Once the sidecar is written a deletion or restore stands when the index cannot
  follow (logged; the next scan resyncs it).
- XMP companions are listed once per directory, case-insensitively; a restore
  into a new sequence picks a name its companions can follow, and one that still
  cannot is kept in the record as a leftover.
- thumbs.pack is re-keyed with the frames on delete and restore, so the queued
  pass reuses the thumbnails; the server checks the fingerprint of thumbnails of
  frames never hashed too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 0bb631dd83d082e2569b98d127d827adc2e7f899)
2026-10-27 12:45:33 +01:00

1005 lines
58 KiB
C#

using System.Security.Cryptography;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Import;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Repositories;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>Deleting photos of a sequence into the archive's trash, restoring them and emptying the trash (owner review 2, R6).</summary>
[Category("archive")]
public abstract class FrameTrashTests(TestContext context) : DatabaseTest(context)
{
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
/// <summary>A sequence of <paramref name="count"/> uploaded photos (hashed, under <c>originals/</c>), ten seconds apart.</summary>
private static async Task<(Sequence Sequence, List<Frame> Frames)> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1", int seed = 100)
{
Guid? id = null;
for (var i = 0; i < count; i++)
{
var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i), seed + i);
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG",
size, sha, null, null), fx.Ct);
id = result.SequenceId;
}
return ((await fx.Db.Sequences.GetAsync(id!.Value))!, [.. await fx.Db.Frames.ListBySequenceAsync(id.Value)]);
}
private static async Task<FrameDeleteResult> DeleteAsync(ArchiveFixture fx, Sequence sequence, params Frame[] frames)
{
var plan = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [.. frames.Select(f => f.Id)], sequence.FramesVersion, fx.Ct);
Assert.True(plan.CanDelete, string.Join(", ", plan.Blocked));
return await fx.Service.FrameTrash.DeleteAsync(plan, "alberto", null, fx.Ct);
}
private static Job Running(JobKind kind, IReadOnlyList<Guid> targets) => new()
{
Id = Guid.CreateVersion7(),
Kind = kind,
TargetIds = targets,
State = JobState.Running,
NodeId = "test#0",
Created = DateTimeOffset.UtcNow,
Started = DateTimeOffset.UtcNow,
};
private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
private static string EntryFolder(ArchiveFixture fx, TrashEntry entry) => FrameTrash.Location(fx.Paths, entry.Name).Folder;
private static async Task DeleteSequenceAsync(ArchiveFixture fx, Guid id) =>
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([id], null, fx.Ct), null, null, fx.Ct);
/// <summary>A sequence a scan adopted in place from photos written into <paramref name="folder"/>: its frames are not hashed.</summary>
private static async Task<(Sequence Sequence, List<Frame> Frames)> AdoptedAsync(ArchiveFixture fx, string folder, int count, int seed = 1)
{
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, folder), Start, count, seed);
await fx.ScanAsync();
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == folder).Id;
return ((await fx.Db.Sequences.GetAsync(id))!, [.. await fx.Db.Frames.ListBySequenceAsync(id)]);
}
/// <summary>The journal a restore of <paramref name="entry"/> into its sequence writes before its first move (names free: as they were).</summary>
private static FrameRestoreJournal RestoreJournal(ArchiveFixture fx, TrashEntry entry)
{
var (folder, record) = FrameTrash.Location(fx.Paths, entry.Name);
var photos = FrameTrash.ReadEntry(fx.Paths, entry.Name)!.Photos;
return new FrameRestoreJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(folder),
[.. photos.Select(p => new RestoredPhoto(p.Frame.Id, [.. p.Files.Select(f => new FrameRemovalMove(f.From, fx.Paths.ToRelative(Path.Combine(folder, f.Path))))]))]);
}
/// <summary>Makes every <paramref name="operation"/> on <paramref name="table"/> fail, as a lost database connection would.</summary>
private async Task FailAsync(ArchiveFixture fx, string table, string operation)
{
await using var session = await fx.Db.Database.OpenAsync(fx.Ct);
await session.ExecuteScriptAsync(ForDialect(
$"CREATE TRIGGER fail_{table} BEFORE {operation} ON {table} BEGIN SELECT RAISE(ABORT, 'injected'); END;",
$"CREATE FUNCTION fail_{table}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected'; END $$; " +
$"CREATE TRIGGER fail_{table} BEFORE {operation} ON {table} FOR EACH ROW EXECUTE FUNCTION fail_{table}();"), fx.Ct);
}
private async Task HealAsync(ArchiveFixture fx, string table)
{
await using var session = await fx.Db.Database.OpenAsync(fx.Ct);
await session.ExecuteScriptAsync(ForDialect($"DROP TRIGGER fail_{table};", $"DROP TRIGGER fail_{table} ON {table};"), fx.Ct);
}
[Test]
public async Task TheDryRunCountsThePhotosAndChangesNothing()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var plan = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [frames[1].Id, frames[3].Id], sequence.FramesVersion, fx.Ct);
Assert.True(plan.CanDelete);
Assert.SequenceEqual([frames[1].Id, frames[3].Id], plan.Frames.Select(f => f.Id));
Assert.Equal(frames[1].Size + frames[3].Size, plan.Bytes);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.False(Directory.Exists(FrameTrash.Directory(fx.Paths)), "a dry run moves nothing");
foreach (var frame in frames) Assert.FileExists(fx.Paths.FromRelative(frame.RelativePath));
}
[Test]
public async Task APlanSaysWhyPhotosCannotGo()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var trash = fx.Service.FrameTrash;
Assert.SequenceEqual([FrameDeleteBlock.WouldEmpty], (await trash.PlanDeleteAsync(sequence.Id, [.. frames.Select(f => f.Id)], null, fx.Ct)).Blocked);
Assert.SequenceEqual([FrameDeleteBlock.FramesChanged], (await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id], sequence.FramesVersion - 1, fx.Ct)).Blocked);
Assert.SequenceEqual([FrameDeleteBlock.FramesChanged], (await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id, Guid.CreateVersion7()], null, fx.Ct)).Blocked);
await Assert.ThrowsAsync<ArgumentException>(() => trash.PlanDeleteAsync(Guid.CreateVersion7(), [frames[0].Id], null, fx.Ct));
// A job on the sequence, or on its whole archive, holds it.
await fx.Db.Jobs.EnqueueAsync(Running(JobKind.Render, [sequence.Id]));
var busy = await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id], null, fx.Ct);
Assert.SequenceEqual([FrameDeleteBlock.Busy], busy.Blocked);
Assert.False(busy.CanDelete);
await Assert.ThrowsAsync<StaleDeletePlanException>(() => trash.DeleteAsync(busy, null, null, fx.Ct));
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task DeletingMovesThePhotosToTheTrashAndTombstonesThem()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
var folder = fx.Paths.Sequence(sequence.Folder);
// Proxies are named by frame index: they follow their frames.
Directory.CreateDirectory(folder.Proxies);
for (var i = 0; i < frames.Count; i++) File.WriteAllBytes(folder.Proxy(i), [(byte)i]);
await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt, SequenceStatus.None);
var result = await DeleteAsync(fx, sequence, frames[1], frames[3]);
foreach (var gone in new[] { frames[1], frames[3] }) Assert.FileMissing(fx.Paths.FromRelative(gone.RelativePath));
var entry = result.Entry;
Assert.Equal(TrashKind.Photos, entry.Kind);
Assert.True(FrameTrash.IsPhotoEntry(entry.Name));
Assert.Equal(2, entry.Frames);
Assert.Equal(frames[1].Size + frames[3].Size, entry.Bytes);
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), entry.Name[FrameTrash.EntryPrefix.Length..]);
Assert.Equal(frames[1].Sha256, Sha(Path.Combine(entryFolder, "originals", Path.GetFileName(frames[1].RelativePath))));
Assert.FileMissing(folder.RemovalJournalPath);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
Assert.SequenceEqual([frames[0].Id, frames[2].Id, frames[4].Id], manifest.Frames.Select(f => f.Id));
Assert.SequenceEqual([0, 1, 2], manifest.Frames.Select(f => f.I));
Assert.Equal(sequence.FramesVersion + 1, manifest.Versions.Frames);
Assert.SequenceEqual([frames[1].Sha256, frames[3].Sha256], Assert.NotNull(manifest.Removed).Select(r => r.Sha256));
Assert.True(manifest.Removed.All(r => r.Reason == RemovalReason.Deleted));
var row = result.Sequence;
Assert.Equal(3, row.FrameCount);
Assert.Equal(manifest.Versions.Frames, row.FramesVersion);
Assert.Equal(SequenceStatus.None, row.Status & (SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt));
Assert.SequenceEqual([frames[0].Id, frames[2].Id, frames[4].Id], (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.SequenceEqual([frames[1].Sha256, frames[3].Sha256], (await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id)).Select(r => r.Sha256).Order());
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[3].Sha256!.ToUpperInvariant()));
Assert.SequenceEqual<byte>([0, 2, 4], Enumerable.Range(0, 3).Select(i => File.ReadAllBytes(folder.Proxy(i))[0]));
Assert.FileMissing(folder.Proxy(3));
Assert.FileMissing(folder.Proxy(4));
var listed = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == entry.Name);
Assert.Equal(sequence.DisplayTitle, listed.Title);
Assert.Equal("alberto", listed.DeletedBy);
}
[Test]
public async Task PositionsOnDeletedPhotosWaitAndComeBackWithThem()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
var folder = fx.Paths.Sequence(sequence.Folder);
var project = new ProjectSettings { ExcludedFrames = [1, 4], HeroFrame = 3 }.ToJson();
var version = (await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = project, HeroFrameId = frames[3].Id }, null, fx.Ct))
.Sequence!.SettingsVersion;
var deleted = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[1], frames[3]);
var row = deleted.Sequence;
var parked = ProjectSettings.FromJson(row.SettingsJson!);
Assert.SequenceEqual([2], parked.ExcludedFrames, "the exclusion of the fifth photo follows it");
Assert.Null(parked.HeroFrame);
Assert.Count(2, Assert.NotNull(parked.PendingFrames));
Assert.True(parked.HeroPending, "the hero waits for its photo");
Assert.Null(row.HeroFrameId);
Assert.Equal(version, row.SettingsVersion);
Assert.True(File.Exists(folder.ProjectPath), "project.json mirrors the index");
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(TrashKind.Photos, restored.Kind);
Assert.Equal(2, restored.Photos);
Assert.Empty(restored.Kept);
var back = (await fx.Db.Sequences.GetAsync(sequence.Id))!;
var placed = ProjectSettings.FromJson(back.SettingsJson!);
Assert.SequenceEqual([1, 4], placed.ExcludedFrames);
Assert.Equal(3, placed.HeroFrame);
Assert.Null(placed.PendingFrames);
Assert.Equal(frames[3].Id, back.HeroFrameId);
Assert.Equal(version, back.SettingsVersion);
}
[Test]
public async Task ARestorePutsThePhotosBackWithTheirIdsAndDropsTheirTombstones()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var folder = fx.Paths.Sequence(sequence.Folder);
Directory.CreateDirectory(folder.Proxies);
for (var i = 0; i < frames.Count; i++) File.WriteAllBytes(folder.Proxy(i), [(byte)i]);
var deleted = await DeleteAsync(fx, sequence, frames[0], frames[2]);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.False(restored.NewSequence);
Assert.Equal(sequence.Id, restored.Sequence.Id);
var now = await fx.Db.Frames.ListBySequenceAsync(sequence.Id);
Assert.SequenceEqual(frames.Select(f => f.Id), now.Select(f => f.Id));
Assert.SequenceEqual(frames.Select(f => f.RelativePath), now.Select(f => f.RelativePath));
Assert.SequenceEqual(frames.Select(f => f.Sha256), now.Select(f => f.Sha256));
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.Null(ManifestStore.ReadSequence(folder.ManifestPath).Removed);
Assert.Empty(await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id));
Assert.Equal(sequence.FramesVersion + 2, restored.Sequence.FramesVersion);
Assert.DoesNotContain(deleted.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
Assert.False(Directory.Exists(Path.Combine(FrameTrash.Directory(fx.Paths), deleted.Entry.Name[FrameTrash.EntryPrefix.Length..])));
// The proxies of the frames that stayed are back at their index; those of the photos that went are made again.
Assert.SequenceEqual<byte>([1], File.ReadAllBytes(folder.Proxy(1)));
Assert.SequenceEqual<byte>([3], File.ReadAllBytes(folder.Proxy(3)));
Assert.FileMissing(folder.Proxy(0));
}
[Test]
public async Task ARestoreKeepsAPhotoWhoseFileNoLongerMatchesItsRecord()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), deleted.Entry.Name[FrameTrash.EntryPrefix.Length..]);
var damaged = Path.Combine(entryFolder, "originals", Path.GetFileName(frames[2].RelativePath));
var bytes = File.ReadAllBytes(damaged);
bytes[^10] ^= 0x01;
File.WriteAllBytes(damaged, bytes);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
Assert.SequenceEqual([frames[2].OriginalFileName!], restored.Kept);
Assert.SequenceEqual([frames[0].Id, frames[1].Id, frames[3].Id], (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.SequenceEqual([frames[2].Sha256], (await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id)).Select(r => r.Sha256));
var left = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal(1, left.Frames);
Assert.FileExists(damaged);
// Nothing left that matches: the entry stays as it is.
await Assert.ThrowsAsync<InvalidDataException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct));
Assert.FileExists(damaged);
}
[Test]
public async Task PhotosOfASequenceThatIsGoneComeBackAsANewOne()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
var whole = await fx.Service.PlanDeleteAsync([sequence.Id], null, fx.Ct);
await fx.Service.DeleteAsync(whole, null, null, fx.Ct);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.True(restored.NewSequence);
Assert.NotEqual(sequence.Id, restored.Sequence.Id);
Assert.Equal(sequence.DisplayTitle, restored.Sequence.DisplayTitle);
var frame = (await fx.Db.Frames.ListBySequenceAsync(restored.Sequence.Id)).Single();
Assert.Equal(frames[1].Sha256, frame.Sha256);
Assert.Equal(frames[1].Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
// The whole sequence is still in the trash, restorable on its own.
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Kind == TrashKind.Sequence && e.SequenceId == sequence.Id);
}
[Test]
public async Task TombstonesSurviveARebuildFromTheSidecars()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
await DeleteAsync(fx, sequence, frames[0]);
await fx.ReopenWithEmptyIndexAsync();
await fx.ScanAsync(ScanMode.Rebuild);
var removals = await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id);
Assert.SequenceEqual([frames[0].Sha256], removals.Select(r => r.Sha256));
Assert.Equal(RemovalReason.Deleted, removals[0].Reason);
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task TombstonesAreReplacedWholeAndGoWithTheirSequence()
{
var fx = await FixtureAsync();
var (sequence, _) = await UploadedAsync(fx, 1);
var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero);
var many = Enumerable.Range(0, 450).Select(i => new FrameRemoval(sequence.Id, i.ToString("x64", System.Globalization.CultureInfo.InvariantCulture),
i % 2 == 0 ? null : new string('A', 64), i % 2 == 0 ? RemovalReason.Deleted : RemovalReason.Replaced, at)).ToList();
Assert.Equal(450, await fx.Db.FrameRemovals.SyncSequenceAsync(sequence.Id, many, fx.Ct));
var stored = await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id, fx.Ct);
Assert.Count(450, stored);
Assert.Equal((new string('a', 64), RemovalReason.Replaced), (stored.Single(r => r.Sha256 == many[1].Sha256).ReplacementSha256, stored[1].Reason));
Assert.Equal(1, await fx.Db.FrameRemovals.SyncSequenceAsync(sequence.Id, [many[3], many[3]], fx.Ct));
Assert.Count(1, await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id, fx.Ct));
await fx.Db.Sequences.DeleteAsync(sequence.Id, fx.Ct);
Assert.Empty(await fx.Db.FrameRemovals.ListBySequenceAsync(sequence.Id, fx.Ct));
}
[Test]
public async Task ADeletionACrashInterruptedIsUndoneByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
// The journal was written and one of two files moved when the process died; the sidecar still lists both frames.
var entry = Path.Combine(FrameTrash.Directory(fx.Paths), $"{sequence.Id:N}-20261027120000");
var record = entry + ".json";
Directory.CreateDirectory(Path.Combine(entry, "originals"));
File.WriteAllText(record, "{}");
var moves = frames.Take(2).Select(f => new FrameRemovalMove(f.RelativePath[(sequence.Folder.Length + 1)..],
fx.Paths.ToRelative(Path.Combine(entry, "originals", Path.GetFileName(f.RelativePath))))).ToList();
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(entry), [frames[0].Id, frames[1].Id], moves));
File.Move(fx.Paths.FromRelative(frames[0].RelativePath), fx.Paths.FromRelative(moves[0].To));
// Any writer of the sequence reads its sidecar under the lease first.
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.FileMissing(folder.RemovalJournalPath);
Assert.FileMissing(record);
Assert.False(Directory.Exists(entry));
Assert.Count(3, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
}
[Test]
public async Task ErasingTheCardKeepsTheCardCopyOfADeletedPhoto()
{
var fx = await FixtureAsync();
var card = Context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 4);
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592), fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null), Guid.NewGuid(), 0,
_ => ValueTask.CompletedTask, null, fx.Ct);
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions.Single().SessionKey))!.Value;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(id))!, frames[2]);
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
CardEraser.Execute(erase);
// The deleted photo has no frame any more: nothing in the archive vouches for its card copy, which stays.
Assert.Equal(3, erase.ToDelete.Count());
var left = Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetFileName(f)).ToList();
Assert.SequenceEqual([frames[2].OriginalFileName!], left);
}
[Test]
public async Task EmptyingErasesExactlyThePlannedEntries()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var (other, _) = await UploadedAsync(fx, 2, "upload-2", 200);
var photos = await DeleteAsync(fx, sequence, frames[1]);
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([other.Id], null, fx.Ct), null, null, fx.Ct);
var engine = await fx.StartEngineAsync();
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
Assert.Count(2, plan.Entries);
Assert.Equal(3, plan.Photos);
Assert.True(plan.Bytes > 0);
Assert.Equal(plan.Fingerprint, (await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct)).Fingerprint, "the plan reads only");
Assert.SequenceEqual(["nope"], (await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [photos.Entry.Name, "nope"], fx.Ct)).Unknown);
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
Assert.Empty(Directory.EnumerateFileSystemEntries(FrameTrash.Directory(fx.Paths)));
Assert.Empty(Directory.EnumerateFiles(SequenceTrash.TrashDirectory(fx.Paths), "*", SearchOption.AllDirectories));
Assert.Contains(Chronos.Core.Localization.Loc.L("Archive.Log.TrashEmptied", 2, Housekeeping.CleanupSteps.Megabytes(plan.Bytes)), done.LogTail);
// Tombstones outlive the trash: the deleted photo is still not offered back.
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[1].Sha256!));
}
[Test]
public async Task AnEmptyingPlannedBeforeTheTrashChangedErasesNothing()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var first = await DeleteAsync(fx, sequence, frames[1]);
var second = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[2]);
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var engine = await fx.StartEngineAsync();
// A planned entry came back (restored) before the job ran.
await fx.Service.RestoreEntryAsync(fx.Archive.Id, first.Entry.Name, null, fx.Ct);
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Archive.Error.TrashChanged"), failed.Error);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == second.Entry.Name);
// A planned entry whose folder changed since makes the plan stale too.
var again = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var folder = Path.Combine(FrameTrash.Directory(fx.Paths), second.Entry.Name[FrameTrash.EntryPrefix.Length..]);
File.WriteAllBytes(Path.Combine(folder, "added.bin"), [1, 2, 3]);
Assert.NotEqual(again.Fingerprint, (await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct)).Fingerprint);
var stale = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, again), cancellationToken: fx.Ct);
await fx.WaitForAsync(stale.Id, JobState.Failed);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == second.Entry.Name);
Assert.FileExists(Path.Combine(folder, "added.bin"));
}
[Test]
public async Task AReplacedEntryIsErasedOnlyWhileItsReplacementMatches()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 2);
var replacement = frames[1];
var good = WriteReplaced(fx, sequence, "20261027100000", replacement.Sha256!);
var bad = WriteReplaced(fx, sequence, "20261027110000", new string('a', 64));
var engine = await fx.StartEngineAsync();
var listed = await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct);
Assert.True(listed.All(e => e.Kind == TrashKind.Replaced));
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.Plural("Archive.Error.TrashKept", 1), failed.Error);
var left = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == FrameTrash.EntryPrefix + bad);
Assert.Equal(TrashKind.Replaced, left.Kind);
Assert.DoesNotContain(FrameTrash.EntryPrefix + good, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
// A replacement whose file no longer hashes to it keeps its originals too.
var other = WriteReplaced(fx, sequence, "20261027120000", replacement.Sha256!);
var file = fx.Paths.FromRelative(replacement.RelativePath);
var bytes = File.ReadAllBytes(file);
bytes[^10] ^= 0x01;
File.WriteAllBytes(file, bytes);
var again = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [FrameTrash.EntryPrefix + other], fx.Ct);
var second = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, again), cancellationToken: fx.Ct);
await fx.WaitForAsync(second.Id, JobState.Failed);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == FrameTrash.EntryPrefix + other);
}
[Test]
public async Task CompanionsGoWithTheirPhotoAndFollowItsNameBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var second = fx.Paths.FromRelative(frames[1].RelativePath);
var third = fx.Paths.FromRelative(frames[2].RelativePath);
// An upper-case extension (listed on Linux too), and one named after the whole file name.
var upper = Path.ChangeExtension(second, ".XMP");
var whole = third + ".xmp";
File.WriteAllText(upper, "<rating>2</rating>");
File.WriteAllText(whole, "<rating>3</rating>");
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
Assert.FileMissing(upper);
Assert.FileMissing(whole);
Assert.Equal((2, 4), (deleted.Entry.Frames, deleted.Entry.Files));
var entryFolder = EntryFolder(fx, deleted.Entry);
Assert.Equal("<rating>2</rating>", File.ReadAllText(Path.Combine(entryFolder, "originals", Path.GetFileName(upper))));
Assert.Equal("<rating>3</rating>", File.ReadAllText(Path.Combine(entryFolder, "originals", Path.GetFileName(whole))));
// A file took the second photo's name since: it comes back as _2, and its XMP follows that name.
File.WriteAllBytes(second, Photos.Jpeg(Start, 999));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(2, restored.Photos);
Assert.Empty(restored.Kept);
var now = await fx.Db.Frames.ListBySequenceAsync(sequence.Id);
Assert.SequenceEqual(frames.Select(f => f.Id), now.Select(f => f.Id));
var renamed = fx.Paths.FromRelative(now[1].RelativePath);
Assert.Equal($"{Path.GetFileNameWithoutExtension(second)}_2{Path.GetExtension(second)}", Path.GetFileName(renamed));
Assert.Equal(frames[1].Sha256, Sha(renamed));
Assert.Equal("<rating>2</rating>", File.ReadAllText(Path.ChangeExtension(renamed, ".XMP")));
Assert.Equal(frames[2].RelativePath, now[2].RelativePath);
Assert.Equal("<rating>3</rating>", File.ReadAllText(whole));
Assert.False(Directory.Exists(entryFolder));
}
[Test]
public async Task APhotoRestoredIntoANewSequenceTakesANameItsCompanionCanFollow()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var name = fx.Paths.FromRelative(frames[1].RelativePath);
File.WriteAllText(Path.ChangeExtension(name, ".xmp"), "first");
var first = await DeleteAsync(fx, sequence, frames[1]);
// Another photo filed under the same name since, with an XMP of its own, deleted in turn.
var (path, size, sha) = fx.Stage(Start.AddSeconds(10), 300);
var again = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, path, frames[1].OriginalFileName!, size, sha, null, null), fx.Ct);
Assert.Equal(frames[1].RelativePath, again.RelativePath);
File.WriteAllText(Path.ChangeExtension(name, ".xmp"), "second");
var row = (await fx.Db.Sequences.GetAsync(sequence.Id))!;
var second = await DeleteAsync(fx, row, (await fx.Db.Frames.GetAsync(again.FrameId))!);
await DeleteSequenceAsync(fx, sequence.Id);
var one = await fx.Service.RestoreEntryAsync(fx.Archive.Id, first.Entry.Name, null, fx.Ct);
var two = await fx.Service.RestoreEntryAsync(fx.Archive.Id, second.Entry.Name, null, fx.Ct);
Assert.Equal(one.Sequence.Id, two.Sequence.Id);
var back = await fx.Db.Frames.ListBySequenceAsync(one.Sequence.Id);
Assert.Count(2, back);
foreach (var (frame, xmp) in back.OrderBy(f => f.RelativePath, StringComparer.Ordinal).Zip(["first", "second"]))
{
var file = fx.Paths.FromRelative(frame.RelativePath);
Assert.Equal(frame.Sha256, Sha(file));
Assert.Equal(xmp, File.ReadAllText(Path.ChangeExtension(file, ".xmp")));
}
Assert.DoesNotContain(TrashKind.Photos, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Kind));
}
[Test]
public async Task TwoRestoresOfOneDeletedSequenceAtOnceLoseNothing()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var first = await DeleteAsync(fx, sequence, frames[1]);
var second = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[2]);
await DeleteSequenceAsync(fx, sequence.Id);
var restores = await Task.WhenAll(
Task.Run(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, first.Entry.Name, null, fx.Ct)),
Task.Run(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, second.Entry.Name, null, fx.Ct)));
Assert.Equal(restores[0].Sequence.Id, restores[1].Sequence.Id);
var back = await fx.Db.Frames.ListBySequenceAsync(restores[0].Sequence.Id);
Assert.SequenceEqual([frames[1].Sha256, frames[2].Sha256], back.Select(f => f.Sha256));
foreach (var frame in back) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.False(Directory.Exists(EntryFolder(fx, first.Entry)));
Assert.False(Directory.Exists(EntryFolder(fx, second.Entry)));
}
[Test]
public async Task AnOriginalAStoppedRestoreLeftStagedSurvivesMaintenanceAndComesBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
await DeleteSequenceAsync(fx, sequence.Id);
var entryFolder = EntryFolder(fx, deleted.Entry);
// A restore into a new sequence stopped (the service, or a crash) with an original staged in its entry, days ago.
var name = Path.GetFileName(frames[2].RelativePath);
var staged = Path.Combine(entryFolder, ".restoring", "originals", name);
Directory.CreateDirectory(Path.GetDirectoryName(staged)!);
File.Move(Path.Combine(entryFolder, "originals", name), staged);
File.SetLastWriteTimeUtc(staged, DateTime.UtcNow.AddDays(-3));
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(frames[2].Sha256, Sha(staged));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(2, restored.Photos);
Assert.Empty(restored.Kept);
var back = await fx.Db.Frames.ListBySequenceAsync(restored.Sequence.Id);
Assert.SequenceEqual([frames[1].Sha256, frames[2].Sha256], back.Select(f => f.Sha256));
foreach (var frame in back) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.False(Directory.Exists(entryFolder));
Assert.False(Directory.Exists(fx.Paths.StagingDirectory) && Directory.EnumerateDirectories(fx.Paths.StagingDirectory, "trash-*").Any());
}
[Test]
public async Task ARestoreIntoANewSequenceThatFailsPutsTheOriginalBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
await DeleteSequenceAsync(fx, sequence.Id);
var entryFolder = EntryFolder(fx, deleted.Entry);
var original = Path.Combine(entryFolder, "originals", Path.GetFileName(frames[1].RelativePath));
await FailAsync(fx, "sequences", "INSERT");
// The ingest fails after the original left the entry, with no file error: it goes back all the same.
await Assert.ThrowsAsync<DatabaseException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct));
Assert.Equal(frames[1].Sha256, Sha(original));
Assert.False(Directory.Exists(Path.Combine(entryFolder, ".restoring")));
var listed = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal((1, deleted.Entry.Bytes), (listed.Frames, listed.Bytes));
await HealAsync(fx, "sequences");
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
Assert.Equal(frames[1].Sha256, (await fx.Db.Frames.ListBySequenceAsync(restored.Sequence.Id)).Single().Sha256);
Assert.False(Directory.Exists(entryFolder));
}
[Test]
public async Task ARestoreACrashInterruptedBeforeTheSidecarIsUndoneByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
// The journal was written and one of two originals moved back when the process died; the sidecar lists neither.
var journal = RestoreJournal(fx, deleted.Entry);
journal.Write(folder);
var moved = journal.Photos[0].Moves[0];
File.Move(fx.Paths.FromRelative(moved.To), folder.File(moved.From));
// Any writer of the sequence reads its sidecar under the lease first.
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
Assert.FileMissing(folder.RestoreJournalPath);
Assert.FileMissing(folder.File(moved.From));
Assert.Equal(frames[1].Sha256, Sha(fx.Paths.FromRelative(moved.To)));
Assert.Count(2, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
Assert.Equal(2, Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name).Frames);
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(2, restored.Photos);
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
}
[Test]
public async Task ARestoreACrashInterruptedAfterTheSidecarIsCompletedByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 4);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1], frames[2]);
var entryFolder = EntryFolder(fx, deleted.Entry);
// Every file moved back and the sidecar was written when the process died; the trash record still lists the photos.
var journal = RestoreJournal(fx, deleted.Entry);
journal.Write(folder);
foreach (var move in journal.Photos.SelectMany(p => p.Moves)) File.Move(fx.Paths.FromRelative(move.To), folder.File(move.From));
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
var back = SequenceFrames.Insert(manifest, [.. FrameTrash.ReadEntry(fx.Paths, deleted.Entry.Name)!.Photos.Select(p => p.Frame)]);
ManifestStore.WriteSequence(folder.ManifestPath, back with { Removed = null });
// The journal makes a scan read the folder under the lease, which settles the restore and indexes its frames.
await fx.ScanAsync();
Assert.FileMissing(folder.RestoreJournalPath);
Assert.DoesNotContain(deleted.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
Assert.False(Directory.Exists(entryFolder));
Assert.FileMissing(FrameTrash.Location(fx.Paths, deleted.Entry.Name).Record);
Assert.SequenceEqual(frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
}
[Test]
public async Task ARestoreCancelledBeforeItsFirstMoveLeavesEverythingAsItWas()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
using var cancel = CancellationTokenSource.CreateLinkedTokenSource(fx.Ct);
// Another process holds the sequence: the restore waits for it, and is cancelled meanwhile.
using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, $"seq-{sequence.Id:N}", "other")))
{
var restore = fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, cancel.Token);
cancel.CancelAfter(TimeSpan.FromMilliseconds(200));
await Assert.ThrowsAsync<OperationCanceledException>(() => restore);
}
Assert.FileMissing(folder.RestoreJournalPath);
Assert.FileMissing(fx.Paths.FromRelative(frames[1].RelativePath));
Assert.Equal(frames[1].Sha256, Sha(Path.Combine(EntryFolder(fx, deleted.Entry), "originals", Path.GetFileName(frames[1].RelativePath))));
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal(1, (await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct)).Photos);
}
[Test]
public async Task APhotoWhoseCompanionCannotFollowStaysWholeInTheTrash()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var photo = fx.Paths.FromRelative(frames[1].RelativePath);
File.WriteAllText(Path.ChangeExtension(photo, ".xmp"), "edits");
var deleted = await DeleteAsync(fx, sequence, frames[1]);
// Something that is no file holds the companion's place: the original moves back in, the XMP cannot follow it.
Directory.CreateDirectory(Path.ChangeExtension(photo, ".xmp"));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(0, restored.Photos);
Assert.SequenceEqual([frames[1].OriginalFileName!], restored.Kept);
Assert.FileMissing(photo);
Assert.FileMissing(folder.RestoreJournalPath);
var entryFolder = EntryFolder(fx, deleted.Entry);
Assert.Equal(frames[1].Sha256, Sha(Path.Combine(entryFolder, "originals", Path.GetFileName(photo))));
Assert.Equal("edits", File.ReadAllText(Path.Combine(entryFolder, "originals", Path.GetFileName(Path.ChangeExtension(photo, ".xmp")))));
Assert.Count(2, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[1].Sha256!));
var listed = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
Assert.Equal((1, 2), (listed.Frames, listed.Files));
}
[Test]
public async Task AnInterruptedDeletionIsNeitherListedNorEmptiedAndAScanUndoesIt()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
// As a deletion leaves it when the process dies after its first move: journal, record under its pending name, one file moved.
var name = $"{sequence.Id:N}-20261027120000";
var entry = Path.Combine(FrameTrash.Directory(fx.Paths), name);
var record = entry + ".json";
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath).Frames.Take(2).ToList();
var files = sidecar.Select(f => new TrashedFile(f.File, "originals/" + Path.GetFileName(f.File), f.Size, f.Sha256)).ToList();
Directory.CreateDirectory(Path.Combine(entry, "originals"));
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), JsonSerializer.Serialize(new PhotoTrashRecord(TrashKind.Photos, sequence.Id, sequence.DisplayTitle,
sequence.Folder, [.. sidecar.Zip(files, (f, t) => new TrashedPhoto(f, [t], null))], files.Sum(f => f.Size), DateTimeOffset.UtcNow, "alberto"),
TrashJson.Default.PhotoTrashRecord));
var moves = files.Select(f => new FrameRemovalMove(f.From, fx.Paths.ToRelative(Path.Combine(entry, f.Path)))).ToList();
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(entry), [.. sidecar.Select(f => f.Id)], moves));
File.Move(folder.File(moves[0].From), fx.Paths.FromRelative(moves[0].To));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
Assert.Empty((await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct)).Entries);
await Assert.ThrowsAsync<DirectoryNotFoundException>(() => fx.Service.RestoreEntryAsync(fx.Archive.Id, FrameTrash.EntryPrefix + name, null, fx.Ct));
// The journal makes a scan read the folder under the lease, which undoes the deletion.
await fx.ScanAsync();
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.FileMissing(folder.RemovalJournalPath);
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
Assert.False(Directory.Exists(entry));
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task ADeletionWhoseSidecarWasWrittenIsListedByTheNextWriter()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var deleted = await DeleteAsync(fx, sequence, frames[1]);
var (entry, record) = FrameTrash.Location(fx.Paths, deleted.Entry.Name);
// As a deletion leaves it when the process dies after its sidecar was written: the record not yet under its own name.
File.Move(record, FrameRemovalJournal.PendingRecord(record));
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(fx.Paths.ToRelative(record), fx.Paths.ToRelative(entry), [frames[1].Id], []));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
await fx.Service.SetSequenceInfoAsync(sequence.Id, "Harbour", [], 0, null, fx.Ct);
Assert.FileMissing(folder.RemovalJournalPath);
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == deleted.Entry.Name);
}
[Test]
[WindowsOnly]
public async Task ADeletionWhoseSidecarCannotBeWrittenPutsEveryPhotoBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var folder = fx.Paths.Sequence(sequence.Folder);
var plan = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [frames[0].Id, frames[2].Id], sequence.FramesVersion, fx.Ct);
// Another program holds the sidecar open: it cannot be replaced.
Exception error;
using (new FileStream(folder.ManifestPath, FileMode.Open, FileAccess.Read, FileShare.Read))
error = await Assert.ThrowsAsync<Exception>(() => fx.Service.FrameTrash.DeleteAsync(plan, "alberto", null, fx.Ct));
Assert.True(error is IOException or UnauthorizedAccessException, error.ToString());
foreach (var frame in frames) Assert.Equal(frame.Sha256, Sha(fx.Paths.FromRelative(frame.RelativePath)));
Assert.FileMissing(folder.RemovalJournalPath);
Assert.Empty(Directory.EnumerateFileSystemEntries(FrameTrash.Directory(fx.Paths)));
Assert.Count(3, ManifestStore.ReadSequence(folder.ManifestPath).Frames);
}
[Test]
public async Task APhotoNeverHashedIsHashedAsItGoesAndLeavesItsTombstone()
{
var fx = await FixtureAsync();
var (sequence, frames) = await AdoptedAsync(fx, "Shoot", 4);
Assert.True(frames.All(f => f.Sha256 is null), "adopted, not verified");
var sha = Sha(fx.Paths.FromRelative(frames[1].RelativePath));
var deleted = await DeleteAsync(fx, sequence, frames[1]);
var folder = fx.Paths.Sequence(sequence.Folder);
Assert.SequenceEqual([sha], Assert.NotNull(ManifestStore.ReadSequence(folder.ManifestPath).Removed).Select(r => r.Sha256));
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, sha));
Assert.Equal(sha, Sha(Path.Combine(EntryFolder(fx, deleted.Entry), "originals", Path.GetFileName(frames[1].RelativePath))));
// One whose original is gone as well could leave none: the plan refuses it, and says which.
File.Delete(fx.Paths.FromRelative(frames[3].RelativePath));
var row = (await fx.Db.Sequences.GetAsync(sequence.Id))!;
var refused = await fx.Service.FrameTrash.PlanDeleteAsync(sequence.Id, [frames[2].Id, frames[3].Id], row.FramesVersion, fx.Ct);
Assert.SequenceEqual([FrameDeleteBlock.Unhashable], refused.Blocked);
Assert.SequenceEqual([frames[3].Id], refused.Unhashable.Select(f => f.Id));
await Assert.ThrowsAsync<StaleDeletePlanException>(() => fx.Service.FrameTrash.DeleteAsync(refused, null, null, fx.Ct));
}
[Test]
public async Task ADuplicateThatStaysNeverHashedKeepsItsContentOutOfTheTombstones()
{
var fx = await FixtureAsync();
var shoot = Path.Combine(fx.Archive.RootPath, "Twins");
var files = Photos.WriteSeries(shoot, Start, 3);
File.Copy(files[1], Path.Combine(shoot, "IMG_0009.JPG"));
await fx.ScanAsync();
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Twins").Id;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
var twin = frames.Single(f => f.RelativePath.EndsWith("IMG_0002.JPG", StringComparison.Ordinal));
await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(id))!, twin);
Assert.Null(ManifestStore.ReadSequence(fx.Paths.Sequence("Twins").ManifestPath).Removed);
Assert.False(await fx.Db.FrameRemovals.IsRemovedAsync(id, Sha(Path.Combine(shoot, "IMG_0009.JPG"))));
}
[Test]
public async Task ARestoreBringsBackAPhotoWhoseOnlyTwinInTheSequenceIsMissing()
{
var fx = await FixtureAsync();
var shoot = Path.Combine(fx.Archive.RootPath, "Twins");
var files = Photos.WriteSeries(shoot, Start, 3);
File.Copy(files[1], Path.Combine(shoot, "IMG_0009.JPG"));
await fx.ScanAsync();
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Twins").Id;
await fx.Service.Verifier.VerifyAsync(fx.Archive, [id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
var twins = (await fx.Db.Frames.ListBySequenceAsync(id)).Where(f => f.Sha256 == Sha(files[1])).ToList();
Assert.Count(2, twins);
var deleted = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(id))!, twins[1]);
Assert.False(await fx.Db.FrameRemovals.IsRemovedAsync(id, twins[1].Sha256!), "its twin holds the content");
// The twin that stayed loses its file since; a scan flags it missing.
File.Delete(fx.Paths.FromRelative(twins[0].RelativePath));
await fx.ScanAsync();
Assert.True((await fx.Db.Frames.GetAsync(twins[0].Id))!.Flags.HasFlag(FrameFlags.Missing));
var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
Assert.Equal(1, restored.Photos);
Assert.Empty(restored.Kept);
var back = Assert.NotNull(await fx.Db.Frames.GetAsync(twins[1].Id));
Assert.Equal(twins[1].Sha256, Sha(fx.Paths.FromRelative(back.RelativePath)));
Assert.False(back.Flags.HasFlag(FrameFlags.Missing));
Assert.False(Directory.Exists(EntryFolder(fx, deleted.Entry)));
}
[Test]
public async Task ASidecarRebuiltFromThePhotosKeepsTheTombstones()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
await DeleteAsync(fx, sequence, frames[0]);
var folder = fx.Paths.Sequence(sequence.Folder);
File.WriteAllText(folder.ManifestPath, "{ \"damaged\": ");
await fx.ScanAsync();
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[0].Sha256!));
Assert.SequenceEqual([frames[0].Sha256], Assert.NotNull(ManifestStore.ReadSequence(folder.ManifestPath).Removed).Select(r => r.Sha256));
// A sequence restored from the trash without its sidecar takes them back from its record.
await DeleteSequenceAsync(fx, sequence.Id);
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(e => e.Kind == TrashKind.Sequence);
File.Delete(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), entry.Name, ".chronos", "sequence.json"));
await fx.Service.RestoreAsync(fx.Archive.Id, entry.Name, fx.Ct);
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[0].Sha256!));
}
[Test]
public async Task ADeletionStandsWhenTheIndexCannotFollowAndTheNextScanResyncsIt()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
await FailAsync(fx, "frame_removals", "INSERT");
var result = await DeleteAsync(fx, sequence, frames[1]);
Assert.FileMissing(fx.Paths.FromRelative(frames[1].RelativePath));
Assert.Equal((sequence.FramesVersion + 1, 2), (result.Sequence.FramesVersion, result.Sequence.FrameCount));
Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == result.Entry.Name);
Assert.Equal(sequence.FramesVersion, (await fx.Db.Sequences.GetAsync(sequence.Id))!.FramesVersion);
await HealAsync(fx, "frame_removals");
await fx.ScanAsync();
Assert.Equal(result.Sequence.FramesVersion, (await fx.Db.Sequences.GetAsync(sequence.Id))!.FramesVersion);
Assert.SequenceEqual([frames[0].Id, frames[2].Id], (await fx.Db.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id));
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(sequence.Id, frames[1].Sha256!));
}
[Test]
public async Task ThumbnailsFollowTheirFramesThroughADeletionAndARestore()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 5);
async Task<ImagePassResult> ThumbnailsAsync() =>
await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(sequence.Id))!, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(5, (await ThumbnailsAsync()).Built);
var deleted = await DeleteAsync(fx, (await fx.Db.Sequences.GetAsync(sequence.Id))!, frames[1]);
var after = await ThumbnailsAsync();
Assert.Equal((0, 4), (after.Built, after.Reused));
await fx.Service.RestoreEntryAsync(fx.Archive.Id, deleted.Entry.Name, null, fx.Ct);
var again = await ThumbnailsAsync();
Assert.Equal((1, 4), (again.Built, again.Reused));
}
[Test]
public async Task AnEntryRestoredAndDeletedAgainWhileTheTrashEmptiesIsKept()
{
var fx = await FixtureAsync();
var (a, _) = await UploadedAsync(fx, 2);
var (b, _) = await UploadedAsync(fx, 2, "upload-2", 200);
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([a.Id, b.Id], null, fx.Ct), null, null, fx.Ct);
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var items = plan.Entries.OrderBy(e => e.Name, StringComparer.Ordinal).Select(e => new EmptyTrashItem(e.Name, SequenceTrash.EntryFingerprint(e))).ToList();
var later = items[1].Name;
// Another host sharing the archive restores the second entry and deletes its sequence again once the first is erased.
var result = await fx.Service.Trash.EmptyAsync(fx.Archive, new EmptyTrashParameters(items, plan.Fingerprint), 0, async (done, _) =>
{
if (done != 1) return;
var back = await fx.Service.RestoreAsync(fx.Archive.Id, later, fx.Ct);
await DeleteSequenceAsync(fx, back.Id);
}, fx.Ct);
Assert.Equal(1, result.Erased);
Assert.Count(1, result.Refused);
var kept = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == later);
Assert.True(kept.Files > 0);
Assert.FileExists(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), later, ".chronos", "sequence.json"));
}
[Test]
public async Task AResumedEmptyingKeepsAnEntryThatChangedSinceThePlan()
{
var fx = await FixtureAsync();
var (a, _) = await UploadedAsync(fx, 2);
var (b, _) = await UploadedAsync(fx, 2, "upload-2", 200);
await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([a.Id, b.Id], null, fx.Ct), null, null, fx.Ct);
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var names = plan.Entries.Select(e => e.Name).Order(StringComparer.Ordinal).ToList();
File.WriteAllBytes(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), names[1], "added.bin"), [1, 2, 3]);
var engine = await fx.StartEngineAsync();
// An earlier run erased the first entry and stopped; the second changed since the plan.
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.EmptyTrash(fx.Archive, plan), "done:1");
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.Plural("Archive.Error.TrashKept", 1), failed.Error);
var listed = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name).ToList();
Assert.Contains(names[0], listed);
Assert.Contains(names[1], listed);
Assert.FileExists(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), names[1], "added.bin"));
}
/// <summary>An entry of originals a conversion replaced (a later phase), its one photo replaced by <paramref name="replacement"/>.</summary>
private static string WriteReplaced(ArchiveFixture fx, Sequence sequence, string stamp, string replacement)
{
var name = $"{sequence.Id:N}-{stamp}";
var folder = Path.Combine(FrameTrash.Directory(fx.Paths), name);
Directory.CreateDirectory(Path.Combine(folder, "originals"));
var bytes = Photos.Jpeg(Start, 999);
File.WriteAllBytes(Path.Combine(folder, "originals", "IMG_0001.CR2"), bytes);
var sha = Convert.ToHexStringLower(SHA256.HashData(bytes));
var frame = new SidecarFrame { Id = Guid.CreateVersion7(), File = "originals/IMG_0001.CR2", Size = bytes.Length, Sha256 = sha };
var record = new PhotoTrashRecord(TrashKind.Replaced, sequence.Id, sequence.DisplayTitle, sequence.Folder,
[new TrashedPhoto(frame, [new TrashedFile(frame.File, "originals/IMG_0001.CR2", bytes.Length, sha)], replacement)], bytes.Length,
DateTimeOffset.UtcNow, "converter");
File.WriteAllText(folder + ".json", JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord));
return name;
}
}
public sealed class SqliteFrameTrashTests(TestContext context) : FrameTrashTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresFrameTrashTests(TestContext context) : FrameTrashTests(context)
{
protected override Backend Backend => Backend.Postgres;
}