Owner review 2, phase 4b, fix round 2 (trash and sync). - Emptying a Replaced entry decided "conversion deleted" from this host's index alone, so a conversion another host made, or one this index had not read yet, could be taken for deleted and its originals erased without re-hashing it. The survey now reads the archive: the sequence's sidecar where the index or the entry says it is, in the trash, or found by walking the archive's folders. Its Replaced tombstones are followed and every conversion it holds is re-hashed. A conversion counts as deleted only when the sidecar shows it (a Deleted tombstone, or the frame gone while every frame is hashed and no photo of the folder is unlisted), when the sequence is in the trash, or when no sidecar of it exists and this index has no row either. Whatever the archive cannot tell, or a converted frame whose file changed since, keeps the entry, and the job log says why (changed after the plan, sequence busy, conversion no longer matches, conversion not located). - The erase weighs a Replaced entry under the entry's and the sequence's leases and settles the sequence first (an undo waiting for that entry's lease completes), so the entry it reads again and fingerprints is the one it erases. Read before the settle, a consumed entry's folder vanished under the erase. - Each sidecar is indexed once per survey: one conversion's entry (every frame it swapped) is weighed in one pass, not one pass over the sidecar per photo. - The desktop Empty confirmation names every entry whose conversion was deleted first, with why, wherever the trash lists it; the detail scrolls when long. The panel lists them first as well. - Download originals queues the passes a frame swapped to another encoding cleared (light analysis, the deep analysis it had, thumbnails, proxies), as any replacement does. - The job log lines of uploads and downloads are resources (en, it). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 27d9a512183e64da9a3d7fa69a267470ca3ce744)
596 lines
35 KiB
C#
596 lines
35 KiB
C#
using System.Security.Cryptography;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Import;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Core.Project;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Archiving.Jobs;
|
|
using Chronos.Data.Repositories;
|
|
using Chronos.Testing;
|
|
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Chronos.Data.Tests.Archiving;
|
|
|
|
/// <summary>
|
|
/// Frames whose files a conversion replaced (owner review 2, R7) and everything that meets them afterwards: a replacement
|
|
/// that could not be undone, a second conversion, the conversion deleted, the trash emptied, the job that made an entry,
|
|
/// project positions, and the original filed again.
|
|
/// </summary>
|
|
[Category("archive")]
|
|
public sealed class ReplacedTrashTests(TestContext context)
|
|
{
|
|
/// <summary>A sequence of <paramref name="count"/> uploaded photos ten seconds apart, and their bytes.</summary>
|
|
private static async Task<(Guid Id, List<byte[]> Photos)> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1")
|
|
{
|
|
Guid? id = null;
|
|
var photos = new List<byte[]>();
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
var bytes = Photos.Jpeg(Start.AddSeconds(10 * i), 100 + i);
|
|
photos.Add(bytes);
|
|
id = (await CommitAsync(fx, id, id is null ? session : null, bytes, $"IMG_{i + 1:0000}.JPG")).SequenceId;
|
|
}
|
|
return (id!.Value, photos);
|
|
}
|
|
|
|
private static Task<IngestResult> CommitAsync(ArchiveFixture fx, Guid? sequenceId, string? session, byte[] bytes, string name)
|
|
{
|
|
var staging = Path.Combine(fx.Paths.StagingDirectory, "uploads");
|
|
Directory.CreateDirectory(staging);
|
|
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
|
|
File.WriteAllBytes(path, bytes);
|
|
return fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequenceId, null, sequenceId is null ? session : null, path, name, bytes.Length,
|
|
Sha(bytes), null, null), fx.Ct);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Replaces the files of the frames at <paramref name="indices"/> (in the sidecar now) with other photos, as a
|
|
/// conversion's stage B does (<see cref="FrameTrash.ReplaceAsync"/>), on this host or on <paramref name="by"/> (another
|
|
/// host sharing the archive, whose index this one does not read); returns the trash entry of the originals.
|
|
/// </summary>
|
|
private static async Task<TrashEntry> ReplaceAsync(ArchiveFixture fx, Guid sequenceId, IReadOnlyList<int> indices, int seed, Guid? job = null,
|
|
(ArchiveService Service, Archive Archive)? by = null)
|
|
{
|
|
var sequence = (await fx.Db.Sequences.GetAsync(sequenceId))!;
|
|
var manifest = Sidecar(fx, sequence);
|
|
var staging = Path.Combine(fx.Paths.StagingDirectory, "replace-test");
|
|
Directory.CreateDirectory(staging);
|
|
var swaps = new List<FrameSwap>();
|
|
foreach (var i in indices)
|
|
{
|
|
var frame = manifest.Frames[i];
|
|
var bytes = Photos.Jpeg(frame.CapturedAt, seed + i);
|
|
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.jpeg");
|
|
File.WriteAllBytes(path, bytes);
|
|
var sha = Sha(bytes);
|
|
swaps.Add(new FrameSwap(frame.Id, fx.Paths.ToRelative(path), [], (current, _) => Path.ChangeExtension(current.File, ".jpeg"), current => current with
|
|
{
|
|
Sha256 = sha,
|
|
Size = bytes.Length,
|
|
Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size },
|
|
}));
|
|
}
|
|
var (service, archive) = by ?? (fx.Service, fx.Archive);
|
|
var result = Assert.NotNull(await service.FrameTrash.ReplaceAsync(archive, sequenceId, null, swaps, null, job, null, null, fx.Ct));
|
|
return Assert.NotNull(result.Entry);
|
|
}
|
|
|
|
/// <summary>Flips a byte near the end of <paramref name="path"/>; returns its bytes before.</summary>
|
|
private static byte[] Damage(string path)
|
|
{
|
|
var bytes = File.ReadAllBytes(path);
|
|
var damaged = (byte[])bytes.Clone();
|
|
damaged[^10] ^= 0x01;
|
|
File.WriteAllBytes(path, damaged);
|
|
return bytes;
|
|
}
|
|
|
|
private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
|
|
|
|
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
|
|
|
private static string Sha(string path) => Sha(File.ReadAllBytes(path));
|
|
|
|
private static async Task<Job> EmptyAsync(ArchiveFixture fx, Chronos.Data.Jobs.JobEngine engine, TrashEmptyPlan plan, JobState state)
|
|
{
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
|
|
return await fx.WaitForAsync(job.Id, state);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplacementThatCouldNotBeUndoneIsUndoneBeforeAnythingElseIsFiled()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, _) = await UploadedAsync(fx, 2);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
var frame = manifest.Frames[0];
|
|
// A replacement that failed and whose undo failed too: the original in its entry, the converted file in the sequence,
|
|
// the record pending, the sidecar as it was, and the folder the converted file must go back to taken by a file.
|
|
var staging = Path.Combine(fx.Paths.StagingDirectory, "convert-test");
|
|
var staged = Path.Combine(staging, "converted.jpeg");
|
|
var converted = Photos.Jpeg(frame.CapturedAt, 900);
|
|
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000");
|
|
var record = entryFolder + ".json";
|
|
var trashed = Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File));
|
|
var placed = Path.ChangeExtension(frame.File, ".jpeg");
|
|
new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder),
|
|
[new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null)
|
|
{
|
|
Names = new Dictionary<Guid, string> { [frame.Id] = placed },
|
|
}.Write(folder);
|
|
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
|
|
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}");
|
|
File.Move(folder.File(frame.File), trashed);
|
|
File.WriteAllBytes(folder.File(placed), converted);
|
|
Directory.CreateDirectory(fx.Paths.StagingDirectory);
|
|
File.WriteAllText(staging, "in the way");
|
|
|
|
// An upload into the sequence meanwhile: the undo fails again, and nothing is filed.
|
|
await Assert.ThrowsAsync<IOException>(() => CommitAsync(fx, id, null, Photos.Jpeg(Start.AddSeconds(30), 300), "IMG_0003.JPG"));
|
|
Assert.Equal(manifest.Versions.Frames, Sidecar(fx, sequence).Versions.Frames);
|
|
Assert.Equal(ReplacePhase.Undoing, Assert.NotNull(FrameReplaceJournal.Read(folder)).Phase);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
|
|
// Another writer of an earlier build moved the frames version meanwhile: the replacement is undone all the same, never
|
|
// taken for one whose sidecar was written.
|
|
ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 } });
|
|
File.Delete(staging);
|
|
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
|
|
Assert.FileMissing(folder.ReplaceJournalPath);
|
|
Assert.Equal(frame.Sha256, Sha(folder.File(frame.File)));
|
|
Assert.FileMissing(folder.File(placed));
|
|
Assert.Equal(Sha(converted), Sha(staged));
|
|
Assert.False(Directory.Exists(entryFolder));
|
|
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
Assert.SequenceEqual(manifest.Frames.Select(f => (f.Id, f.File, f.Sha256)), Sidecar(fx, sequence).Frames.Select(f => (f.Id, f.File, f.Sha256)));
|
|
|
|
// Settled, the upload goes in.
|
|
await CommitAsync(fx, id, null, Photos.Jpeg(Start.AddSeconds(30), 300), "IMG_0003.JPG");
|
|
Assert.Count(3, Sidecar(fx, sequence).Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnInterruptedReplacementIsSettledByWhatItsSidecarLists()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, _) = await UploadedAsync(fx, 2);
|
|
var entry = await ReplaceAsync(fx, id, [0], 500);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
var (entryFolder, record) = FrameTrash.Location(fx.Paths, entry.Name);
|
|
// As a crash right after the sidecar was written leaves it (at the version an ingest would give too): completed.
|
|
File.Move(record, FrameRemovalJournal.PendingRecord(record));
|
|
new FrameReplaceJournal(manifest.Versions.Frames - 1, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), [], [], false, null)
|
|
{
|
|
Names = new Dictionary<Guid, string> { [manifest.Frames[0].Id] = manifest.Frames[0].File },
|
|
}.Write(folder);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
|
|
Assert.FileMissing(folder.ReplaceJournalPath);
|
|
Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
|
|
// As a crash before the sidecar was written leaves it, with the frames version moved by another writer since: undone.
|
|
manifest = Sidecar(fx, sequence);
|
|
var frame = manifest.Frames[1];
|
|
var staged = Path.Combine(fx.Paths.StagingDirectory, "convert-test", "converted.jpeg");
|
|
var converted = Photos.Jpeg(frame.CapturedAt, 901);
|
|
var second = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027130000");
|
|
var trashed = Path.Combine(second, "originals", Path.GetFileName(frame.File));
|
|
var placed = Path.ChangeExtension(frame.File, ".jpeg");
|
|
new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(second + ".json"), fx.Paths.ToRelative(second),
|
|
[new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null)
|
|
{
|
|
Names = new Dictionary<Guid, string> { [frame.Id] = placed },
|
|
}.Write(folder);
|
|
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
|
|
File.WriteAllText(FrameRemovalJournal.PendingRecord(second + ".json"), "{}");
|
|
File.Move(folder.File(frame.File), trashed);
|
|
File.WriteAllBytes(folder.File(placed), converted);
|
|
ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 } });
|
|
|
|
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 3 }, fx.Ct);
|
|
Assert.FileMissing(folder.ReplaceJournalPath);
|
|
Assert.Equal(frame.Sha256, Sha(folder.File(frame.File)));
|
|
Assert.FileMissing(folder.File(placed));
|
|
Assert.Equal(Sha(converted), Sha(staged));
|
|
Assert.False(Directory.Exists(second));
|
|
Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OriginalsConvertedTwiceAreErasedWhileTheLastConversionHolds()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, _) = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
await ReplaceAsync(fx, id, [0, 1], 500);
|
|
await ReplaceAsync(fx, id, [0, 1], 600);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var current = fx.Paths.Sequence(sequence.Folder).File(Sidecar(fx, sequence).Frames[0].File);
|
|
var bytes = File.ReadAllBytes(current);
|
|
|
|
// The last conversion damaged: neither the originals nor the first conversion go.
|
|
var damaged = (byte[])bytes.Clone();
|
|
damaged[^10] ^= 0x01;
|
|
File.WriteAllBytes(current, damaged);
|
|
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
Assert.Count(2, plan.Entries);
|
|
Assert.True(plan.Entries.All(e => !e.ConversionDeleted), "the conversions are in the sequence");
|
|
await EmptyAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Count(2, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
|
|
// Whole again: the first originals are held by the second conversion, through the first one's tombstone.
|
|
File.WriteAllBytes(current, bytes);
|
|
await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task OriginalsWhoseConversionWasDeletedAreErasedAsTheDryRunSays()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, _) = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
|
var before = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
Assert.False(before.Entries.Single().ConversionDeleted);
|
|
|
|
// The user deletes the converted photo.
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var deletion = await fx.Service.FrameTrash.DeleteAsync(await fx.Service.FrameTrash.PlanDeleteAsync(id, [frames[0].Id], sequence.FramesVersion, fx.Ct),
|
|
null, null, fx.Ct);
|
|
|
|
// The plan made before no longer holds: nothing is erased.
|
|
await EmptyAsync(fx, engine, before, JobState.Failed);
|
|
Assert.Count(2, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
|
|
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
Assert.True(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted, "the dry run says the conversion was deleted");
|
|
Assert.False(plan.Entries.Single(e => e.Name == deletion.Entry.Name).ConversionDeleted);
|
|
Assert.NotEqual(SequenceTrash.EntryFingerprint(plan.Entries.Single(e => e.Name == replaced.Name) with { ConversionDeleted = false }),
|
|
SequenceTrash.EntryFingerprint(plan.Entries.Single(e => e.Name == replaced.Name)));
|
|
|
|
// The deleted conversion is still in the trash, damaged: the originals stay.
|
|
var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, deletion.Entry.Name));
|
|
var copy = Path.Combine(FrameTrash.Location(fx.Paths, deletion.Entry.Name).Folder, record.Photos[0].Files[0].Path);
|
|
var bytes = File.ReadAllBytes(copy);
|
|
var damaged = (byte[])bytes.Clone();
|
|
damaged[^10] ^= 0x01;
|
|
File.WriteAllBytes(copy, damaged);
|
|
await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Failed);
|
|
Assert.Contains(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
|
|
|
|
// Whole again, both go; and once the conversion's entry is erased, the originals go with nothing left to check.
|
|
File.WriteAllBytes(copy, bytes);
|
|
await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task OriginalsWhoseConvertedSequenceWasDeletedAreErasedWithIt()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, _) = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var replaced = await ReplaceAsync(fx, id, [0, 1], 500);
|
|
await fx.Service.Trash.DeleteAsync(await fx.Service.Trash.PlanAsync([id], null, fx.Ct), null, null, fx.Ct);
|
|
|
|
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
Assert.Count(2, plan.Entries);
|
|
Assert.True(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted);
|
|
Assert.False(plan.Entries.Single(e => e.Kind == TrashKind.Sequence).ConversionDeleted);
|
|
// Only the originals: the sequence's entry still holds the conversions, which hash to their records.
|
|
await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Done);
|
|
Assert.Equal(TrashKind.Sequence, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Kind);
|
|
|
|
await fx.Service.RestoreAsync(fx.Archive.Id, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name, fx.Ct);
|
|
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(id));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AConversionAnotherHostMadeIsHashedWhereTheArchiveHoldsIt()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, photos) = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
// A desktop sharing the archive converts the first photo; this index still holds the original there.
|
|
var replaced = await ReplaceAsync(fx, id, [0], 500, by: await fx.OtherWriterAsync());
|
|
Assert.Equal(Sha(photos[0]), (await fx.Db.Frames.ListBySequenceAsync(id))[0].Sha256);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var converted = fx.Paths.Sequence(sequence.Folder).File(Sidecar(fx, sequence).Frames[0].File);
|
|
|
|
// The conversion is damaged since: the dry run finds it in the sequence, and the emptying keeps the originals.
|
|
var bytes = Damage(converted);
|
|
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
Assert.False(plan.Entries.Single().ConversionDeleted, "the archive holds the conversion, whatever this index has");
|
|
var kept = await EmptyAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), kept.LogTail);
|
|
Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
|
|
// Whole again: the originals go.
|
|
File.WriteAllBytes(converted, bytes);
|
|
await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AConversionAnotherIndexMovedIsHashedWhereItIsNow()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, _) = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
|
// Another index moves the sequence into another folder; this one has not scanned since.
|
|
var before = (await fx.Db.Sequences.GetAsync(id))!.Folder;
|
|
var (other, archive) = await fx.OtherWriterAsync();
|
|
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], "moved/{title}", null, fx.Ct), null, fx.Ct);
|
|
Assert.Equal(before, (await fx.Db.Sequences.GetAsync(id))!.Folder);
|
|
var folder = ArchiveScanner.FindSequenceFolders(fx.Paths, fx.Ct).Select(s => fx.Paths.Sequence(s.RelativeFolder)).Single(s => s.RelativeFolder != before);
|
|
Assert.True(folder.RelativeFolder.StartsWith("moved", StringComparison.Ordinal), folder.RelativeFolder);
|
|
var converted = folder.File(ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].File);
|
|
|
|
var bytes = Damage(converted);
|
|
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
Assert.False(plan.Entries.Single().ConversionDeleted, "found in the folder the sequence is in now");
|
|
Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
|
|
|
|
File.WriteAllBytes(converted, bytes);
|
|
await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task OriginalsStayWhileTheArchiveCannotTellWhereTheirConversionIs()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, _) = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
var unknown = Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title);
|
|
|
|
// A sidecar rebuilt from the photos, as a scan adopts a folder that lost its own: new frame ids, nothing hashed, no
|
|
// tombstones. It no longer says what became of the conversion.
|
|
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
|
|
{
|
|
Frames = [.. manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7(), Sha256 = null })],
|
|
Removed = null,
|
|
});
|
|
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
Assert.False(plan.Entries.Single().ConversionDeleted);
|
|
Assert.Contains(unknown, (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
|
|
|
|
// A sidecar that cannot be read.
|
|
File.WriteAllText(folder.ManifestPath, "{ \"damaged");
|
|
Assert.Contains(unknown, (await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Failed)).LogTail);
|
|
|
|
// The sequence gone from the archive (deleted and erased by another host) while this index still has it: kept until a
|
|
// scan settles the index. Then the dry run says the conversion was deleted, and the emptying confirmed with it erases.
|
|
Directory.Delete(folder.Folder, recursive: true);
|
|
Assert.Contains(unknown, (await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Failed)).LogTail);
|
|
Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
await fx.ScanAsync(ScanMode.Rebuild);
|
|
Assert.Null(await fx.Db.Sequences.GetAsync(id));
|
|
var gone = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
Assert.True(gone.Entries.Single().ConversionDeleted, "no sequence of that id anywhere");
|
|
await EmptyAsync(fx, engine, gone, JobState.Done);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task OriginalsOfABusySequenceStayInTheTrash()
|
|
{
|
|
await using var fx = await CreateAsync(context, configure: o => o with { LeaseWait = TimeSpan.FromMilliseconds(300) });
|
|
var (id, _) = await UploadedAsync(fx, 2);
|
|
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
|
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
var parameters = new EmptyTrashParameters([.. plan.Entries.Select(e => new EmptyTrashItem(e.Name, SequenceTrash.EntryFingerprint(e)))], plan.Fingerprint);
|
|
|
|
// Another host writes the sequence meanwhile: the conversion is not weighed under it.
|
|
using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, $"seq-{id:N}", "another host")))
|
|
{
|
|
var busy = await fx.Service.Trash.EmptyAsync(fx.Archive, parameters, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.Equal((0, TrashKeptReason.Busy), (busy.Erased, busy.Refused.Single().Reason));
|
|
Assert.Equal(Loc.L("Archive.Log.TrashKeptBusy", replaced.Title), busy.Refused.Single().LogLine);
|
|
}
|
|
Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
|
|
var free = await fx.Service.Trash.EmptyAsync(fx.Archive, parameters, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.Equal((1, 0), (free.Erased, free.Refused.Count));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AConversionAnotherHostDeletedIsReadFromTheArchiveNotThisIndex()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, _) = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
|
// A desktop sharing the archive deletes the converted photo; this index still holds it.
|
|
var (other, _) = await fx.OtherWriterAsync();
|
|
var deletion = await other.FrameTrash.DeleteAsync(await other.FrameTrash.PlanDeleteAsync(id, [frames[0].Id], null, fx.Ct), null, null, fx.Ct);
|
|
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(id));
|
|
|
|
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
Assert.True(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted, "the sidecar's tombstone says so, whatever this index holds");
|
|
Assert.False(plan.Entries.Single(e => e.Name == deletion.Entry.Name).ConversionDeleted);
|
|
|
|
// The deleted conversion is in the trash, damaged: the originals stay.
|
|
var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, deletion.Entry.Name));
|
|
var copy = Path.Combine(FrameTrash.Location(fx.Paths, deletion.Entry.Name).Folder, record.Photos[0].Files[0].Path);
|
|
var bytes = Damage(copy);
|
|
var kept = await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Failed);
|
|
Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), kept.LogTail);
|
|
|
|
// Whole again: the originals go, the deleted conversion stays where it is.
|
|
File.WriteAllBytes(copy, bytes);
|
|
await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Done);
|
|
Assert.Equal(deletion.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OriginalsStayWhenTheirConversionChangedSince()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, _) = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
// The converted file was edited in place and hashed again: its frame holds other content, and nothing says what
|
|
// became of the conversion.
|
|
var edited = Photos.Jpeg(manifest.Frames[0].CapturedAt, 700);
|
|
File.WriteAllBytes(folder.File(manifest.Frames[0].File), edited);
|
|
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
|
|
{
|
|
Frames = [.. manifest.Frames.Select((f, i) => i == 0 ? f with { Sha256 = Sha(edited), Size = edited.Length } : f)],
|
|
});
|
|
|
|
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
Assert.False(plan.Entries.Single().ConversionDeleted, "the frame is there");
|
|
Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
|
|
Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnEmptyingReadsAnEntryOnlyOnceAnUndoWaitingForItIsSettled()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, photos) = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
|
var (entryFolder, recordPath) = FrameTrash.Location(fx.Paths, replaced.Name);
|
|
var record = await File.ReadAllTextAsync(recordPath, fx.Ct);
|
|
// Undone: the original is back in its frame, the conversion in an entry of its own.
|
|
await fx.Service.RestoreEntryAsync(fx.Archive.Id, replaced.Name, null, fx.Ct);
|
|
var undo = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
// As a crash right after the undo's sidecar was written leaves it: the entry its original came from still lists it,
|
|
// and the undo's completion waits for that entry's lease.
|
|
Directory.CreateDirectory(entryFolder);
|
|
await File.WriteAllTextAsync(recordPath, record, fx.Ct);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
var (undoFolder, undoRecord) = FrameTrash.Location(fx.Paths, undo.Name);
|
|
new FrameReplaceJournal(manifest.Versions.Frames - 1, fx.Paths.ToRelative(undoRecord), fx.Paths.ToRelative(undoFolder), [], [], false,
|
|
new ConsumedEntry(fx.Paths.ToRelative(recordPath), fx.Paths.ToRelative(entryFolder), SequenceTrash.EntryLease(replaced.Name), [manifest.Frames[0].Id],
|
|
null))
|
|
{
|
|
Phase = ReplacePhase.Written,
|
|
}.Write(folder);
|
|
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct);
|
|
Assert.Equal(replaced.Name, plan.Entries.Single().Name);
|
|
|
|
// The emptying settles the undo under the entry's lease before it reads the entry again, which takes the photo out of
|
|
// it: the entry is not erased as planned, and the sequence keeps its original.
|
|
var kept = await EmptyAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Contains(Loc.L("Archive.Log.TrashKept", replaced.Title), kept.LogTail);
|
|
Assert.FileMissing(folder.ReplaceJournalPath);
|
|
Assert.Equal(undo.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
Assert.Equal(Sha(photos[0]), Sha(folder.File(Sidecar(fx, sequence).Frames[0].File)));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnEntryNamesTheJobThatReplacedItsPhotos()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, photos) = await UploadedAsync(fx, 2);
|
|
var job = Guid.CreateVersion7();
|
|
var entry = await ReplaceAsync(fx, id, [0, 1], 500, job);
|
|
Assert.Equal(job, entry.JobId);
|
|
Assert.Equal(job, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().JobId);
|
|
|
|
// Its undo swaps the originals back; the entry of the converted files it makes names no job.
|
|
var undone = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
|
|
Assert.False(undone.NewSequence);
|
|
Assert.SequenceEqual(photos.Select(Sha), (await fx.Db.Frames.ListBySequenceAsync(id)).Select(f => f.Sha256!));
|
|
var swapped = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
Assert.Equal(TrashKind.Replaced, swapped.Kind);
|
|
Assert.Null(swapped.JobId);
|
|
}
|
|
|
|
[Test]
|
|
public async Task PositionsOnAConvertedFrameStayWhenAFrameWentBehindTheIndex()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, _) = await UploadedAsync(fx, 3);
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
|
var project = new ProjectSettings { ExcludedFrames = [2], HeroFrame = 2 }.ToJson();
|
|
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = project, HeroFrameId = frames[2].Id }, null, fx.Ct);
|
|
// A desktop sharing the archive deletes the first photo; this index has not read the sidecar since.
|
|
var (other, _) = await fx.OtherWriterAsync();
|
|
await other.FrameTrash.DeleteAsync(await other.FrameTrash.PlanDeleteAsync(id, [frames[0].Id], null, fx.Ct), null, null, fx.Ct);
|
|
|
|
// This host converts the photo the positions are on (second in the sidecar now).
|
|
await ReplaceAsync(fx, id, [1], 500);
|
|
|
|
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var settings = ProjectSettings.FromJson(row.SettingsJson!);
|
|
Assert.SequenceEqual([1], settings.ExcludedFrames);
|
|
Assert.Equal((1, frames[2].Id), (settings.HeroFrame, row.HeroFrameId));
|
|
Assert.True(settings.PendingFrames is not { Count: > 0 }, "nothing waits for the photo the conversion replaced");
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheOriginalOfAConvertedPhotoFiledAgainIsADuplicateOfItsConversion()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var (id, photos) = await UploadedAsync(fx, 2, "card-1");
|
|
await ReplaceAsync(fx, id, [0], 500);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var converted = Sidecar(fx, sequence).Frames[0];
|
|
|
|
// The card imported again on a PC without its history (same session, so the same sequence).
|
|
var again = await CommitAsync(fx, null, "card-1", photos[0], "IMG_0001.JPG");
|
|
Assert.Equal((id, converted.Id, true), (again.SequenceId, again.FrameId, again.Duplicate));
|
|
var after = Sidecar(fx, sequence);
|
|
Assert.Count(2, after.Frames);
|
|
Assert.Contains(after.Removed!, r => r.Sha256 == Sha(photos[0]) && r.Replacement == converted.Sha256 && r.Reason == RemovalReason.Replaced);
|
|
Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(id, Sha(photos[0]), fx.Ct), "the alias stays in the index");
|
|
|
|
// Converted once more: the original is a duplicate of the last conversion.
|
|
await ReplaceAsync(fx, id, [0], 600);
|
|
var last = Sidecar(fx, sequence).Frames[0];
|
|
var third = await CommitAsync(fx, null, "card-1", photos[0], "IMG_0001.JPG");
|
|
Assert.Equal((last.Id, true), (third.FrameId, third.Duplicate));
|
|
Assert.Count(2, Sidecar(fx, sequence).Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFolderImportedAgainDoesNotFileTheOriginalsOfItsConvertedPhotos()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var source = Path.Combine(context.CreateTempDirectory("source"), "Etna eruption");
|
|
Photos.WriteSeries(source, Start, 3);
|
|
var whole = new ImportOptions { Grouping = ImportGrouping.WholeFolder };
|
|
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, source, whole, null, null, fx.Ct);
|
|
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, null, true, null), Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null,
|
|
fx.Ct);
|
|
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions.Single().SessionKey))!.Value;
|
|
await ReplaceAsync(fx, id, [0, 1, 2], 500);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var aliases = Sidecar(fx, sequence).Removed!.Count;
|
|
|
|
// A new shot in the folder, imported with the rest of it.
|
|
Photos.WriteSeries(source, Start.AddMinutes(5), 1, firstSeed: 40, prefix: "LATE_");
|
|
var later = await fx.Service.PlanImportAsync(fx.Archive.Id, source, whole, null, null, fx.Ct);
|
|
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, later, null, true, null), Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null,
|
|
fx.Ct);
|
|
|
|
var after = Sidecar(fx, sequence);
|
|
Assert.Count(4, after.Frames);
|
|
Assert.Equal(aliases, after.Removed!.Count);
|
|
Assert.Equal(3, after.Frames.Count(f => f.Source is not null));
|
|
}
|
|
}
|