Files
Chronos/tests/Chronos.Data.Tests/Archiving/SequenceMergerTests.cs
T
Alby96andClaude Opus 5.5 b4070ddfd4 feat(archive): merge two local sequences into the kept one (owner review 6, P4)
SequenceMerger plans the merge of a local pair (kept sequence, photos
copied by policy, last copies, renders, details, projects, session keys,
links, blocks and a fingerprint) and queues the MergeSequences job.
MergeJobHandler applies the confirmed plan through a journal, so every
stage resumes: copy and file the other's photos, move renders, hash-verify
every original of the other in the kept one (or list it as a last copy),
trash the kept one's losing versions, write the details
(SequencePayloadWriter), copy projects, move keys and links, then move the
other to its archive's trash only at the verified frames version. A cancel
drops the journal and returns the pair to its previous state.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 10:24:32 +01:00

631 lines
35 KiB
C#

using System.Security.Cryptography;
using Chronos.Core.Archiving;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Copies;
using Chronos.Data.Jobs;
using Chronos.Data.Repositories;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>
/// Merging two local copies of one time-lapse (owner review 6, #73; design §5d, §11 Merge): the dry run (kept sequence,
/// photos copied, last copies, session keys, links, blocks, fingerprint) and the job that applies it, verifies every photo
/// of the other in the kept one and only then moves the other to its archive's trash.
/// </summary>
[Category("archive")]
public abstract class SequenceMergerTests(TestContext context) : DatabaseTest(context)
{
private const string Nas = "nas.local";
private static readonly CopyPlanContext NoContext = PlanContext();
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
private static CopyPlanContext PlanContext(params Guid[] originals) =>
new("PC", null, false, false, false, false, false, false, null, new HashSet<Guid>(), originals.ToHashSet());
private static SequenceMerger Merger(ArchiveFixture fx) => new(fx.Db, fx.Service, new CopySides(fx.Db));
/// <summary>A sequence uploaded photo by photo into <paramref name="archive"/>: (seed, capture offset in ms, file name).</summary>
private static async Task<Sequence> SequenceAsync(ArchiveFixture fx, string session, IReadOnlyList<(int Seed, int Ms, string Name)> photos, Archive? archive = null)
{
Guid? id = null;
var staging = Path.Combine(new ArchivePaths((archive ?? fx.Archive).RootPath).StagingDirectory, "uploads");
Directory.CreateDirectory(staging);
foreach (var (seed, ms, name) in photos)
{
var bytes = Photos.Jpeg(Start.AddMilliseconds(ms), seed);
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
File.WriteAllBytes(path, bytes);
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest((archive ?? fx.Archive).Id, id, null, id is null ? session : null, path, name,
bytes.Length, Sha(bytes), null, null), fx.Ct);
id = result.SequenceId;
}
return (await fx.Db.Sequences.GetAsync(id!.Value))!;
}
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private static string Sha(int seed, int ms) => Sha(Photos.Jpeg(Start.AddMilliseconds(ms), seed));
private static async Task<FoundCopy> PairAsync(ArchiveFixture fx, Guid a, Guid b) =>
await new FoundCopyStore(fx.Db).UpsertFoundAsync(new FoundCopy
{
Id = Guid.NewGuid(),
Kind = CopyKind.LocalLocal,
LocalId = a,
OtherId = b,
Evidence = CopyEvidence.Hashes,
Fingerprint = "f",
}, fx.Ct);
/// <summary>Choices with every detail line on the kept side but those in <paramref name="other"/>.</summary>
private static async Task<MergeChoices> ChoicesAsync(ArchiveFixture fx, Guid keep, Guid drop, CopyPhotoPolicy policy,
IReadOnlyDictionary<string, PhotoChoice>? perPhoto = null, params string[] other)
{
var keepRow = (await fx.Db.Sequences.GetAsync(keep))!;
var dropRow = (await fx.Db.Sequences.GetAsync(drop))!;
var lines = CopyPayload.Lines(SyncPayload.Of(keepRow), await SequencePayloadWriter.PhotosAsync(fx.Db, keep, null, fx.Ct),
SyncPayload.Of(dropRow), await SequencePayloadWriter.PhotosAsync(fx.Db, drop, null, fx.Ct));
var details = lines.Select(l => l.Path).Distinct().ToDictionary(p => p, p => other.Contains(p) ? CopySide.Other : CopySide.ThisPc);
return new MergeChoices(policy, perPhoto ?? new Dictionary<string, PhotoChoice>(), details);
}
private static async Task<Sequence> TitledAsync(ArchiveFixture fx, Sequence sequence, string title)
{
var result = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Title = title }, null, fx.Ct);
return result.Sequence!;
}
private static Task<Job> RunningAsync(ArchiveFixture fx, JobKind kind, params Guid[] targets) => EnqueuedAsync(fx, new Job
{
Id = Guid.CreateVersion7(),
Kind = kind,
TargetIds = targets,
State = JobState.Running,
NodeId = "test#0",
Created = DateTimeOffset.UtcNow,
Started = DateTimeOffset.UtcNow,
});
private static async Task<Job> EnqueuedAsync(ArchiveFixture fx, Job job)
{
await fx.Db.Jobs.EnqueueAsync(job);
return job;
}
private (JobEngine Engine, MergeJobHandler Handler) Engine(ArchiveFixture fx, SequenceMerger merger, FakeProjects projects, FakeWorkspace workspace)
{
var handler = (MergeJobHandler)SequenceMerger.CreateHandler(fx.Db, merger, fx.Service, projects, workspace);
var engine = Own(new JobEngine(fx.Db, [.. fx.Service.CreateJobHandlers(), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }));
return (engine, handler);
}
private static async Task<IReadOnlySet<string>> HashesAsync(ArchiveFixture fx, Guid sequenceId) =>
(await fx.Db.Frames.ListBySequenceAsync(sequenceId)).Select(f => f.Sha256!).ToHashSet(StringComparer.Ordinal);
/// <summary>
/// A pair with something in every stage: shared photo 1, photo 2 kept only, photo 3 only in the other, photo 4 different at
/// one instant (same name), a render inside the other's folder, a project, a session key, a link and a title.
/// </summary>
private static async Task<Scenario> ScenarioAsync(ArchiveFixture fx, int n)
{
var b = 100 * n;
var keep = await SequenceAsync(fx, $"keep-{n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 2, 10_000, "IMG_0002.JPG"), (b + 4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, $"drop-{n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 3, 20_000, "IMG_0003.JPG"), (b + 5, 30_250, "IMG_0004.JPG")]);
drop = await TitledAsync(fx, drop, $"Harbour {n}");
var exports = fx.Paths.Sequence(drop.Folder).Exports;
Directory.CreateDirectory(exports);
var clip = Path.Combine(exports, "clip.mp4");
var bytes = RandomNumberGenerator.GetBytes(4096);
File.WriteAllBytes(clip, bytes);
var render = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = drop.Id,
PresetName = "1080p",
OutputPath = fx.Paths.ToRelative(clip),
Bytes = bytes.Length,
Sha256 = Sha(bytes),
Created = DateTimeOffset.UtcNow,
};
await fx.Db.Renders.InsertAsync(render, fx.Ct);
var project = await fx.Service.Projects.CreateAsync(drop.Id, "Night cut", cancellationToken: fx.Ct);
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = drop.DisplayTitle }, fx.Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var differs = Sha(b + 4, 30_250);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.OtherWins, null, SyncRules.MetadataPath + "title");
return new Scenario(keep, drop, copy, render, project, choices, differs);
}
private sealed record Scenario(Sequence Keep, Sequence Drop, FoundCopy Copy, Render Render, SequenceProject Project, MergeChoices Choices, string KeepLoser);
// ---- the dry run ----
[Test]
public async Task TheKeptOneIsPreselectedOnlyWhenOneIsTheLinkedOriginalsCopy()
{
var fx = await FixtureAsync();
var a = await SequenceAsync(fx, "a", [(1, 0, "IMG_0001.JPG")]);
var b = await SequenceAsync(fx, "b", [(1, 0, "IMG_0001.JPG")]);
var copy = await PairAsync(fx, a.Id, b.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, a.Id, b.Id, CopyPhotoPolicy.Union);
// Nothing linked: the user chooses; the plan keeps the one the dialog does not name.
var plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
Assert.Equal((a.Id, b.Id), (plan.KeepId, plan.DropId));
// One linked, in a server's originals archive: preselected (also when the dialog names it as the other).
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = b.Id, Title = "b" }, Ct);
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
Assert.DoesNotContain(CopyBlock.KeptUnchosen, plan.Blocks);
Assert.Equal((b.Id, a.Id), (plan.KeepId, plan.DropId));
// Linked but not in an originals archive: asked.
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
// Both linked: asked again.
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = "other.local", RemoteId = Guid.CreateVersion7(), LocalId = a.Id, Title = "a" }, Ct);
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
// Chosen: never asked.
plan = await merger.PlanAsync(copy.Id, a.Id, b.Id, choices, NoContext, Ct);
Assert.Empty(plan.Blocks);
Assert.Equal((a.Id, b.Id), (plan.KeepId, plan.DropId));
}
[Test]
public async Task OnlyThereIsAlwaysCopiedAndThePolicyDecidesDifferentPhotosOnly()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (3, 20_000, "IMG_0003.JPG"), (5, 30_250, "IMG_0004.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (only, theirs, mine) = (Sha(3, 20_000), Sha(5, 30_250), Sha(4, 30_250));
// Choose each with nothing chosen: the only-there photo still comes; the other's different version is a last copy.
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ChoosePerPhoto), NoContext, Ct);
Assert.Empty(plan.Blocks);
Assert.SequenceEqual([(PhotoAction.Fetch, only)], plan.Photos.Select(p => (p.Action, p.Sha256!)));
Assert.SequenceEqual([(CopySide.Other, theirs)], plan.LastCopies.Select(l => (l.Side, l.Key)));
// Choose each, Take the other's: its version comes, the kept one's goes (a last copy of this PC).
var key = mine;
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id,
await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ChoosePerPhoto, new Dictionary<string, PhotoChoice> { [key] = PhotoChoice.TakeThere }), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, only), (PhotoAction.FetchVersion, theirs), (PhotoAction.TrashHere, mine)],
plan.Photos.Select(p => (p.Action, p.Sha256!)).OrderBy(p => p.Action));
Assert.SequenceEqual([(CopySide.ThisPc, mine)], plan.LastCopies.Select(l => (l.Side, l.Key)));
// Union keeps both versions: nothing is a last copy.
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, only), (PhotoAction.FetchVersion, theirs)], plan.Photos.Select(p => (p.Action, p.Sha256!)).OrderBy(p => p.Action));
Assert.Empty(plan.LastCopies);
Assert.Equal(plan.Photos.Sum(p => p.Bytes), plan.CopyBytes);
// The kept one wins: nothing of the different photo moves; the other's version is a last copy.
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ThisPcWins), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, only)], plan.Photos.Select(p => (p.Action, p.Sha256!)));
Assert.SequenceEqual([theirs], plan.LastCopies.Select(l => l.Key));
}
[Test]
public async Task APhotoTombstonedInTheKeptOneIsALastCopyUnlessAnotherSequenceHoldsIt()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var gone = Sha(2, 10_000);
await fx.Db.FrameRemovals.SyncSequenceAsync(keep.Id, [new FrameRemoval(keep.Id, gone, null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Empty(plan.Photos);
Assert.SequenceEqual([(CopySide.Other, gone)], plan.LastCopies.Select(l => (l.Side, l.Key)));
await SequenceAsync(fx, "third", [(2, 10_000, "IMG_0002.JPG")]);
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Empty(plan.LastCopies);
}
[Test]
public async Task SessionKeysFollowTheKeptOneButOneItsArchiveMapsElsewhereAndLinksArePlanned()
{
var fx = await FixtureAsync();
var second = await fx.Service.AdoptAsync(Context.CreateTempDirectory("second"), "Second", Ct);
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var third = await SequenceAsync(fx, "card-7", [(9, 0, "IMG_0009.JPG")]);
var drop = await SequenceAsync(fx, "card-7", [(1, 0, "IMG_0001.JPG")], second);
await fx.Db.Sequences.MapSessionKeyAsync(second.Id, "card-8", drop.Id, Ct);
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = "d" }, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var plan = await Merger(fx).PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Blocks);
Assert.SequenceEqual(["card-8"], plan.SessionKeys);
Assert.SequenceEqual(["card-7"], plan.SkippedSessionKeys);
Assert.SequenceEqual([Nas], plan.Links);
Assert.Equal(third.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "card-7", Ct));
}
[Test]
public async Task AnotherJobOnEitherSequenceAndBothLinkedToOneServerBlockThePlan()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
var render = await RunningAsync(fx, JobKind.Render, keep.Id);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.Busy, plan.Blocks);
await fx.Db.Jobs.SetStateAsync(render.Id, JobState.Done, null, Ct);
Assert.Empty((await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks);
var links = new SyncLinkStore(fx.Db);
await links.UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = keep.Id, Title = "k" }, Ct);
await links.UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = "d" }, Ct);
Assert.Contains(CopyBlock.BothLinked, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks);
var unsent = PlanContext() with { UnsentWorkspaceSequences = new HashSet<Guid> { drop.Id } };
Assert.Contains(CopyBlock.UnsentWorkspace, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, unsent, Ct)).Blocks);
// A detail that differs and is not chosen.
await TitledAsync(fx, drop, "Other title");
var unchosen = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.DetailsUnchosen, unchosen.Blocks);
Assert.Null(unchosen.Payload);
}
[Test]
public async Task TheFingerprintMovesWithADetailOfTheKeptOneNotWithAnImportElsewhere()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
var first = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
await SequenceAsync(fx, "elsewhere", [(7, 0, "IMG_0007.JPG")]);
Assert.Equal(first.Fingerprint, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Fingerprint);
await fx.Db.Sequences.EditAsync(keep.Id, new SequenceEdit { Rating = 4 }, null, Ct);
Assert.NotEqual(first.Fingerprint, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Fingerprint);
}
// ---- the job ----
[Test]
public async Task AMergeCopiesVerifiesAndMovesEverythingThenTrashesTheOther()
{
var fx = await FixtureAsync();
var s = await ScenarioAsync(fx, 0);
var merger = Merger(fx);
var projects = new FakeProjects();
var (engine, _) = Engine(fx, merger, projects, new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
Assert.Empty(plan.Blocks);
Assert.SequenceEqual([(CopySide.ThisPc, s.KeepLoser)], plan.LastCopies.Select(l => (l.Side, l.Key)));
Assert.SequenceEqual([s.Render.Id], plan.Renders.Select(r => r.RenderId));
Assert.SequenceEqual([("Night cut", "Night cut")], plan.Projects.Select(p => (p.SourceName, p.TargetName)));
Assert.Equal(1, plan.DetailChanges);
Assert.Equal("Harbour 0", plan.Payload!.Title);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
Assert.Equal(CopyState.Integrating, (await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct))!.State);
await fx.WaitForAsync(job.Id, JobState.Done);
// Photos: shared, kept only, only in the other and the other's winning version; the kept one's losing version went.
Assert.True((await HashesAsync(fx, s.Keep.Id)).SetEquals([Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(5, 30_250)]));
var keep = (await fx.Db.Sequences.GetAsync(s.Keep.Id))!;
Assert.Equal("Harbour 0", keep.Title);
// The render keeps its id, its file copied into the kept sequence.
var render = (await fx.Db.Renders.GetAsync(s.Render.Id))!;
Assert.Equal(s.Keep.Id, render.SequenceId);
Assert.Equal(s.Render.Sha256, Sha(File.ReadAllBytes(fx.Paths.FromRelative(render.OutputPath))));
Assert.Equal(1, keep.RenderedCount);
Assert.SequenceEqual([("Night cut", s.Project.Id)], projects.Copied.Select(c => (c.Name, c.Source)));
// Keys, links and the pair.
Assert.Equal(s.Keep.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "drop-0", Ct));
Assert.Contains("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(keep.Folder).ManifestPath).SessionKeys);
Assert.Equal(s.Keep.Id, (await new SyncLinkStore(fx.Db).ListAsync(Nas, Ct)).Single().LocalId);
Assert.Null(await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
// The other is in its archive's trash, restorable, without the keys it gave away.
Assert.Null(await fx.Db.Sequences.GetAsync(s.Drop.Id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, Ct)).Single(e => e.SequenceId == s.Drop.Id && e.Kind == TrashKind.Sequence);
var restored = await fx.Service.RestoreAsync(fx.Archive.Id, entry.Name, Ct);
Assert.Equal(s.Drop.Id, restored.Id);
Assert.Equal(3, restored.FrameCount);
Assert.DoesNotContain("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(restored.Folder).ManifestPath).SessionKeys);
}
[Test]
public async Task ACrashAtEachStageResumesWhereItStopped()
{
var fx = await FixtureAsync();
var merger = Merger(fx);
var projects = new FakeProjects();
var (engine, handler) = Engine(fx, merger, projects, new FakeWorkspace());
await engine.StartAsync(Ct);
var stages = Enum.GetValues<MergeStage>().Where(s => s != MergeStage.Done).ToList();
for (var i = 0; i < stages.Count; i++)
{
var s = await ScenarioAsync(fx, i + 1);
var crashed = false;
var at = stages[i];
handler.BeforeStage = stage =>
{
if (stage != at || crashed) return Task.CompletedTask;
crashed = true;
throw new IOException($"crash before {at}");
};
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.NotNull(await fx.Db.Sequences.GetAsync(s.Drop.Id));
Assert.Equal(CopyState.Candidate, (await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct))!.State);
await engine.ResumeAsync(job.Id, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var b = 100 * (i + 1);
Assert.True((await HashesAsync(fx, s.Keep.Id)).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
Assert.Null(await fx.Db.Sequences.GetAsync(s.Drop.Id));
Assert.Equal(s.Keep.Id, (await fx.Db.Renders.GetAsync(s.Render.Id))!.SequenceId);
}
// Each project copied once, whatever stage the run stopped at.
Assert.Equal(stages.Count, projects.Copied.Count);
}
[Test]
public async Task APhotoOfTheOtherThatChangedOnDiskFailsTheVerifyAndNothingGoesToTheTrash()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var shared = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).First(f => f.Sha256 == Sha(1, 0));
File.WriteAllBytes(fx.Paths.FromRelative(shared.RelativePath), Photos.Jpeg(Start, 77));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains(shared.FileName, failed.Error!);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
Assert.Equal(CopyState.Candidate, (await new FoundCopyStore(fx.Db).GetAsync(copy.Id, Ct))!.State);
}
[Test]
public async Task AnUnhashedPhotoThatIsAnotherPhotoAfterAllIsKeptAsASecondVersion()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(2, 0, "IMG_0001.JPG")]);
// Neither hashed, the same size and time: compared as the same photo.
var access = new DbAccess(fx.Db.Database);
await access.ExecuteAsync("UPDATE frames SET sha256 = NULL, size = 1000 WHERE sequence_id = $1 OR sequence_id = $2", [keep.Id, drop.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Photos);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var frames = await fx.Db.Frames.ListBySequenceAsync(keep.Id);
Assert.Count(2, frames);
Assert.Contains(Sha(2, 0), frames.Select(f => f.Sha256));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task ADamagedPhotoOfTheKeptOneIsRepairedFromTheOther()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var damaged = (await fx.Db.Frames.ListBySequenceAsync(keep.Id)).First(f => f.Sha256 == Sha(2, 10_000));
File.WriteAllBytes(fx.Paths.FromRelative(damaged.RelativePath), [1, 2, 3]);
await fx.Db.Frames.UpdateFlagsAsync([damaged.Id], FrameFlags.Corrupt, FrameFlags.None, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, Sha(2, 10_000))], plan.Photos.Select(p => (p.Action, p.Sha256!)));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var frames = await fx.Db.Frames.ListBySequenceAsync(keep.Id);
Assert.Count(2, frames);
var repaired = frames.Single(f => f.Sha256 == Sha(2, 10_000));
Assert.Equal(Sha(2, 10_000), Sha(File.ReadAllBytes(fx.Paths.FromRelative(repaired.RelativePath))));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task APhotoAddedToTheOtherOrDeletedFromTheKeptOneMeanwhileKeepsTheOtherOutOfTheTrash()
{
var fx = await FixtureAsync();
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
// A photo added to the other after the verify.
var keep = await SequenceAsync(fx, "keep-a", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop-a", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
handler.BeforeStage = async stage =>
{
if (stage != MergeStage.Trash) return;
var (path, size, sha) = fx.Stage(Start.AddSeconds(40), 3);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, drop.Id, null, null, path, "IMG_0003.JPG", size, sha, null, null), fx.Ct);
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
// A verified photo deleted from the kept one before the trash.
keep = await SequenceAsync(fx, "keep-b", [(11, 0, "IMG_0001.JPG")]);
drop = await SequenceAsync(fx, "drop-b", [(11, 0, "IMG_0001.JPG"), (12, 10_000, "IMG_0002.JPG")]);
copy = await PairAsync(fx, keep.Id, drop.Id);
var keepId = keep.Id;
Guid? running = null;
handler.BeforeStage = async stage =>
{
if (stage != MergeStage.Trash) return;
// Another hand deletes it (the job's own hold does not count, so the deletion is not refused as busy).
var gone = (await fx.Db.Frames.ListBySequenceAsync(keepId)).First(f => f.Sha256 == Sha(12, 10_000));
var trash = await fx.Service.FrameTrash.PlanDeleteAsync(keepId, [gone.Id], null, fx.Ct, running);
await fx.Service.FrameTrash.DeleteAsync(trash, null, null, fx.Ct, running);
};
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
running = job.Id;
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains(keep.DisplayTitle, failed.Error!);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task UnsentWorkspaceWorkStopsTheTrashAndTheTrashStepIsNotBusyWithItsOwnJob()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var workspace = new FakeWorkspace();
var (engine, handler) = Engine(fx, merger, new FakeProjects(), workspace);
await engine.StartAsync(Ct);
handler.BeforeStage = stage =>
{
if (stage == MergeStage.Trash) workspace.Unsent.Add(drop.Id);
return Task.CompletedTask;
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
// The photos are in the kept one already.
Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(1, 0), Sha(2, 10_000)]));
handler.BeforeStage = null;
workspace.Unsent.Clear();
await engine.ResumeAsync(job.Id, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task ACancelReturnsThePairToItsStateAndLeavesTheOtherAlone()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
Guid? id = null;
handler.BeforeStage = async stage =>
{
if (stage == MergeStage.Verify) await engine.CancelAsync(id!.Value, CancellationToken.None);
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var store = new FoundCopyStore(fx.Db);
var job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
id = job.Id;
await fx.WaitForAsync(job.Id, JobState.Cancelled);
var until = DateTime.UtcNow.AddSeconds(10);
while (File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)) && DateTime.UtcNow < until) await Task.Delay(20, Ct);
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
var pair = (await store.GetAsync(copy.Id, Ct))!;
Assert.Equal(CopyState.Candidate, pair.State);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
// Planned and merged again from the start.
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
handler.BeforeStage = null;
job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AJobWhosePlanMovedSinceTheDialogRefusesAndChangesNothing()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await SequenceAsync(fx, "drop", [(3, 20_000, "IMG_0003.JPG")]); // the session key files it into the other
await engine.StartAsync(Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.Count(1, await fx.Db.Frames.ListBySequenceAsync(keep.Id));
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
}
private sealed class FakeProjects : IProjectCopier
{
public List<(Guid Source, string Name)> Copied { get; } = [];
public async Task<SequenceProject> CopyAsync(Chronos.Data.Projects.IProjectHome from, SequenceProject source, Chronos.Data.Projects.IProjectHome to,
string name, TransplantStep? journal, Func<TransplantStep, ValueTask> save, CancellationToken ct)
{
await save(new TransplantStep(name, DateTimeOffset.UtcNow, null, null, 0, []));
lock (Copied) Copied.Add((source.Id, name));
return source with { Id = Guid.CreateVersion7(), Name = name };
}
}
private sealed class FakeWorkspace : IWorkspaceGuard
{
public HashSet<Guid> Unsent { get; } = [];
public Task<bool> HasUnsentAsync(Guid sequenceId, CancellationToken ct) => Task.FromResult(Unsent.Contains(sequenceId));
}
}
public sealed class SqliteSequenceMergerTests(TestContext context) : SequenceMergerTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresSequenceMergerTests(TestContext context) : SequenceMergerTests(context)
{
protected override Backend Backend => Backend.Postgres;
}