Files
Chronos/tests/Chronos.Data.Tests/Archiving/SequenceMergerTests.cs
T
Alby96andClaude Opus 5.5 2c06e7ac2b fix(copies): close the fourth verification round of owner review 6
The fourth verification round of the matches work found 18 defects in
the merge, Integrate and Matches surfaces. This commit fixes them:

- An Integrate job now carries the planning account's id, so its re-plan
  decides ownership the way the dialog did.
- A merge skips only a project the kept sequence holds a real copy of
  (name, offset head and history content) and re-points its renders to
  that copy.
- A merge runs its trash checks before the Links stage and records an
  undo, so an abandoned merge gives the other sequence back its link,
  session keys and pairs.
- The metered hold adopts automatic transfers that were paused before a
  restart.
- Apply in the Compare dialog re-plans when the plan is pending or stale.
- The Library's Two versions... lists the second versions a Union merge
  leaves, each with Drop... (dry run, confirmation, verified partner).
  A merge job's Open shows that list.
- The merge's typed confirmation is the invariant count, and the prompt
  names it in all 38 languages.
- Italian text uses Principale and Gemello. The Main project is named
  through WorkspaceText in the Projects part and in the twin tooltip.
- The session keys tooltip is split into two keys, and To trash... and
  Delete on server... say why they are disabled.
- The capture restores the editor account before the Matches scenes.
- Upload again follows SyncHost.CanSendSync (Upload with Edit).
- Matches rows trim archive names in the middle on a rebalanced grid;
  the Projects part rows trim, with the full text in the tooltip.
- The sample project targets use ProjectTransplant.FreeName.

DECISIONS, CHANGELOG, STATUS 8.7, the request doc and the 16 changed
capture baselines are updated.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 19:20:30 +01:00

1009 lines
58 KiB
C#

using System.Security.Cryptography;
using Chronos.Core.Archiving;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Copies;
using Chronos.Data.Jobs;
using Chronos.Data.Repositories;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>
/// Merging two local copies of one time-lapse (owner review 6, #73; design §5d, §11 Merge): the dry run (kept sequence,
/// photos copied, last copies, session keys, links, blocks, fingerprint) and the job that applies it, verifies every photo
/// of the other in the kept one and only then moves the other to its archive's trash.
/// </summary>
[Category("archive")]
public abstract class SequenceMergerTests(TestContext context) : DatabaseTest(context)
{
private const string Nas = "nas.local";
private static readonly CopyPlanContext NoContext = PlanContext();
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
private static CopyPlanContext PlanContext(params Guid[] originals) =>
new("PC", null, false, false, false, false, false, false, null, new HashSet<Guid>(), originals.ToHashSet());
private static SequenceMerger Merger(ArchiveFixture fx) => new(fx.Db, fx.Service, new CopySides(fx.Db));
/// <summary>A sequence uploaded photo by photo into <paramref name="archive"/>: (seed, capture offset in ms, file name).</summary>
private static async Task<Sequence> SequenceAsync(ArchiveFixture fx, string session, IReadOnlyList<(int Seed, int Ms, string Name)> photos, Archive? archive = null)
{
Guid? id = null;
var staging = Path.Combine(new ArchivePaths((archive ?? fx.Archive).RootPath).StagingDirectory, "uploads");
Directory.CreateDirectory(staging);
foreach (var (seed, ms, name) in photos)
{
var bytes = Photos.Jpeg(Start.AddMilliseconds(ms), seed);
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
File.WriteAllBytes(path, bytes);
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest((archive ?? fx.Archive).Id, id, null, id is null ? session : null, path, name,
bytes.Length, Sha(bytes), null, null), fx.Ct);
id = result.SequenceId;
}
return (await fx.Db.Sequences.GetAsync(id!.Value))!;
}
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private static string Sha(int seed, int ms) => Sha(Photos.Jpeg(Start.AddMilliseconds(ms), seed));
private static async Task<FoundCopy> PairAsync(ArchiveFixture fx, Guid a, Guid b) =>
await new FoundCopyStore(fx.Db).UpsertFoundAsync(new FoundCopy
{
Id = Guid.NewGuid(),
Kind = CopyKind.LocalLocal,
LocalId = a,
OtherId = b,
Evidence = CopyEvidence.Hashes,
Fingerprint = "f",
}, fx.Ct);
/// <summary>Choices with every detail line on the kept side but those in <paramref name="other"/>.</summary>
private static async Task<MergeChoices> ChoicesAsync(ArchiveFixture fx, Guid keep, Guid drop, CopyPhotoPolicy policy,
IReadOnlyDictionary<string, PhotoChoice>? perPhoto = null, params string[] other)
{
var keepRow = (await fx.Db.Sequences.GetAsync(keep))!;
var dropRow = (await fx.Db.Sequences.GetAsync(drop))!;
var lines = CopyPayload.Lines(SyncPayload.Of(keepRow), await SequencePayloadWriter.PhotosAsync(fx.Db, keep, null, fx.Ct),
SyncPayload.Of(dropRow), await SequencePayloadWriter.PhotosAsync(fx.Db, drop, null, fx.Ct));
var details = lines.Select(l => l.Path).Distinct().ToDictionary(p => p, p => other.Contains(p) ? CopySide.Other : CopySide.ThisPc);
return new MergeChoices(policy, perPhoto ?? new Dictionary<string, PhotoChoice>(), details);
}
private static async Task<Sequence> TitledAsync(ArchiveFixture fx, Sequence sequence, string title)
{
var result = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Title = title }, null, fx.Ct);
return result.Sequence!;
}
private static Task<Job> RunningAsync(ArchiveFixture fx, JobKind kind, params Guid[] targets) => EnqueuedAsync(fx, new Job
{
Id = Guid.CreateVersion7(),
Kind = kind,
TargetIds = targets,
State = JobState.Running,
NodeId = "test#0",
Created = DateTimeOffset.UtcNow,
Started = DateTimeOffset.UtcNow,
});
private static async Task<Job> EnqueuedAsync(ArchiveFixture fx, Job job)
{
await fx.Db.Jobs.EnqueueAsync(job);
return job;
}
private (JobEngine Engine, MergeJobHandler Handler) Engine(ArchiveFixture fx, SequenceMerger merger, FakeProjects projects, FakeWorkspace workspace)
{
var handler = (MergeJobHandler)SequenceMerger.CreateHandler(fx.Db, merger, fx.Service, projects, workspace);
var engine = Own(new JobEngine(fx.Db, [.. fx.Service.CreateJobHandlers(), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }));
return (engine, handler);
}
private static async Task<IReadOnlySet<string>> HashesAsync(ArchiveFixture fx, Guid sequenceId) =>
(await fx.Db.Frames.ListBySequenceAsync(sequenceId)).Select(f => f.Sha256!).ToHashSet(StringComparer.Ordinal);
/// <summary>
/// A pair with something in every stage: shared photo 1, photo 2 kept only, photo 3 only in the other, photo 4 different at
/// one instant (same name), a render inside the other's folder, a project, a session key, a link and a title.
/// </summary>
private static async Task<Scenario> ScenarioAsync(ArchiveFixture fx, int n)
{
var b = 100 * n;
var keep = await SequenceAsync(fx, $"keep-{n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 2, 10_000, "IMG_0002.JPG"), (b + 4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, $"drop-{n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 3, 20_000, "IMG_0003.JPG"), (b + 5, 30_250, "IMG_0004.JPG")]);
drop = await TitledAsync(fx, drop, $"Harbour {n}");
var exports = fx.Paths.Sequence(drop.Folder).Exports;
Directory.CreateDirectory(exports);
var clip = Path.Combine(exports, "clip.mp4");
var bytes = RandomNumberGenerator.GetBytes(4096);
File.WriteAllBytes(clip, bytes);
var render = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = drop.Id,
PresetName = "1080p",
OutputPath = fx.Paths.ToRelative(clip),
Bytes = bytes.Length,
Sha256 = Sha(bytes),
Created = DateTimeOffset.UtcNow,
};
await fx.Db.Renders.InsertAsync(render, fx.Ct);
var project = await fx.Service.Projects.CreateAsync(drop.Id, "Night cut", cancellationToken: fx.Ct);
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = drop.DisplayTitle }, fx.Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var differs = Sha(b + 4, 30_250);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.OtherWins, null, SyncRules.MetadataPath + "title");
return new Scenario(keep, drop, copy, render, project, choices, differs);
}
private sealed record Scenario(Sequence Keep, Sequence Drop, FoundCopy Copy, Render Render, SequenceProject Project, MergeChoices Choices, string KeepLoser);
// ---- the dry run ----
[Test]
public async Task TheKeptOneIsPreselectedOnlyWhenOneIsTheLinkedOriginalsCopy()
{
var fx = await FixtureAsync();
var a = await SequenceAsync(fx, "a", [(1, 0, "IMG_0001.JPG")]);
var b = await SequenceAsync(fx, "b", [(1, 0, "IMG_0001.JPG")]);
var copy = await PairAsync(fx, a.Id, b.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, a.Id, b.Id, CopyPhotoPolicy.Union);
// Nothing linked: the user chooses; the plan keeps the one the dialog does not name.
var plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
Assert.Equal((a.Id, b.Id), (plan.KeepId, plan.DropId));
// One linked, in a server's originals archive: preselected (also when the dialog names it as the other).
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = b.Id, Title = "b" }, Ct);
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
Assert.DoesNotContain(CopyBlock.KeptUnchosen, plan.Blocks);
Assert.Equal((b.Id, a.Id), (plan.KeepId, plan.DropId));
// Linked but not in an originals archive: asked.
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
// Both linked: asked again.
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = "other.local", RemoteId = Guid.CreateVersion7(), LocalId = a.Id, Title = "a" }, Ct);
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
// Chosen: never asked.
plan = await merger.PlanAsync(copy.Id, a.Id, b.Id, choices, NoContext, Ct);
Assert.Empty(plan.Blocks);
Assert.Equal((a.Id, b.Id), (plan.KeepId, plan.DropId));
}
[Test]
public async Task OnlyThereIsAlwaysCopiedAndThePolicyDecidesDifferentPhotosOnly()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (3, 20_000, "IMG_0003.JPG"), (5, 30_250, "IMG_0004.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (only, theirs, mine) = (Sha(3, 20_000), Sha(5, 30_250), Sha(4, 30_250));
// Choose each with nothing chosen: the only-there photo still comes; the other's different version is a last copy.
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ChoosePerPhoto), NoContext, Ct);
Assert.Empty(plan.Blocks);
Assert.SequenceEqual([(PhotoAction.Fetch, only)], plan.Photos.Select(p => (p.Action, p.Sha256!)));
Assert.SequenceEqual([(CopySide.Other, theirs)], plan.LastCopies.Select(l => (l.Side, l.Key)));
// Choose each, Take the other's: its version comes, the kept one's goes (a last copy of this PC).
var key = mine;
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id,
await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ChoosePerPhoto, new Dictionary<string, PhotoChoice> { [key] = PhotoChoice.TakeThere }), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, only), (PhotoAction.FetchVersion, theirs), (PhotoAction.TrashHere, mine)],
plan.Photos.Select(p => (p.Action, p.Sha256!)).OrderBy(p => p.Action));
Assert.SequenceEqual([(CopySide.ThisPc, mine)], plan.LastCopies.Select(l => (l.Side, l.Key)));
// Union keeps both versions: nothing is a last copy.
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, only), (PhotoAction.FetchVersion, theirs)], plan.Photos.Select(p => (p.Action, p.Sha256!)).OrderBy(p => p.Action));
Assert.Empty(plan.LastCopies);
Assert.Equal(plan.Photos.Sum(p => p.Bytes), plan.CopyBytes);
// The kept one wins: nothing of the different photo moves; the other's version is a last copy.
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ThisPcWins), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, only)], plan.Photos.Select(p => (p.Action, p.Sha256!)));
Assert.SequenceEqual([theirs], plan.LastCopies.Select(l => l.Key));
}
[Test]
public async Task APhotoTombstonedInTheKeptOneIsALastCopyUnlessAnotherSequenceHoldsIt()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var gone = Sha(2, 10_000);
await fx.Db.FrameRemovals.SyncSequenceAsync(keep.Id, [new FrameRemoval(keep.Id, gone, null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Empty(plan.Photos);
Assert.SequenceEqual([(CopySide.Other, gone)], plan.LastCopies.Select(l => (l.Side, l.Key)));
await SequenceAsync(fx, "third", [(2, 10_000, "IMG_0002.JPG")]);
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Empty(plan.LastCopies);
}
[Test]
public async Task SessionKeysFollowTheKeptOneButOneItsArchiveMapsElsewhereAndLinksArePlanned()
{
var fx = await FixtureAsync();
var second = await fx.Service.AdoptAsync(Context.CreateTempDirectory("second"), "Second", Ct);
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var third = await SequenceAsync(fx, "card-7", [(9, 0, "IMG_0009.JPG")]);
var drop = await SequenceAsync(fx, "card-7", [(1, 0, "IMG_0001.JPG")], second);
await fx.Db.Sequences.MapSessionKeyAsync(second.Id, "card-8", drop.Id, Ct);
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = "d" }, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var plan = await Merger(fx).PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Blocks);
Assert.SequenceEqual(["card-8"], plan.SessionKeys);
Assert.SequenceEqual(["card-7"], plan.SkippedSessionKeys);
Assert.SequenceEqual([Nas], plan.Links);
Assert.Equal(third.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "card-7", Ct));
}
[Test]
public async Task AnotherJobOnEitherSequenceAndBothLinkedToOneServerBlockThePlan()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = new SequenceMerger(fx.Db, fx.Service, new CopySides(fx.Db)) { ServerNames = key => key == Nas ? "Studio NAS" : null };
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
var render = await RunningAsync(fx, JobKind.Render, keep.Id);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.Busy, plan.Blocks);
await fx.Db.Jobs.SetStateAsync(render.Id, JobState.Done, null, Ct);
Assert.Empty((await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks);
var links = new SyncLinkStore(fx.Db);
await links.UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = keep.Id, Title = "k" }, Ct);
await links.UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = "d" }, Ct);
var both = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.BothLinked, both.Blocks);
// Named by the server's name, never its key.
Assert.Equal("Studio NAS", both.BothLinkedServer);
Assert.SequenceEqual(["Studio NAS"], both.LinkNames);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.BothLinked", "Studio NAS"), await merger.BlockMessageAsync(both with { Blocks = [CopyBlock.BothLinked] }, Ct));
var unsent = PlanContext() with { UnsentWorkspaceSequences = new HashSet<Guid> { drop.Id } };
Assert.Contains(CopyBlock.UnsentWorkspace, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, unsent, Ct)).Blocks);
// A detail that differs and is not chosen.
await TitledAsync(fx, drop, "Other title");
var unchosen = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.DetailsUnchosen, unchosen.Blocks);
Assert.Null(unchosen.Payload);
}
[Test]
public async Task TheFingerprintMovesWithADetailOfTheKeptOneNotWithAnImportElsewhere()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
var first = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
await SequenceAsync(fx, "elsewhere", [(7, 0, "IMG_0007.JPG")]);
Assert.Equal(first.Fingerprint, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Fingerprint);
await fx.Db.Sequences.EditAsync(keep.Id, new SequenceEdit { Rating = 4 }, null, Ct);
Assert.NotEqual(first.Fingerprint, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Fingerprint);
}
// ---- the job ----
[Test]
public async Task AMergeCopiesVerifiesAndMovesEverythingThenTrashesTheOther()
{
var fx = await FixtureAsync();
var s = await ScenarioAsync(fx, 0);
var merger = Merger(fx);
var projects = new FakeProjects();
var (engine, _) = Engine(fx, merger, projects, new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
Assert.Empty(plan.Blocks);
Assert.SequenceEqual([(CopySide.ThisPc, s.KeepLoser)], plan.LastCopies.Select(l => (l.Side, l.Key)));
Assert.SequenceEqual([s.Render.Id], plan.Renders.Select(r => r.RenderId));
Assert.SequenceEqual([("Night cut", "Night cut")], plan.Projects.Select(p => (p.SourceName, p.TargetName)));
Assert.Equal(1, plan.DetailChanges);
Assert.Equal("Harbour 0", plan.Payload!.Title);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
Assert.Equal(CopyState.Integrating, (await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct))!.State);
await fx.WaitForAsync(job.Id, JobState.Done);
// Photos: shared, kept only, only in the other and the other's winning version; the kept one's losing version went.
Assert.True((await HashesAsync(fx, s.Keep.Id)).SetEquals([Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(5, 30_250)]));
var keep = (await fx.Db.Sequences.GetAsync(s.Keep.Id))!;
Assert.Equal("Harbour 0", keep.Title);
// The render keeps its id, its file copied into the kept sequence.
var render = (await fx.Db.Renders.GetAsync(s.Render.Id))!;
Assert.Equal(s.Keep.Id, render.SequenceId);
Assert.Equal(s.Render.Sha256, Sha(File.ReadAllBytes(fx.Paths.FromRelative(render.OutputPath))));
Assert.Equal(1, keep.RenderedCount);
Assert.SequenceEqual([("Night cut", s.Project.Id)], projects.Copied.Select(c => (c.Name, c.Source)));
// Keys, links and the pair.
Assert.Equal(s.Keep.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "drop-0", Ct));
Assert.Contains("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(keep.Folder).ManifestPath).SessionKeys);
Assert.Equal(s.Keep.Id, (await new SyncLinkStore(fx.Db).ListAsync(Nas, Ct)).Single().LocalId);
Assert.Null(await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
// The other is in its archive's trash, restorable, without the keys it gave away.
Assert.Null(await fx.Db.Sequences.GetAsync(s.Drop.Id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, Ct)).Single(e => e.SequenceId == s.Drop.Id && e.Kind == TrashKind.Sequence);
var restored = await fx.Service.RestoreAsync(fx.Archive.Id, entry.Name, Ct);
Assert.Equal(s.Drop.Id, restored.Id);
Assert.Equal(3, restored.FrameCount);
Assert.DoesNotContain("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(restored.Folder).ManifestPath).SessionKeys);
}
[Test]
public async Task ACrashAtEachStageResumesWhereItStopped()
{
var fx = await FixtureAsync();
var merger = Merger(fx);
var projects = new FakeProjects();
var (engine, handler) = Engine(fx, merger, projects, new FakeWorkspace());
await engine.StartAsync(Ct);
var stages = Enum.GetValues<MergeStage>().Where(s => s != MergeStage.Done).ToList();
for (var i = 0; i < stages.Count; i++)
{
var s = await ScenarioAsync(fx, i + 1);
var crashed = false;
var at = stages[i];
handler.BeforeStage = stage =>
{
if (stage != at || crashed) return Task.CompletedTask;
crashed = true;
throw new IOException($"crash before {at}");
};
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.NotNull(await fx.Db.Sequences.GetAsync(s.Drop.Id));
Assert.Equal(CopyState.Candidate, (await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct))!.State);
await engine.ResumeAsync(job.Id, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var b = 100 * (i + 1);
Assert.True((await HashesAsync(fx, s.Keep.Id)).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
Assert.Null(await fx.Db.Sequences.GetAsync(s.Drop.Id));
Assert.Equal(s.Keep.Id, (await fx.Db.Renders.GetAsync(s.Render.Id))!.SequenceId);
}
// Each project copied once, whatever stage the run stopped at.
Assert.Equal(stages.Count, projects.Copied.Count);
}
[Test]
public async Task APhotoOfTheOtherThatChangedOnDiskFailsTheVerifyAndNothingGoesToTheTrash()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var shared = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).First(f => f.Sha256 == Sha(1, 0));
File.WriteAllBytes(fx.Paths.FromRelative(shared.RelativePath), Photos.Jpeg(Start, 77));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains(shared.FileName, failed.Error!);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
Assert.Equal(CopyState.Candidate, (await new FoundCopyStore(fx.Db).GetAsync(copy.Id, Ct))!.State);
}
[Test]
public async Task AnUnhashedPhotoThatIsAnotherPhotoAfterAllIsKeptAsASecondVersion()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(2, 0, "IMG_0001.JPG")]);
// Neither hashed, the same size and time: compared as the same photo.
var access = new DbAccess(fx.Db.Database);
await access.ExecuteAsync("UPDATE frames SET sha256 = NULL, size = 1000 WHERE sequence_id = $1 OR sequence_id = $2", [keep.Id, drop.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Photos);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var frames = await fx.Db.Frames.ListBySequenceAsync(keep.Id);
Assert.Count(2, frames);
Assert.Contains(Sha(2, 0), frames.Select(f => f.Sha256));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task ADamagedPhotoOfTheKeptOneIsRepairedFromTheOther()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var damaged = (await fx.Db.Frames.ListBySequenceAsync(keep.Id)).First(f => f.Sha256 == Sha(2, 10_000));
File.WriteAllBytes(fx.Paths.FromRelative(damaged.RelativePath), [1, 2, 3]);
await fx.Db.Frames.UpdateFlagsAsync([damaged.Id], FrameFlags.Corrupt, FrameFlags.None, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, Sha(2, 10_000))], plan.Photos.Select(p => (p.Action, p.Sha256!)));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var frames = await fx.Db.Frames.ListBySequenceAsync(keep.Id);
Assert.Count(2, frames);
var repaired = frames.Single(f => f.Sha256 == Sha(2, 10_000));
Assert.Equal(Sha(2, 10_000), Sha(File.ReadAllBytes(fx.Paths.FromRelative(repaired.RelativePath))));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AChangedPhotoOfTheOtherNothingElseHoldsBlocksTheMerge()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
// A photo of the other edited after import: a verify flags it Corrupt (changed), its file holds the only copy.
var edited = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).First(f => f.Sha256 == Sha(2, 10_000));
var file = fx.Paths.FromRelative(edited.RelativePath);
File.WriteAllBytes(file, Photos.Jpeg(Start.AddMilliseconds(10_000), 88));
await fx.Db.Frames.UpdateFlagsAsync([edited.Id], FrameFlags.Corrupt, FrameFlags.None, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.DamagedPhotos, plan.Blocks);
Assert.False(plan.CanApply);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.DamagedPhotos", "Drop"), SequenceMerger.BlockText(plan, CopyBlock.DamagedPhotos, "Keep", "Drop"));
// Another sequence of this index holds that content too: nothing would be lost, the merge may go on.
await SequenceAsync(fx, "elsewhere", [(88, 10_000, "IMG_0002.JPG")]);
Assert.DoesNotContain(CopyBlock.DamagedPhotos, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks);
}
[Test]
public async Task APhotoFlaggedMissingWhoseFileIsThereBlocksTheMerge()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
// Flagged Missing by an earlier verify (a drive away then), its file back now: the merge would not copy it.
var missing = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).First(f => f.Sha256 == Sha(2, 10_000));
await fx.Db.Frames.UpdateFlagsAsync([missing.Id], FrameFlags.Missing, FrameFlags.None, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var plan = await Merger(fx).PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Contains(CopyBlock.DamagedPhotos, plan.Blocks);
Assert.False(plan.CanApply);
}
[Test]
public async Task WorkSentHomeIntoTheOtherAfterAStoppedTrashAsksForANewPlan()
{
var fx = await FixtureAsync();
var merger = Merger(fx);
var workspace = new FakeWorkspace();
var (engine, handler) = Engine(fx, merger, new FakeProjects(), workspace);
await engine.StartAsync(Ct);
var store = new FoundCopyStore(fx.Db);
// Details sent home (a settings version), then a project made in the other (a new head), each after the stop.
Func<Sequence, Task>[] sends =
[
drop => fx.Db.Sequences.SetTagsAsync(drop.Id, ["sent"], Ct),
drop => fx.Service.Projects.CreateAsync(drop.Id, "Sent cut", cancellationToken: fx.Ct),
];
for (var i = 0; i < sends.Length; i++)
{
var keep = await SequenceAsync(fx, $"keep-{i}", [(10 * i + 1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, $"drop-{i}", [(10 * i + 1, 0, "IMG_0001.JPG"), (10 * i + 2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
handler.BeforeStage = stage =>
{
if (stage == MergeStage.Trash) workspace.Unsent.Add(drop.Id);
return Task.CompletedTask;
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
// The workspace sends its work home into the other, then Retry.
await sends[i](drop);
handler.BeforeStage = null;
workspace.Unsent.Clear();
await engine.ResumeAsync(job.Id, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.Changed", drop.DisplayTitle), failed.Error);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)), "the next run plans again");
}
}
[Test]
public async Task APhotoSettledThroughAThirdSequenceThatIsGoneByTheTrashKeepsTheOther()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (5, 30_250, "IMG_0004.JPG")]);
// A third sequence holds the other's losing version: not a last copy when planned.
var third = await SequenceAsync(fx, "third", [(5, 30_250, "IMG_0004.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ThisPcWins), NoContext, Ct);
Assert.Empty(plan.LastCopies);
handler.BeforeStage = async stage =>
{
if (stage == MergeStage.Trash) await fx.Db.Sequences.DeleteAsync(third.Id, Ct);
};
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
// Now the other holds its only copy: nothing goes to the trash, and the next plan lists it as a last copy.
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.Changed", drop.DisplayTitle), failed.Error);
Assert.Contains(Sha(5, 30_250), await HashesAsync(fx, drop.Id));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
var again = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ThisPcWins), NoContext, Ct);
Assert.Contains(again.LastCopies, l => l.Key == Sha(5, 30_250));
}
[Test]
public async Task APhotoAddedToTheOtherOrDeletedFromTheKeptOneMeanwhileKeepsTheOtherOutOfTheTrash()
{
var fx = await FixtureAsync();
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
// A photo added to the other after the verify.
var keep = await SequenceAsync(fx, "keep-a", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop-a", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
handler.BeforeStage = async stage =>
{
if (stage != MergeStage.Trash) return;
var (path, size, sha) = fx.Stage(Start.AddSeconds(40), 3);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, drop.Id, null, null, path, "IMG_0003.JPG", size, sha, null, null), fx.Ct);
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
// A verified photo deleted from the kept one before the trash.
keep = await SequenceAsync(fx, "keep-b", [(11, 0, "IMG_0001.JPG")]);
drop = await SequenceAsync(fx, "drop-b", [(11, 0, "IMG_0001.JPG"), (12, 10_000, "IMG_0002.JPG")]);
copy = await PairAsync(fx, keep.Id, drop.Id);
var keepId = keep.Id;
Guid? running = null;
handler.BeforeStage = async stage =>
{
if (stage != MergeStage.Trash) return;
// Another hand deletes it (the job's own hold does not count, so the deletion is not refused as busy).
var gone = (await fx.Db.Frames.ListBySequenceAsync(keepId)).First(f => f.Sha256 == Sha(12, 10_000));
var trash = await fx.Service.FrameTrash.PlanDeleteAsync(keepId, [gone.Id], null, fx.Ct, running);
await fx.Service.FrameTrash.DeleteAsync(trash, null, null, fx.Ct, running);
};
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
running = job.Id;
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains(keep.DisplayTitle, failed.Error!);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task UnsentWorkspaceWorkStopsTheTrashAndTheTrashStepIsNotBusyWithItsOwnJob()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var workspace = new FakeWorkspace();
var (engine, handler) = Engine(fx, merger, new FakeProjects(), workspace);
await engine.StartAsync(Ct);
handler.BeforeStage = stage =>
{
if (stage == MergeStage.Trash) workspace.Unsent.Add(drop.Id);
return Task.CompletedTask;
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
// The photos are in the kept one already.
Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(1, 0), Sha(2, 10_000)]));
handler.BeforeStage = null;
workspace.Unsent.Clear();
await engine.ResumeAsync(job.Id, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AUnionMergeLeavesItsSecondVersionsListedInTheKeptOne()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (5, 30_250, "IMG_0004.JPG")]);
// The other deleted photo 2, which the kept one still holds.
await fx.Db.FrameRemovals.SyncSequenceAsync(drop.Id, [new FrameRemoval(drop.Id, Sha(2, 10_000), null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Equal(1, plan.DeletedInOther);
Assert.Empty(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
// The pair is gone with the other; the kept one still lists the photo it holds in two versions.
Assert.Null(await new FoundCopyStore(fx.Db).GetAsync(copy.Id, Ct));
var versions = (await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct)).Single();
Assert.True(new[] { versions.FirstSha256, versions.SecondSha256 }.ToHashSet().SetEquals([Sha(4, 30_250), Sha(5, 30_250)]));
}
[Test]
public async Task ARetryAfterKeepApartRefusesAndTrashesNothing()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var workspace = new FakeWorkspace();
var (engine, handler) = Engine(fx, merger, new FakeProjects(), workspace);
await engine.StartAsync(Ct);
// The trash stage stops on unsent workspace work: the pair is still the failed job's.
handler.BeforeStage = stage =>
{
if (stage == MergeStage.Trash) workspace.Unsent.Add(drop.Id);
return Task.CompletedTask;
};
var store = new FoundCopyStore(fx.Db);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
// Kept apart, the work sent, then Retry: the journal is stale, the other stays where it is.
Assert.True(await store.SetStateAsync(copy.Id, null, CopyState.KeptApart, new CopyAnswer("f", 1, null, null, DateTimeOffset.UtcNow), Ct));
handler.BeforeStage = null;
workspace.Unsent.Clear();
await engine.ResumeAsync(job.Id, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
var still = await fx.Db.Sequences.GetAsync(drop.Id);
Assert.NotNull(still);
Assert.True(File.Exists(fx.Paths.Sequence(still!.Folder).ManifestPath));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
Assert.Equal(CopyState.KeptApart, (await store.GetAsync(copy.Id, Ct))!.State);
}
[Test]
public async Task ARetryAfterAnotherMergeOfThePairRefusesAndTrashesNothing()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
handler.BeforeStage = stage => stage == MergeStage.Verify ? throw new IOException("crash before Verify") : Task.CompletedTask;
var store = new FoundCopyStore(fx.Db);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
// A new merge keeping the other one takes the pair (and stops before its own trash).
handler.BeforeStage = stage => stage == MergeStage.Copy ? throw new IOException("crash before Copy") : Task.CompletedTask;
var swapped = await merger.PlanAsync(copy.Id, drop.Id, keep.Id, await ChoicesAsync(fx, drop.Id, keep.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var second = await merger.EnqueueAsync(engine, store, swapped, drop.DisplayTitle, Ct);
await fx.WaitForAsync(second.Id, JobState.Failed);
handler.BeforeStage = null;
await engine.ResumeAsync(job.Id, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.NotNull(await fx.Db.Sequences.GetAsync(keep.Id));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
// The other merge's journal is untouched.
Assert.True(File.Exists(MergeJournal.PathFor(fx.Paths, second.Id)));
}
[Test]
public async Task AVerifyThatCannotReadTheOtherStopsAndTheRetryVerifiesEverything()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(2, 0, "IMG_0001.JPG")]);
// Neither hashed, the same size and time: compared as the same photo, only the verify tells them apart.
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL, size = 1000 WHERE sequence_id = $1 OR sequence_id = $2",
[keep.Id, drop.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var folder = fx.Paths.Sequence(drop.Folder).Folder;
var aside = folder + "-away";
handler.BeforeStage = stage =>
{
if (stage == MergeStage.Verify && Directory.Exists(folder)) Directory.Move(folder, aside);
return Task.CompletedTask;
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.Count(1, await fx.Db.Frames.ListBySequenceAsync(keep.Id));
handler.BeforeStage = null;
Directory.Move(aside, folder);
await engine.ResumeAsync(job.Id, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
// The other photo was read and filed as a second version before the other went to the trash.
Assert.Contains(Sha(2, 0), (await fx.Db.Frames.ListBySequenceAsync(keep.Id)).Select(f => f.Sha256));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AnUnhashedPhotoTheKeptOneDeletedIsALastCopyNotACopy()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var gone = Sha(2, 10_000);
await fx.Db.FrameRemovals.SyncSequenceAsync(keep.Id, [new FrameRemoval(keep.Id, gone, null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct);
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1 AND file_name = $2", [drop.Id, "IMG_0002.JPG"], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Photos);
Assert.SequenceEqual([(CopySide.Other, gone)], plan.LastCopies.Select(l => (l.Side, l.Key)));
// Confirmed as a last copy: the merge goes through and does not file it in the kept one.
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.DoesNotContain(gone, await HashesAsync(fx, keep.Id));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task ACancelReturnsThePairToItsStateAndLeavesTheOtherAlone()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
Guid? id = null;
handler.BeforeStage = async stage =>
{
if (stage == MergeStage.Verify) await engine.CancelAsync(id!.Value, CancellationToken.None);
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var store = new FoundCopyStore(fx.Db);
var job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
id = job.Id;
await fx.WaitForAsync(job.Id, JobState.Cancelled);
var until = DateTime.UtcNow.AddSeconds(10);
while (File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)) && DateTime.UtcNow < until) await Task.Delay(20, Ct);
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
var pair = (await store.GetAsync(copy.Id, Ct))!;
Assert.Equal(CopyState.Candidate, pair.State);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
// Planned and merged again from the start.
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
handler.BeforeStage = null;
job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AMergeAbandonedAtTheTrashGivesTheOtherBackItsLinkAndSessionKeys()
{
var fx = await FixtureAsync();
var s = await ScenarioAsync(fx, 0);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
// Details saved into the other after the links stage: the trash abandons the merge.
handler.BeforeStage = stage => stage == MergeStage.Trash ? fx.Db.Sequences.SetTagsAsync(s.Drop.Id, ["edited"], Ct) : Task.CompletedTask;
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.Changed", s.Drop.DisplayTitle), failed.Error);
var drop = Assert.NotNull(await fx.Db.Sequences.GetAsync(s.Drop.Id));
var keep = (await fx.Db.Sequences.GetAsync(s.Keep.Id))!;
Assert.Equal(s.Drop.Id, (await new SyncLinkStore(fx.Db).ListAsync(Nas, Ct)).Single().LocalId);
Assert.Equal(s.Drop.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "drop-0", Ct));
Assert.Contains("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(drop.Folder).ManifestPath).SessionKeys);
Assert.DoesNotContain("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(keep.Folder).ManifestPath).SessionKeys);
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)), "the next run plans again");
}
[Test]
public async Task AProjectOfTheOtherWithTheHeadOfAnUnrelatedKeptProjectIsCopiedWithItsRenders()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG")]);
// Two untouched projects made from equal Mains: equal heads, unrelated projects.
var instagram = await fx.Service.Projects.CreateAsync(keep.Id, "Instagram", cancellationToken: fx.Ct);
var export = await fx.Service.Projects.CreateAsync(drop.Id, "4K export", cancellationToken: fx.Ct);
Assert.Equal((await fx.Service.Projects.HeadAsync(instagram.Id, Ct)).ContentHash, (await fx.Service.Projects.HeadAsync(export.Id, Ct)).ContentHash);
var render = await RenderOfAsync(fx, drop, export);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var handler = (MergeJobHandler)SequenceMerger.CreateHandler(fx.Db, merger, fx.Service, new Chronos.Data.Projects.ProjectTransplant(), new FakeWorkspace());
var engine = Own(new JobEngine(fx.Db, [.. fx.Service.CreateJobHandlers(), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }));
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.SequenceEqual([("4K export", "4K export")], plan.Projects.Select(p => (p.SourceName, p.TargetName)));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var copied = (await fx.Db.Projects.ListBySequenceAsync(keep.Id, includeDeleted: false, Ct)).Single(p => p.Name == "4K export");
var moved = (await fx.Db.Renders.GetAsync(render.Id))!;
Assert.Equal((keep.Id, (Guid?)copied.Id, copied.HeadVersionId), (moved.SequenceId, moved.ProjectId, moved.ProjectVersionId));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AProjectTheKeptOneHoldsACopyOfIsNotCopiedAgainAndItsRendersNameTheCopy()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG")]);
var cut = await fx.Service.Projects.CreateAsync(drop.Id, "Night cut", cancellationToken: fx.Ct);
var render = await RenderOfAsync(fx, drop, cut);
// An earlier run of the merge copied it before it stopped.
var held = await new Chronos.Data.Projects.ProjectTransplant().CopyAsync(new Chronos.Data.Projects.ArchiveProjectHome(fx.Db, fx.Service.Projects, drop.Id), cut,
new Chronos.Data.Projects.ArchiveProjectHome(fx.Db, fx.Service.Projects, keep.Id), "Night cut", null, _ => ValueTask.CompletedTask, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Projects);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var moved = (await fx.Db.Renders.GetAsync(render.Id))!;
var head = (await fx.Db.Projects.GetAsync(held.Id, Ct))!.HeadVersionId;
Assert.Equal((keep.Id, (Guid?)held.Id, head), (moved.SequenceId, moved.ProjectId, moved.ProjectVersionId));
}
/// <summary>A render of <paramref name="project"/>'s head, its file outside the sequence's folder.</summary>
private static async Task<Render> RenderOfAsync(ArchiveFixture fx, Sequence sequence, SequenceProject project)
{
var bytes = RandomNumberGenerator.GetBytes(2048);
var file = Path.Combine(fx.Paths.Root, $"{project.Id:N}.mp4");
await File.WriteAllBytesAsync(file, bytes, fx.Ct);
var render = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = sequence.Id,
PresetName = "1080p",
OutputPath = file,
Bytes = bytes.Length,
Sha256 = Sha(bytes),
Created = DateTimeOffset.UtcNow,
ProjectId = project.Id,
ProjectVersionId = project.HeadVersionId,
};
await fx.Db.Renders.InsertAsync(render, fx.Ct);
return render;
}
[Test]
public async Task AJobWhosePlanMovedSinceTheDialogRefusesAndChangesNothing()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await SequenceAsync(fx, "drop", [(3, 20_000, "IMG_0003.JPG")]); // the session key files it into the other
await engine.StartAsync(Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.Count(1, await fx.Db.Frames.ListBySequenceAsync(keep.Id));
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
}
private sealed class FakeProjects : IProjectCopier
{
public List<(Guid Source, string Name)> Copied { get; } = [];
public async Task<SequenceProject> CopyAsync(Chronos.Data.Projects.IProjectHome from, SequenceProject source, Chronos.Data.Projects.IProjectHome to,
string name, TransplantStep? journal, Func<TransplantStep, ValueTask> save, CancellationToken ct)
{
await save(new TransplantStep(name, DateTimeOffset.UtcNow, null, null, 0, []));
lock (Copied) Copied.Add((source.Id, name));
return source with { Id = Guid.CreateVersion7(), Name = name };
}
}
private sealed class FakeWorkspace : IWorkspaceGuard
{
public HashSet<Guid> Unsent { get; } = [];
public Task<bool> HasUnsentAsync(Guid sequenceId, CancellationToken ct) => Task.FromResult(Unsent.Contains(sequenceId));
}
}
public sealed class SqliteSequenceMergerTests(TestContext context) : SequenceMergerTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresSequenceMergerTests(TestContext context) : SequenceMergerTests(context)
{
protected override Backend Backend => Backend.Postgres;
}