Files
Alby96andClaude Opus 5.5 f33bcac32e
CI / Build and test (push) Successful in 13m44s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 16s
Release / Build, test and publish (push) Successful in 26m34s
CI / PostgreSQL 16 integration (push) Failing after 17m36s
CI / PostgreSQL 17 integration (push) Failing after 15m36s
CI / Docker image (push) Successful in 1m12s
Release / Server image (push) Failing after 15s
feat(copies): matches between this PC and the server, Integrate and merge (owner review 6, v1.2.0) (#74)
The same time-lapse on a server and on this PC under different ids, or twice on this PC, is recognised,
compared and settled as the owner asked in owner review 6.

Matches are found after each sync refresh, through the server's new photo lookup, and after the jobs that
change photos. The default is to ask every time. Server › Matches offers:
- Keep apart, which is remembered;
- Link only and Unlink;
- the Compare dialog: photos, details line by line, projects and the plan.

Integrate completes both sides. Photos default to Union, projects are copied across, and a losing version
goes to the trash only after a hash check. Afterwards new photos move by themselves, and deletions wait for
the user.

A merge of two local sequences verifies every photo before the other sequence goes to its archive's trash.
Upload all local skips suspected matches and flags them.

Migration 0012 adds the found_copies and copy_probes tables. Jobs 25 (Integrate) and 26 (MergeSequences) are
new, and so are the server capabilities photo-lookup and upload-into.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-11-18 15:20:29 +01:00

1670 lines
97 KiB
C#

using System.Security.Cryptography;
using Chronos.Core.Archiving;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Copies;
using Chronos.Data.Jobs;
using Chronos.Data.Repositories;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>
/// Merging two local copies of one time-lapse (owner review 6, #73; design §5d, §11 Merge): the dry run (kept sequence,
/// photos copied, last copies, session keys, links, blocks, fingerprint) and the job that applies it, verifies every photo
/// of the other in the kept one and only then moves the other to its archive's trash.
/// </summary>
[Category("archive")]
public abstract class SequenceMergerTests(TestContext context) : DatabaseTest(context)
{
private const string Nas = "nas.local";
private static readonly CopyPlanContext NoContext = PlanContext();
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
private static CopyPlanContext PlanContext(params Guid[] originals) =>
new("PC", null, false, false, false, false, false, false, null, new HashSet<Guid>(), originals.ToHashSet());
private static SequenceMerger Merger(ArchiveFixture fx) => new(fx.Db, fx.Service, new CopySides(fx.Db));
/// <summary>A sequence uploaded photo by photo into <paramref name="archive"/>: (seed, capture offset in ms, file name).</summary>
private static async Task<Sequence> SequenceAsync(ArchiveFixture fx, string session, IReadOnlyList<(int Seed, int Ms, string Name)> photos, Archive? archive = null)
{
Guid? id = null;
var staging = Path.Combine(new ArchivePaths((archive ?? fx.Archive).RootPath).StagingDirectory, "uploads");
Directory.CreateDirectory(staging);
foreach (var (seed, ms, name) in photos)
{
var bytes = Photos.Jpeg(Start.AddMilliseconds(ms), seed);
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
File.WriteAllBytes(path, bytes);
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest((archive ?? fx.Archive).Id, id, null, id is null ? session : null, path, name,
bytes.Length, Sha(bytes), null, null), fx.Ct);
id = result.SequenceId;
}
return (await fx.Db.Sequences.GetAsync(id!.Value))!;
}
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private static string Sha(int seed, int ms) => Sha(Photos.Jpeg(Start.AddMilliseconds(ms), seed));
private static async Task<FoundCopy> PairAsync(ArchiveFixture fx, Guid a, Guid b) =>
await new FoundCopyStore(fx.Db).UpsertFoundAsync(new FoundCopy
{
Id = Guid.NewGuid(),
Kind = CopyKind.LocalLocal,
LocalId = a,
OtherId = b,
Evidence = CopyEvidence.Hashes,
Fingerprint = "f",
}, fx.Ct);
/// <summary>Choices with every detail line on the kept side but those in <paramref name="other"/>.</summary>
private static async Task<MergeChoices> ChoicesAsync(ArchiveFixture fx, Guid keep, Guid drop, CopyPhotoPolicy policy,
IReadOnlyDictionary<string, PhotoChoice>? perPhoto = null, params string[] other)
{
var keepRow = (await fx.Db.Sequences.GetAsync(keep))!;
var dropRow = (await fx.Db.Sequences.GetAsync(drop))!;
var lines = CopyPayload.Lines(SyncPayload.Of(keepRow), await SequencePayloadWriter.PhotosAsync(fx.Db, keep, null, fx.Ct),
SyncPayload.Of(dropRow), await SequencePayloadWriter.PhotosAsync(fx.Db, drop, null, fx.Ct));
var details = lines.Select(l => l.Path).Distinct().ToDictionary(p => p, p => other.Contains(p) ? CopySide.Other : CopySide.ThisPc);
return new MergeChoices(policy, perPhoto ?? new Dictionary<string, PhotoChoice>(), details);
}
private static async Task<Sequence> TitledAsync(ArchiveFixture fx, Sequence sequence, string title)
{
var result = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Title = title }, null, fx.Ct);
return result.Sequence!;
}
private static Task<Job> RunningAsync(ArchiveFixture fx, JobKind kind, params Guid[] targets) => EnqueuedAsync(fx, new Job
{
Id = Guid.CreateVersion7(),
Kind = kind,
TargetIds = targets,
State = JobState.Running,
NodeId = "test#0",
Created = DateTimeOffset.UtcNow,
Started = DateTimeOffset.UtcNow,
});
private static async Task<Job> EnqueuedAsync(ArchiveFixture fx, Job job)
{
await fx.Db.Jobs.EnqueueAsync(job);
return job;
}
private (JobEngine Engine, MergeJobHandler Handler) Engine(ArchiveFixture fx, SequenceMerger merger, FakeProjects projects, FakeWorkspace workspace)
{
var handler = (MergeJobHandler)SequenceMerger.CreateHandler(fx.Db, merger, fx.Service, projects, workspace);
var engine = Own(new JobEngine(fx.Db, [.. fx.Service.CreateJobHandlers(), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }));
return (engine, handler);
}
private static async Task<IReadOnlySet<string>> HashesAsync(ArchiveFixture fx, Guid sequenceId) =>
(await fx.Db.Frames.ListBySequenceAsync(sequenceId)).Select(f => f.Sha256!).ToHashSet(StringComparer.Ordinal);
/// <summary>
/// A pair with something in every stage: shared photo 1, photo 2 kept only, photo 3 only in the other, photo 4 different at
/// one instant (same name), a render inside the other's folder, a project, a session key, a link and a title.
/// </summary>
private static async Task<Scenario> ScenarioAsync(ArchiveFixture fx, int n)
{
var b = 100 * n;
var keep = await SequenceAsync(fx, $"keep-{n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 2, 10_000, "IMG_0002.JPG"), (b + 4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, $"drop-{n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 3, 20_000, "IMG_0003.JPG"), (b + 5, 30_250, "IMG_0004.JPG")]);
drop = await TitledAsync(fx, drop, $"Harbour {n}");
var exports = fx.Paths.Sequence(drop.Folder).Exports;
Directory.CreateDirectory(exports);
var clip = Path.Combine(exports, "clip.mp4");
var bytes = RandomNumberGenerator.GetBytes(4096);
File.WriteAllBytes(clip, bytes);
var render = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = drop.Id,
PresetName = "1080p",
OutputPath = fx.Paths.ToRelative(clip),
Bytes = bytes.Length,
Sha256 = Sha(bytes),
Created = DateTimeOffset.UtcNow,
};
await fx.Db.Renders.InsertAsync(render, fx.Ct);
var project = await fx.Service.Projects.CreateAsync(drop.Id, "Night cut", cancellationToken: fx.Ct);
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = drop.DisplayTitle }, fx.Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var differs = Sha(b + 4, 30_250);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.OtherWins, null, SyncRules.MetadataPath + "title");
return new Scenario(keep, drop, copy, render, project, choices, differs);
}
private sealed record Scenario(Sequence Keep, Sequence Drop, FoundCopy Copy, Render Render, SequenceProject Project, MergeChoices Choices, string KeepLoser);
// ---- the dry run ----
[Test]
public async Task TheKeptOneIsPreselectedOnlyWhenOneIsTheLinkedOriginalsCopy()
{
var fx = await FixtureAsync();
var a = await SequenceAsync(fx, "a", [(1, 0, "IMG_0001.JPG")]);
var b = await SequenceAsync(fx, "b", [(1, 0, "IMG_0001.JPG")]);
var copy = await PairAsync(fx, a.Id, b.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, a.Id, b.Id, CopyPhotoPolicy.Union);
// Nothing linked: the user chooses; the plan keeps the one the dialog does not name.
var plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
Assert.Equal((a.Id, b.Id), (plan.KeepId, plan.DropId));
// One linked, in a server's originals archive: preselected (also when the dialog names it as the other).
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = b.Id, Title = "b" }, Ct);
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
Assert.DoesNotContain(CopyBlock.KeptUnchosen, plan.Blocks);
Assert.Equal((b.Id, a.Id), (plan.KeepId, plan.DropId));
// Linked but not in an originals archive: asked.
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
// Both linked: asked again.
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = "other.local", RemoteId = Guid.CreateVersion7(), LocalId = a.Id, Title = "a" }, Ct);
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
// Chosen: never asked.
plan = await merger.PlanAsync(copy.Id, a.Id, b.Id, choices, NoContext, Ct);
Assert.Empty(plan.Blocks);
Assert.Equal((a.Id, b.Id), (plan.KeepId, plan.DropId));
}
[Test]
public async Task OnlyThereIsAlwaysCopiedAndThePolicyDecidesDifferentPhotosOnly()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (3, 20_000, "IMG_0003.JPG"), (5, 30_250, "IMG_0004.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (only, theirs, mine) = (Sha(3, 20_000), Sha(5, 30_250), Sha(4, 30_250));
// Choose each with nothing chosen: the plan waits for a choice (a merge offers no Leave, owner review 6); the only-there
// photo still comes; the other's different version is a last copy.
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ChoosePerPhoto), NoContext, Ct);
Assert.SequenceEqual([CopyBlock.PhotosUnchosen], plan.Blocks);
Assert.Equal(1, plan.PhotosUnchosen);
Assert.False(plan.CanApply);
Assert.SequenceEqual([(PhotoAction.Fetch, only)], plan.Photos.Select(p => (p.Action, p.Sha256!)));
Assert.SequenceEqual([(CopySide.Other, theirs)], plan.LastCopies.Select(l => (l.Side, l.Key)));
// Choose each, Take the other's: its version comes, the kept one's goes (a last copy of this PC).
var key = mine;
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id,
await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ChoosePerPhoto, new Dictionary<string, PhotoChoice> { [key] = PhotoChoice.TakeThere }), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, only), (PhotoAction.FetchVersion, theirs), (PhotoAction.TrashHere, mine)],
plan.Photos.Select(p => (p.Action, p.Sha256!)).OrderBy(p => p.Action));
Assert.SequenceEqual([(CopySide.ThisPc, mine)], plan.LastCopies.Select(l => (l.Side, l.Key)));
// Union keeps both versions: nothing is a last copy.
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, only), (PhotoAction.FetchVersion, theirs)], plan.Photos.Select(p => (p.Action, p.Sha256!)).OrderBy(p => p.Action));
Assert.Empty(plan.LastCopies);
Assert.Equal(plan.Photos.Sum(p => p.Bytes), plan.CopyBytes);
// The kept one wins: nothing of the different photo moves; the other's version is a last copy.
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ThisPcWins), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, only)], plan.Photos.Select(p => (p.Action, p.Sha256!)));
Assert.SequenceEqual([theirs], plan.LastCopies.Select(l => l.Key));
}
[Test]
public async Task APhotoTombstonedInTheKeptOneIsALastCopyUnlessAnotherSequenceHoldsIt()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var gone = Sha(2, 10_000);
await fx.Db.FrameRemovals.SyncSequenceAsync(keep.Id, [new FrameRemoval(keep.Id, gone, null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Empty(plan.Photos);
Assert.SequenceEqual([(CopySide.Other, gone)], plan.LastCopies.Select(l => (l.Side, l.Key)));
await SequenceAsync(fx, "third", [(2, 10_000, "IMG_0002.JPG")]);
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Empty(plan.LastCopies);
}
[Test]
public async Task SessionKeysFollowTheKeptOneButOneItsArchiveMapsElsewhereAndLinksArePlanned()
{
var fx = await FixtureAsync();
var second = await fx.Service.AdoptAsync(Context.CreateTempDirectory("second"), "Second", Ct);
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var third = await SequenceAsync(fx, "card-7", [(9, 0, "IMG_0009.JPG")]);
var drop = await SequenceAsync(fx, "card-7", [(1, 0, "IMG_0001.JPG")], second);
await fx.Db.Sequences.MapSessionKeyAsync(second.Id, "card-8", drop.Id, Ct);
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = "d" }, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var plan = await Merger(fx).PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Blocks);
Assert.SequenceEqual(["card-8"], plan.SessionKeys);
Assert.SequenceEqual(["card-7"], plan.SkippedSessionKeys);
Assert.SequenceEqual([Nas], plan.Links);
Assert.Equal(third.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "card-7", Ct));
}
[Test]
public async Task AnotherJobOnEitherSequenceAndBothLinkedToOneServerBlockThePlan()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = new SequenceMerger(fx.Db, fx.Service, new CopySides(fx.Db)) { ServerNames = key => key == Nas ? "Studio NAS" : null };
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
var render = await RunningAsync(fx, JobKind.Render, keep.Id);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.Busy, plan.Blocks);
await fx.Db.Jobs.SetStateAsync(render.Id, JobState.Done, null, Ct);
Assert.Empty((await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks);
var links = new SyncLinkStore(fx.Db);
await links.UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = keep.Id, Title = "k" }, Ct);
await links.UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = "d" }, Ct);
var both = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.BothLinked, both.Blocks);
// Named by the server's name, never its key.
Assert.Equal("Studio NAS", both.BothLinkedServer);
Assert.SequenceEqual(["Studio NAS"], both.LinkNames);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.BothLinked", "Studio NAS"), await merger.BlockMessageAsync(both with { Blocks = [CopyBlock.BothLinked] }, Ct));
var unsent = PlanContext() with { UnsentWorkspaceSequences = new HashSet<Guid> { drop.Id } };
Assert.Contains(CopyBlock.UnsentWorkspace, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, unsent, Ct)).Blocks);
// A detail that differs and is not chosen.
await TitledAsync(fx, drop, "Other title");
var unchosen = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.DetailsUnchosen, unchosen.Blocks);
Assert.Null(unchosen.Payload);
}
[Test]
public async Task TheFingerprintMovesWithADetailOfTheKeptOneNotWithAnImportElsewhere()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
var first = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
await SequenceAsync(fx, "elsewhere", [(7, 0, "IMG_0007.JPG")]);
Assert.Equal(first.Fingerprint, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Fingerprint);
await fx.Db.Sequences.EditAsync(keep.Id, new SequenceEdit { Rating = 4 }, null, Ct);
Assert.NotEqual(first.Fingerprint, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Fingerprint);
}
// ---- the job ----
[Test]
public async Task AMergeCopiesVerifiesAndMovesEverythingThenTrashesTheOther()
{
var fx = await FixtureAsync();
var s = await ScenarioAsync(fx, 0);
var merger = Merger(fx);
var projects = new FakeProjects();
var (engine, _) = Engine(fx, merger, projects, new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
Assert.Empty(plan.Blocks);
Assert.SequenceEqual([(CopySide.ThisPc, s.KeepLoser)], plan.LastCopies.Select(l => (l.Side, l.Key)));
Assert.SequenceEqual([s.Render.Id], plan.Renders.Select(r => r.RenderId));
Assert.SequenceEqual([("Night cut", "Night cut")], plan.Projects.Select(p => (p.SourceName, p.TargetName)));
Assert.Equal(1, plan.DetailChanges);
Assert.Equal("Harbour 0", plan.Payload!.Title);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
Assert.Equal(CopyState.Integrating, (await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct))!.State);
await fx.WaitForAsync(job.Id, JobState.Done);
// Photos: shared, kept only, only in the other and the other's winning version; the kept one's losing version went.
Assert.True((await HashesAsync(fx, s.Keep.Id)).SetEquals([Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(5, 30_250)]));
var keep = (await fx.Db.Sequences.GetAsync(s.Keep.Id))!;
Assert.Equal("Harbour 0", keep.Title);
// The render keeps its id, its file copied into the kept sequence.
var render = (await fx.Db.Renders.GetAsync(s.Render.Id))!;
Assert.Equal(s.Keep.Id, render.SequenceId);
Assert.Equal(s.Render.Sha256, Sha(File.ReadAllBytes(fx.Paths.FromRelative(render.OutputPath))));
Assert.Equal(1, keep.RenderedCount);
Assert.SequenceEqual([("Night cut", s.Project.Id)], projects.Copied.Select(c => (c.Name, c.Source)));
// Keys, links and the pair.
Assert.Equal(s.Keep.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "drop-0", Ct));
Assert.Contains("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(keep.Folder).ManifestPath).SessionKeys);
Assert.Equal(s.Keep.Id, (await new SyncLinkStore(fx.Db).ListAsync(Nas, Ct)).Single().LocalId);
Assert.Null(await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
// The other is in its archive's trash, restorable, without the keys it gave away.
Assert.Null(await fx.Db.Sequences.GetAsync(s.Drop.Id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, Ct)).Single(e => e.SequenceId == s.Drop.Id && e.Kind == TrashKind.Sequence);
var restored = await fx.Service.RestoreAsync(fx.Archive.Id, entry.Name, Ct);
Assert.Equal(s.Drop.Id, restored.Id);
Assert.Equal(3, restored.FrameCount);
Assert.DoesNotContain("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(restored.Folder).ManifestPath).SessionKeys);
}
[Test]
public async Task ACrashAtEachStageResumesWhereItStopped()
{
var fx = await FixtureAsync();
var merger = Merger(fx);
var projects = new FakeProjects();
var (engine, handler) = Engine(fx, merger, projects, new FakeWorkspace());
await engine.StartAsync(Ct);
var stages = Enum.GetValues<MergeStage>().Where(s => s != MergeStage.Done).ToList();
for (var i = 0; i < stages.Count; i++)
{
var s = await ScenarioAsync(fx, i + 1);
var crashed = false;
var at = stages[i];
handler.BeforeStage = stage =>
{
if (stage != at || crashed) return Task.CompletedTask;
crashed = true;
throw new IOException($"crash before {at}");
};
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.NotNull(await fx.Db.Sequences.GetAsync(s.Drop.Id));
Assert.Equal(CopyState.Candidate, (await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct))!.State);
await engine.ResumeAsync(job.Id, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var b = 100 * (i + 1);
Assert.True((await HashesAsync(fx, s.Keep.Id)).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
Assert.Null(await fx.Db.Sequences.GetAsync(s.Drop.Id));
Assert.Equal(s.Keep.Id, (await fx.Db.Renders.GetAsync(s.Render.Id))!.SequenceId);
}
// Each project copied once, whatever stage the run stopped at.
Assert.Equal(stages.Count, projects.Copied.Count);
}
[Test]
public async Task AResumedRendersStageNeverOverwritesARenderMadeAtItsTargetMeanwhile()
{
var fx = await FixtureAsync();
var s = await ScenarioAsync(fx, 0);
// The other also holds an image-sequence export inside its folder.
var frames = Path.Combine(fx.Paths.Sequence(s.Drop.Folder).Exports, "frames");
Directory.CreateDirectory(frames);
File.WriteAllBytes(Path.Combine(frames, "0001.png"), [1, 2, 3]);
var folder = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = s.Drop.Id,
PresetName = "PNG",
OutputPath = fx.Paths.ToRelative(frames),
Bytes = 3,
Created = DateTimeOffset.UtcNow,
};
await fx.Db.Renders.InsertAsync(folder, fx.Ct);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var failedOnce = false;
handler.BeforeStage = stage =>
{
if (stage != MergeStage.Renders || failedOnce) return Task.CompletedTask;
failedOnce = true;
throw new IOException("the other's drive failed a read");
};
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
// The failed run had journaled where the copies go and copied nothing; meanwhile the user renders the kept sequence
// with the same names (same date, title and preset).
var keep = (await fx.Db.Sequences.GetAsync(s.Keep.Id))!;
var exports = fx.Paths.Sequence(keep.Folder).Exports;
var clip = Path.Combine(exports, "clip.mp4");
var images = Path.Combine(exports, "frames");
Directory.CreateDirectory(images);
var fresh = RandomNumberGenerator.GetBytes(2048);
File.WriteAllBytes(clip, fresh);
File.WriteAllBytes(Path.Combine(images, "0001.png"), [9, 9, 9]);
await fx.Db.Renders.InsertAsync(new Render
{
Id = Guid.CreateVersion7(),
SequenceId = keep.Id,
PresetName = "1080p",
OutputPath = fx.Paths.ToRelative(clip),
Bytes = fresh.Length,
Sha256 = Sha(fresh),
Created = DateTimeOffset.UtcNow,
}, fx.Ct);
var journal = MergeJournal.Read(fx.Paths, job.Id)!;
journal.RenderTargets[s.Render.Id] = fx.Paths.ToRelative(clip);
journal.RenderTargets[folder.Id] = fx.Paths.ToRelative(images);
journal.Save(fx.Paths);
await engine.ResumeAsync(job.Id, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
// The new renders are as the user made them; the other's go beside them under free names.
Assert.SequenceEqual(fresh, File.ReadAllBytes(clip));
Assert.SequenceEqual(new byte[] { 9, 9, 9 }, File.ReadAllBytes(Path.Combine(images, "0001.png")));
Assert.Count(1, Directory.GetFiles(images));
var movedClip = (await fx.Db.Renders.GetAsync(s.Render.Id))!;
Assert.NotEqual(fx.Paths.ToRelative(clip), movedClip.OutputPath);
Assert.Equal(s.Render.Sha256, Sha(File.ReadAllBytes(fx.Paths.FromRelative(movedClip.OutputPath))));
var movedFolder = (await fx.Db.Renders.GetAsync(folder.Id))!;
Assert.NotEqual(fx.Paths.ToRelative(images), movedFolder.OutputPath);
Assert.SequenceEqual(new byte[] { 1, 2, 3 }, File.ReadAllBytes(Path.Combine(fx.Paths.FromRelative(movedFolder.OutputPath), "0001.png")));
}
[Test]
public async Task APhotoOfTheOtherThatChangedOnDiskFailsTheVerifyAndNothingGoesToTheTrash()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var shared = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).First(f => f.Sha256 == Sha(1, 0));
File.WriteAllBytes(fx.Paths.FromRelative(shared.RelativePath), Photos.Jpeg(Start, 77));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains(shared.FileName, failed.Error!);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
Assert.Equal(CopyState.Candidate, (await new FoundCopyStore(fx.Db).GetAsync(copy.Id, Ct))!.State);
}
[Test]
public async Task AnUnhashedPhotoThatIsAnotherPhotoAfterAllIsKeptAsASecondVersion()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(2, 0, "IMG_0001.JPG")]);
// Neither hashed, the same size and time: compared as the same photo.
var access = new DbAccess(fx.Db.Database);
await access.ExecuteAsync("UPDATE frames SET sha256 = NULL, size = 1000 WHERE sequence_id = $1 OR sequence_id = $2", [keep.Id, drop.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Photos);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var frames = await fx.Db.Frames.ListBySequenceAsync(keep.Id);
Assert.Count(2, frames);
Assert.Contains(Sha(2, 0), frames.Select(f => f.Sha256));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AnUnhashedPhotoOfTheKeptOneIsHashedByThePlanAndAnotherPhotoOfTheOtherIsKeptAsASecondVersion()
{
var fx = await FixtureAsync();
// The kept one adopted and not verified, the other hashed: one photo of the same size and time, other bytes.
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(2, 0, "IMG_0001.JPG")]);
var access = new DbAccess(fx.Db.Database);
await access.ExecuteAsync("UPDATE frames SET size = 1000 WHERE sequence_id = $1 OR sequence_id = $2", [keep.Id, drop.Id], Ct);
await access.ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1", [keep.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.LastCopies);
Assert.SequenceEqual([(PhotoAction.FetchVersion, Sha(2, 0))], plan.Photos.Select(p => (p.Action, p.Sha256!)));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var frames = await fx.Db.Frames.ListBySequenceAsync(keep.Id);
Assert.Count(2, frames);
Assert.Contains(Sha(2, 0), frames.Select(f => f.Sha256));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task ASecondVersionOfAnUnhashedKeptPhotoIsListedAndLeftOutOfMain()
{
var fx = await FixtureAsync();
// The kept one adopted and not verified, the other hashed: one photo of the same size and sub-second time, other bytes.
var keep = await SequenceAsync(fx, "keep", [(1, 250, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(2, 250, "IMG_0001.JPG")]);
var access = new DbAccess(fx.Db.Database);
await access.ExecuteAsync("UPDATE frames SET size = 1000 WHERE sequence_id = $1 OR sequence_id = $2", [keep.Id, drop.Id], Ct);
await access.ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1", [keep.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
// The kept photo has the hash the merge read, so Two versions lists the pair, the other's version left out of Main.
Assert.Contains(Sha(1, 250), (await fx.Db.Frames.ListBySequenceAsync(keep.Id)).Select(f => f.Sha256));
AssertVersions(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct), Sha(1, 250), Sha(2, 250));
}
[Test]
public async Task APhotoTheVerifyFindsIsAnotherVersionIsListedAtAWholeSecondAndLeftOutOfMain()
{
var fx = await FixtureAsync();
// A camera that records whole seconds, photos of one size (an uncompressed RAW series): neither side hashed, compared
// as the same photo, another photo after all.
var (mine, theirs) = EqualSizeSeeds(0);
var keep = await SequenceAsync(fx, "keep", [(mine, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(theirs, 0, "IMG_0001.JPG")]);
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1 OR sequence_id = $2", [keep.Id, drop.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Photos);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
// Filed at the verify as a second version: listed, and Main renders one photo at that instant.
Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(mine, 0), Sha(theirs, 0)]));
AssertVersions(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct), Sha(mine, 0), Sha(theirs, 0));
}
[Test]
public async Task AVersionTheVerifyFiledBeforeTheRunStoppedIsLeftOutOfMainOnResume()
{
var fx = await FixtureAsync();
var (mine, theirs) = EqualSizeSeeds(0);
var keep = await SequenceAsync(fx, "keep", [(mine, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(theirs, 0, "IMG_0001.JPG")]);
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1 OR sequence_id = $2", [keep.Id, drop.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
// Filed at the verify, then the exclusion fails (a drive drops out, project.json is held) before the journal says so.
var failedOnce = false;
handler.BeforeLeaveOutOfMain = _ =>
{
if (failedOnce) return Task.CompletedTask;
failedOnce = true;
throw new IOException("project.json is in use");
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Photos);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(mine, 0), Sha(theirs, 0)]));
await engine.ResumeAsync(job.Id, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
// The resumed verify finds the version in the kept sequence and leaves it out of Main: one photo at that instant.
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
AssertVersions(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct), Sha(mine, 0), Sha(theirs, 0));
}
[Test]
public async Task AVersionWhoseFileChangedBeforeItWasFiledIsFiledAsItIsOnResume()
{
var fx = await FixtureAsync();
var (mine, theirs) = EqualSizeSeeds(0);
var edited = EqualSizeSeed(0, theirs, mine);
var keep = await SequenceAsync(fx, "keep", [(mine, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(theirs, 0, "IMG_0001.JPG")]);
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1 OR sequence_id = $2", [keep.Id, drop.Id], Ct);
var path = fx.Paths.FromRelative((await fx.Db.Frames.ListBySequenceAsync(drop.Id)).Single().RelativePath);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
// The file is rewritten (an editor saving a rotation) between its hash and its copy: the copy reads other bytes.
var rewritten = false;
handler.BeforeFileVersion = _ =>
{
if (rewritten) return Task.CompletedTask;
rewritten = true;
File.WriteAllBytes(path, Photos.Jpeg(Start, edited));
return Task.CompletedTask;
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(mine, 0)]), "nothing was filed");
await engine.ResumeAsync(job.Id, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
// Nothing was filed: the retry files the photo as it is now, as a second version left out of Main.
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(mine, 0), Sha(edited, 0)]));
AssertVersions(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct), Sha(mine, 0), Sha(edited, 0));
}
[Test]
public async Task AVersionFiledBeforeItsFileChangedIsLeftOutOfMainAndTheMergePlannedAgain()
{
var fx = await FixtureAsync();
var (mine, theirs) = EqualSizeSeeds(0);
var edited = EqualSizeSeed(0, theirs, mine);
var keep = await SequenceAsync(fx, "keep", [(mine, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(theirs, 0, "IMG_0001.JPG")]);
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1 OR sequence_id = $2", [keep.Id, drop.Id], Ct);
var dropFrame = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).Single();
var path = fx.Paths.FromRelative(dropFrame.RelativePath);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
// Filed at the verify, then the exclusion fails; the file is rewritten before the retry.
var failedOnce = false;
handler.BeforeLeaveOutOfMain = _ =>
{
if (failedOnce) return Task.CompletedTask;
failedOnce = true;
File.WriteAllBytes(path, Photos.Jpeg(Start, edited));
throw new IOException("project.json is in use");
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(mine, 0), Sha(theirs, 0)]));
await engine.ResumeAsync(job.Id, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
// The filed version is left out of Main, the other stays, and the journal goes: the next run plans again.
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.ChangedFile", dropFrame.FileName), failed.Error);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
AssertVersions(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct), Sha(mine, 0), Sha(theirs, 0));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)), "the next run plans again");
}
[Test]
public async Task ASecondVersionCopiedInByAMergeAbandonedBeforeItsDetailsStaysOutOfMain()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (5, 30_250, "IMG_0004.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
// While the photos are copied the user tags the kept one: its details moved, so the Details stage abandons the merge.
handler.BeforeStage = stage => stage == MergeStage.Details ? fx.Db.Sequences.SetTagsAsync(keep.Id, ["edited"], Ct) : Task.CompletedTask;
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.FetchVersion, Sha(5, 30_250))], plan.Photos.Select(p => (p.Action, p.Sha256!)));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
// The version copied in is already left out of Main.
AssertVersions(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct), Sha(4, 30_250), Sha(5, 30_250));
// Planned again (the kept one holds both versions now) and merged: still one photo in Main at that instant.
handler.BeforeStage = null;
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Photos);
job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
AssertVersions(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct), Sha(4, 30_250), Sha(5, 30_250));
}
[Test]
public async Task AnUnhashedPhotoMatchedBySizeAndTimeWhoseContentTheKeptOneDeletedIsALastCopy()
{
var fx = await FixtureAsync();
// A whole-second camera shooting a burst of one size: the other holds two photos at one second, not hashed; the kept
// one holds one of them, hashed, and deleted the other.
var (deleted, kept) = EqualSizeSeeds(0);
var keep = await SequenceAsync(fx, "keep", [(kept, 0, "IMG_0002.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(deleted, 0, "IMG_0001.JPG"), (kept, 0, "IMG_0002.JPG")]);
await fx.Db.FrameRemovals.SyncSequenceAsync(keep.Id, [new FrameRemoval(keep.Id, Sha(deleted, 0), null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct);
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1", [drop.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
// Compared by content: the deleted photo is a last copy to confirm, nothing to copy.
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Photos);
Assert.SequenceEqual([(CopySide.Other, Sha(deleted, 0))], plan.LastCopies.Select(l => (l.Side, l.Key)));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(kept, 0)]));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task APhotoTheKeptOneDeletesAfterThePlanMakesTheVerifyAskForANewPlan()
{
var fx = await FixtureAsync();
var (deleted, kept) = EqualSizeSeeds(0);
var keep = await SequenceAsync(fx, "keep", [(kept, 0, "IMG_0002.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(deleted, 0, "IMG_0001.JPG"), (kept, 0, "IMG_0002.JPG")]);
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1", [drop.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.LastCopies);
// Once planned, the kept one's tombstone for the other photo arrives (a sync of a deletion made elsewhere).
handler.BeforeStage = stage => stage == MergeStage.Verify
? fx.Db.FrameRemovals.SyncSequenceAsync(keep.Id, [new FrameRemoval(keep.Id, Sha(deleted, 0), null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct)
: Task.CompletedTask;
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
// Not a verify that fails on every retry: the journal goes, and the new plan lists the photo as a last copy.
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.SequenceEqual([(CopySide.Other, Sha(deleted, 0))], plan.LastCopies.Select(l => (l.Side, l.Key)));
}
/// <summary>One photo listed in two versions: <paramref name="kept"/> in Main, <paramref name="incoming"/> left out.</summary>
private static void AssertVersions(IReadOnlyList<PhotoVersions> listed, string kept, string incoming)
{
Assert.Count(1, listed);
var versions = listed[0];
var excluded = new Dictionary<string, bool> { [versions.FirstSha256] = versions.FirstExcluded, [versions.SecondSha256] = versions.SecondExcluded };
Assert.False(excluded[kept]);
Assert.True(excluded[incoming]);
}
/// <summary>Two seeds whose photos at <paramref name="ms"/> have the same size and other bytes.</summary>
private static (int, int) EqualSizeSeeds(int ms)
{
var bySize = new Dictionary<int, int>();
for (var seed = 1; ; seed++)
{
var length = Photos.Jpeg(Start.AddMilliseconds(ms), seed).Length;
if (bySize.TryGetValue(length, out var other)) return (other, seed);
bySize[length] = seed;
}
}
/// <summary>A seed other than <paramref name="like"/> and <paramref name="not"/> whose photo at <paramref name="ms"/> has the same size as <paramref name="like"/>'s.</summary>
private static int EqualSizeSeed(int ms, int like, params int[] not)
{
var length = Photos.Jpeg(Start.AddMilliseconds(ms), like).Length;
for (var seed = 1; ; seed++)
if (seed != like && !not.Contains(seed) && Photos.Jpeg(Start.AddMilliseconds(ms), seed).Length == length) return seed;
}
[Test]
public async Task AnUnhashedPhotoOfTheKeptOneWithTheSameContentIsNoLastCopy()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG")]);
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1", [keep.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var plan = await Merger(fx).PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.LastCopies);
Assert.Empty(plan.Photos);
}
[Test]
public async Task ALosingVersionWhoseOtherHolderIsFlaggedMeanwhileIsNeverTrashedUnlisted()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (5, 30_250, "IMG_0004.JPG")]);
// A third sequence holds the kept one's losing version: not a last copy when planned.
var third = await SequenceAsync(fx, "third", [(4, 30_250, "IMG_0004.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.OtherWins);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Empty(plan.LastCopies);
Assert.Contains(plan.Photos, p => p.Action == PhotoAction.TrashHere);
// While the photos are copied, a verify flags the third's copy Corrupt.
handler.BeforeStage = async stage =>
{
if (stage != MergeStage.Replace) return;
var held = (await fx.Db.Frames.ListBySequenceAsync(third.Id)).Single();
await fx.Db.Frames.UpdateFlagsAsync([held.Id], FrameFlags.Corrupt, FrameFlags.None, Ct);
};
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
// Its only usable copy now: the kept one still holds it (beside the other's version), nothing went to the trash, and
// the journal went so the next run plans again.
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
var kept = await fx.Db.Frames.ListBySequenceAsync(keep.Id);
Assert.Contains(kept, f => f.Sha256 == Sha(4, 30_250) && f.Flags == FrameFlags.None
&& Sha(File.ReadAllBytes(fx.Paths.FromRelative(f.RelativePath))) == Sha(4, 30_250));
Assert.Contains(Sha(5, 30_250), kept.Select(f => f.Sha256));
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
}
[Test]
public async Task ADetailsChangeOfTheKeptOneBeforeTheDetailsStageMakesTheRetryPlanAgain()
{
var fx = await FixtureAsync();
var s = await ScenarioAsync(fx, 0);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
handler.BeforeStage = stage => stage == MergeStage.Details ? fx.Db.Sequences.SetTagsAsync(s.Keep.Id, ["edited"], Ct) : Task.CompletedTask;
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)), "the next run plans again");
// The Retry plans again and says what the new plan needs (the new detail line) instead of failing the same way.
handler.BeforeStage = null;
await engine.ResumeAsync(job.Id, Ct);
failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.Plural("Errors.Copies.DetailsUnchosen", 1), failed.Error);
Assert.NotNull(await fx.Db.Sequences.GetAsync(s.Drop.Id));
}
[Test]
public async Task ADamagedPhotoOfTheKeptOneIsRepairedFromTheOther()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var damaged = (await fx.Db.Frames.ListBySequenceAsync(keep.Id)).First(f => f.Sha256 == Sha(2, 10_000));
File.WriteAllBytes(fx.Paths.FromRelative(damaged.RelativePath), [1, 2, 3]);
await fx.Db.Frames.UpdateFlagsAsync([damaged.Id], FrameFlags.Corrupt, FrameFlags.None, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, Sha(2, 10_000))], plan.Photos.Select(p => (p.Action, p.Sha256!)));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var frames = await fx.Db.Frames.ListBySequenceAsync(keep.Id);
Assert.Count(2, frames);
var repaired = frames.Single(f => f.Sha256 == Sha(2, 10_000));
Assert.Equal(Sha(2, 10_000), Sha(File.ReadAllBytes(fx.Paths.FromRelative(repaired.RelativePath))));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AChangedPhotoOfTheOtherNothingElseHoldsBlocksTheMerge()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
// A photo of the other edited after import: a verify flags it Corrupt (changed), its file holds the only copy.
var edited = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).First(f => f.Sha256 == Sha(2, 10_000));
var file = fx.Paths.FromRelative(edited.RelativePath);
File.WriteAllBytes(file, Photos.Jpeg(Start.AddMilliseconds(10_000), 88));
await fx.Db.Frames.UpdateFlagsAsync([edited.Id], FrameFlags.Corrupt, FrameFlags.None, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.DamagedPhotos, plan.Blocks);
Assert.False(plan.CanApply);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.DamagedPhotos", "Drop"), SequenceMerger.BlockText(plan, CopyBlock.DamagedPhotos, "Keep", "Drop"));
// Another sequence of this index holds that content too: nothing would be lost, the merge may go on.
await SequenceAsync(fx, "elsewhere", [(88, 10_000, "IMG_0002.JPG")]);
Assert.DoesNotContain(CopyBlock.DamagedPhotos, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks);
}
[Test]
public async Task APhotoFlaggedMissingWhoseFileIsThereBlocksTheMerge()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
// Flagged Missing by an earlier verify (a drive away then), its file back now: the merge would not copy it.
var missing = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).First(f => f.Sha256 == Sha(2, 10_000));
await fx.Db.Frames.UpdateFlagsAsync([missing.Id], FrameFlags.Missing, FrameFlags.None, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var plan = await Merger(fx).PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Contains(CopyBlock.DamagedPhotos, plan.Blocks);
Assert.False(plan.CanApply);
}
[Test]
public async Task APlanMadeAgainHashesNoPhotoThatDidNotChange()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG"), (3, 20_000, "IMG_0003.JPG")]);
// The other's photos not hashed yet (an older import): the plan hashes those only it holds, to compare them by content.
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1", [drop.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
var first = merger.FilesHashed;
Assert.True(first >= 2, $"{first} files hashed");
// Each choice in the dialog plans again: the photos read before are not read again.
await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Equal(first, merger.FilesHashed);
// A photo whose file changed is read again.
var changed = (await fx.Db.Frames.ListBySequenceAsync(drop.Id))[^1]; // in capture order: photo 3
var file = fx.Paths.FromRelative(changed.RelativePath);
File.WriteAllBytes(file, Photos.Jpeg(Start.AddMilliseconds(20_000), 99));
File.SetLastWriteTimeUtc(file, DateTime.UtcNow.AddMinutes(1));
await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Equal(first + 1, merger.FilesHashed);
}
[Test]
public async Task PhotosTheIndexHasNotReadInTheOtherBlockThePlan()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
await fx.ScanAsync();
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
Assert.DoesNotContain(CopyBlock.FolderChanged, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks);
// A photo copied into the other's folder by hand: the index has not read it, so the trash would take it unseen.
var folder = fx.Paths.Sequence(drop.Folder);
var extra = Path.Combine(ArchiveScanner.PhotoDirectory(folder), "IMG_0099.JPG");
File.WriteAllBytes(extra, Photos.Jpeg(Start.AddMilliseconds(99_000), 99));
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
Assert.Contains(CopyBlock.FolderChanged, plan.Blocks);
Assert.False(plan.CanApply);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.FolderChanged", "Drop"), SequenceMerger.BlockText(plan, CopyBlock.FolderChanged, "Keep", "Drop"));
// Read by a scan: the plan may go again.
await fx.ScanAsync();
Assert.DoesNotContain(CopyBlock.FolderChanged, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks);
// A sidecar another index wrote past this one's frames version blocks too.
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 } });
Assert.Contains(CopyBlock.FolderChanged, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks);
}
[Test]
public async Task AResumedMergeStopsBeforeTheTrashWhenTheOtherFolderGotPhotosMeanwhile()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
await fx.ScanAsync();
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var crashed = false;
handler.BeforeStage = stage =>
{
if (stage != MergeStage.Links || crashed) return Task.CompletedTask;
crashed = true;
throw new IOException("crash before the links");
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Blocks);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
// While it waits, a photo lands in the other's folder outside the index.
var extra = Path.Combine(ArchiveScanner.PhotoDirectory(fx.Paths.Sequence(drop.Folder)), "IMG_0099.JPG");
File.WriteAllBytes(extra, Photos.Jpeg(Start.AddMilliseconds(99_000), 99));
await engine.ResumeAsync(job.Id, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.FolderChanged", drop.DisplayTitle), failed.Error);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.True(File.Exists(extra));
}
[Test]
public async Task APhotoOfTheOtherFlaggedDamagedWhileTheMergeWaitedIsCheckedOnResume()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var crashed = false;
handler.BeforeStage = stage =>
{
if (stage != MergeStage.Verify || crashed) return Task.CompletedTask;
crashed = true;
throw new IOException("crash after the copy");
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
// A verify meanwhile: the other's photo 2 was edited after the copy, its file the only copy of those bytes.
var edited = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).First(f => f.Sha256 == Sha(2, 10_000));
File.WriteAllBytes(fx.Paths.FromRelative(edited.RelativePath), Photos.Jpeg(Start.AddMilliseconds(10_000), 88));
await fx.Db.Frames.UpdateFlagsAsync([edited.Id], FrameFlags.Corrupt, FrameFlags.None, Ct);
await engine.ResumeAsync(job.Id, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.DamagedPhotos", drop.DisplayTitle), failed.Error);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
}
[Test]
public async Task WorkSentHomeIntoTheOtherAfterAStoppedTrashAsksForANewPlan()
{
var fx = await FixtureAsync();
var merger = Merger(fx);
var workspace = new FakeWorkspace();
var (engine, handler) = Engine(fx, merger, new FakeProjects(), workspace);
await engine.StartAsync(Ct);
var store = new FoundCopyStore(fx.Db);
// Details sent home (a settings version), then a project made in the other (a new head), each after the stop.
Func<Sequence, Task>[] sends =
[
drop => fx.Db.Sequences.SetTagsAsync(drop.Id, ["sent"], Ct),
drop => fx.Service.Projects.CreateAsync(drop.Id, "Sent cut", cancellationToken: fx.Ct),
];
for (var i = 0; i < sends.Length; i++)
{
var keep = await SequenceAsync(fx, $"keep-{i}", [(10 * i + 1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, $"drop-{i}", [(10 * i + 1, 0, "IMG_0001.JPG"), (10 * i + 2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
handler.BeforeStage = stage =>
{
if (stage == MergeStage.Trash) workspace.Unsent.Add(drop.Id);
return Task.CompletedTask;
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
// The workspace sends its work home into the other, then Retry.
await sends[i](drop);
handler.BeforeStage = null;
workspace.Unsent.Clear();
await engine.ResumeAsync(job.Id, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.Changed", drop.DisplayTitle), failed.Error);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)), "the next run plans again");
}
}
[Test]
public async Task AFrameRemovedFromTheOtherAfterAStoppedTrashMakesTheNextRetryPlanAgain()
{
var fx = await FixtureAsync();
var merger = Merger(fx);
var workspace = new FakeWorkspace();
var (engine, handler) = Engine(fx, merger, new FakeProjects(), workspace);
await engine.StartAsync(Ct);
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG"), (3, 20_000, "IMG_0003.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
handler.BeforeStage = stage =>
{
if (stage == MergeStage.Trash) workspace.Unsent.Add(drop.Id);
return Task.CompletedTask;
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.True(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)), "an unsent workspace keeps the journal");
// A frame of the other goes to the trash while the merge waits: its frames version never goes back.
var gone = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).First(f => f.Sha256 == Sha(3, 20_000));
var trash = await fx.Service.FrameTrash.PlanDeleteAsync(drop.Id, [gone.Id], null, fx.Ct, job.Id);
await fx.Service.FrameTrash.DeleteAsync(trash, null, null, fx.Ct, job.Id);
handler.BeforeStage = null;
workspace.Unsent.Clear();
await engine.ResumeAsync(job.Id, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.Changed", drop.DisplayTitle), failed.Error);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)), "the next run plans again");
// The next Retry plans again and asks for a new comparison instead of the same error.
await engine.ResumeAsync(job.Id, Ct);
failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task APhotoSettledThroughAThirdSequenceThatIsGoneByTheTrashKeepsTheOther()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (5, 30_250, "IMG_0004.JPG")]);
// A third sequence holds the other's losing version: not a last copy when planned.
var third = await SequenceAsync(fx, "third", [(5, 30_250, "IMG_0004.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ThisPcWins), NoContext, Ct);
Assert.Empty(plan.LastCopies);
handler.BeforeStage = async stage =>
{
if (stage == MergeStage.Trash) await fx.Db.Sequences.DeleteAsync(third.Id, Ct);
};
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
// Now the other holds its only copy: nothing goes to the trash, and the next plan lists it as a last copy.
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.Changed", drop.DisplayTitle), failed.Error);
Assert.Contains(Sha(5, 30_250), await HashesAsync(fx, drop.Id));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
var again = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ThisPcWins), NoContext, Ct);
Assert.Contains(again.LastCopies, l => l.Key == Sha(5, 30_250));
}
[Test]
public async Task APhotoHeldByAThirdSequenceThatIsGoneByTheVerifyMakesTheRetryPlanAgain()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (5, 30_250, "IMG_0004.JPG")]);
// A third sequence holds the other's losing version: not a last copy when planned.
var third = await SequenceAsync(fx, "third", [(5, 30_250, "IMG_0004.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ThisPcWins), NoContext, Ct);
Assert.Empty(plan.LastCopies);
handler.BeforeStage = async stage =>
{
if (stage == MergeStage.Verify) await fx.Db.Sequences.DeleteAsync(third.Id, Ct);
};
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
// A last copy the plan did not list, not a failed copy: the journal goes and nothing goes to the trash.
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.Contains(Sha(5, 30_250), await HashesAsync(fx, drop.Id));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)), "the next run plans again");
// The Retry asks for a new comparison instead of the same verify error, and the new plan lists the photo.
handler.BeforeStage = null;
await engine.ResumeAsync(job.Id, Ct);
failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
var again = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ThisPcWins), NoContext, Ct);
Assert.Contains(again.LastCopies, l => l.Key == Sha(5, 30_250));
}
[Test]
public async Task APhotoAddedToTheOtherOrDeletedFromTheKeptOneMeanwhileKeepsTheOtherOutOfTheTrash()
{
var fx = await FixtureAsync();
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
// A photo added to the other after the verify.
var keep = await SequenceAsync(fx, "keep-a", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop-a", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
handler.BeforeStage = async stage =>
{
if (stage != MergeStage.Trash) return;
var (path, size, sha) = fx.Stage(Start.AddSeconds(40), 3);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, drop.Id, null, null, path, "IMG_0003.JPG", size, sha, null, null), fx.Ct);
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
// A verified photo deleted from the kept one before the trash.
keep = await SequenceAsync(fx, "keep-b", [(11, 0, "IMG_0001.JPG")]);
drop = await SequenceAsync(fx, "drop-b", [(11, 0, "IMG_0001.JPG"), (12, 10_000, "IMG_0002.JPG")]);
copy = await PairAsync(fx, keep.Id, drop.Id);
var keepId = keep.Id;
Guid? running = null;
handler.BeforeStage = async stage =>
{
if (stage != MergeStage.Trash) return;
// Another hand deletes it (the job's own hold does not count, so the deletion is not refused as busy).
var gone = (await fx.Db.Frames.ListBySequenceAsync(keepId)).First(f => f.Sha256 == Sha(12, 10_000));
var trash = await fx.Service.FrameTrash.PlanDeleteAsync(keepId, [gone.Id], null, fx.Ct, running);
await fx.Service.FrameTrash.DeleteAsync(trash, null, null, fx.Ct, running);
};
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
running = job.Id;
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains(keep.DisplayTitle, failed.Error!);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task UnsentWorkspaceWorkStopsTheTrashAndTheTrashStepIsNotBusyWithItsOwnJob()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var workspace = new FakeWorkspace();
var (engine, handler) = Engine(fx, merger, new FakeProjects(), workspace);
await engine.StartAsync(Ct);
handler.BeforeStage = stage =>
{
if (stage == MergeStage.Trash) workspace.Unsent.Add(drop.Id);
return Task.CompletedTask;
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
// The photos are in the kept one already.
Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(1, 0), Sha(2, 10_000)]));
handler.BeforeStage = null;
workspace.Unsent.Clear();
await engine.ResumeAsync(job.Id, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AUnionMergeLeavesItsSecondVersionsListedInTheKeptOne()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (5, 30_250, "IMG_0004.JPG")]);
// The other deleted photo 2, which the kept one still holds.
await fx.Db.FrameRemovals.SyncSequenceAsync(drop.Id, [new FrameRemoval(drop.Id, Sha(2, 10_000), null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Equal(1, plan.DeletedInOther);
Assert.Empty(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
// The pair is gone with the other; the kept one still lists the photo it holds in two versions.
Assert.Null(await new FoundCopyStore(fx.Db).GetAsync(copy.Id, Ct));
var versions = (await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct)).Single();
Assert.True(new[] { versions.FirstSha256, versions.SecondSha256 }.ToHashSet().SetEquals([Sha(4, 30_250), Sha(5, 30_250)]));
}
[Test]
public async Task ARetryAfterKeepApartRefusesAndTrashesNothing()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var workspace = new FakeWorkspace();
var (engine, handler) = Engine(fx, merger, new FakeProjects(), workspace);
await engine.StartAsync(Ct);
// The trash stage stops on unsent workspace work: the pair is still the failed job's.
handler.BeforeStage = stage =>
{
if (stage == MergeStage.Trash) workspace.Unsent.Add(drop.Id);
return Task.CompletedTask;
};
var store = new FoundCopyStore(fx.Db);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
// Kept apart, the work sent, then Retry: the journal is stale, the other stays where it is.
Assert.True(await store.SetStateAsync(copy.Id, null, CopyState.KeptApart, new CopyAnswer("f", 1, null, null, DateTimeOffset.UtcNow), Ct));
handler.BeforeStage = null;
workspace.Unsent.Clear();
await engine.ResumeAsync(job.Id, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
var still = await fx.Db.Sequences.GetAsync(drop.Id);
Assert.NotNull(still);
Assert.True(File.Exists(fx.Paths.Sequence(still!.Folder).ManifestPath));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
Assert.Equal(CopyState.KeptApart, (await store.GetAsync(copy.Id, Ct))!.State);
}
[Test]
public async Task ARetryAfterAnotherMergeOfThePairRefusesAndTrashesNothing()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
handler.BeforeStage = stage => stage == MergeStage.Verify ? throw new IOException("crash before Verify") : Task.CompletedTask;
var store = new FoundCopyStore(fx.Db);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
// A new merge keeping the other one takes the pair (and stops before its own trash).
handler.BeforeStage = stage => stage == MergeStage.Copy ? throw new IOException("crash before Copy") : Task.CompletedTask;
var swapped = await merger.PlanAsync(copy.Id, drop.Id, keep.Id, await ChoicesAsync(fx, drop.Id, keep.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var second = await merger.EnqueueAsync(engine, store, swapped, drop.DisplayTitle, Ct);
await fx.WaitForAsync(second.Id, JobState.Failed);
handler.BeforeStage = null;
await engine.ResumeAsync(job.Id, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.NotNull(await fx.Db.Sequences.GetAsync(keep.Id));
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
// The other merge's journal is untouched.
Assert.True(File.Exists(MergeJournal.PathFor(fx.Paths, second.Id)));
}
[Test]
public async Task AVerifyThatCannotReadTheOtherStopsAndTheRetryVerifiesEverything()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(2, 0, "IMG_0001.JPG")]);
// Neither hashed, the same size and time: compared as the same photo, only the verify tells them apart.
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL, size = 1000 WHERE sequence_id = $1 OR sequence_id = $2",
[keep.Id, drop.Id], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var folder = fx.Paths.Sequence(drop.Folder).Folder;
var aside = folder + "-away";
handler.BeforeStage = stage =>
{
if (stage == MergeStage.Verify && Directory.Exists(folder)) Directory.Move(folder, aside);
return Task.CompletedTask;
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.Count(1, await fx.Db.Frames.ListBySequenceAsync(keep.Id));
handler.BeforeStage = null;
Directory.Move(aside, folder);
await engine.ResumeAsync(job.Id, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
// The other photo was read and filed as a second version before the other went to the trash.
Assert.Contains(Sha(2, 0), (await fx.Db.Frames.ListBySequenceAsync(keep.Id)).Select(f => f.Sha256));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AnUnhashedPhotoTheKeptOneDeletedIsALastCopyNotACopy()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var gone = Sha(2, 10_000);
await fx.Db.FrameRemovals.SyncSequenceAsync(keep.Id, [new FrameRemoval(keep.Id, gone, null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct);
await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1 AND file_name = $2", [drop.Id, "IMG_0002.JPG"], Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Photos);
Assert.SequenceEqual([(CopySide.Other, gone)], plan.LastCopies.Select(l => (l.Side, l.Key)));
// Confirmed as a last copy: the merge goes through and does not file it in the kept one.
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.DoesNotContain(gone, await HashesAsync(fx, keep.Id));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task ACancelReturnsThePairToItsStateAndLeavesTheOtherAlone()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
Guid? id = null;
handler.BeforeStage = async stage =>
{
if (stage == MergeStage.Verify) await engine.CancelAsync(id!.Value, CancellationToken.None);
};
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var store = new FoundCopyStore(fx.Db);
var job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
id = job.Id;
await fx.WaitForAsync(job.Id, JobState.Cancelled);
var until = DateTime.UtcNow.AddSeconds(10);
while (File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)) && DateTime.UtcNow < until) await Task.Delay(20, Ct);
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
var pair = (await store.GetAsync(copy.Id, Ct))!;
Assert.Equal(CopyState.Candidate, pair.State);
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
// Planned and merged again from the start.
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
handler.BeforeStage = null;
job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AMergeAbandonedAtTheTrashGivesTheOtherBackItsLinkAndSessionKeys()
{
var fx = await FixtureAsync();
var s = await ScenarioAsync(fx, 0);
var merger = Merger(fx);
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
// Details saved into the other after the links stage: the trash abandons the merge.
handler.BeforeStage = stage => stage == MergeStage.Trash ? fx.Db.Sequences.SetTagsAsync(s.Drop.Id, ["edited"], Ct) : Task.CompletedTask;
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.Changed", s.Drop.DisplayTitle), failed.Error);
var drop = Assert.NotNull(await fx.Db.Sequences.GetAsync(s.Drop.Id));
var keep = (await fx.Db.Sequences.GetAsync(s.Keep.Id))!;
Assert.Equal(s.Drop.Id, (await new SyncLinkStore(fx.Db).ListAsync(Nas, Ct)).Single().LocalId);
Assert.Equal(s.Drop.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "drop-0", Ct));
Assert.Contains("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(drop.Folder).ManifestPath).SessionKeys);
Assert.DoesNotContain("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(keep.Folder).ManifestPath).SessionKeys);
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)), "the next run plans again");
}
[Test]
public async Task AProjectOfTheOtherWithTheHeadOfAnUnrelatedKeptProjectIsCopiedWithItsRenders()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG")]);
// Two untouched projects made from equal Mains: equal heads, unrelated projects.
var instagram = await fx.Service.Projects.CreateAsync(keep.Id, "Instagram", cancellationToken: fx.Ct);
var export = await fx.Service.Projects.CreateAsync(drop.Id, "4K export", cancellationToken: fx.Ct);
Assert.Equal((await fx.Service.Projects.HeadAsync(instagram.Id, Ct)).ContentHash, (await fx.Service.Projects.HeadAsync(export.Id, Ct)).ContentHash);
var render = await RenderOfAsync(fx, drop, export);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var handler = (MergeJobHandler)SequenceMerger.CreateHandler(fx.Db, merger, fx.Service, new Chronos.Data.Projects.ProjectTransplant(), new FakeWorkspace());
var engine = Own(new JobEngine(fx.Db, [.. fx.Service.CreateJobHandlers(), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }));
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.SequenceEqual([("4K export", "4K export")], plan.Projects.Select(p => (p.SourceName, p.TargetName)));
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var copied = (await fx.Db.Projects.ListBySequenceAsync(keep.Id, includeDeleted: false, Ct)).Single(p => p.Name == "4K export");
var moved = (await fx.Db.Renders.GetAsync(render.Id))!;
Assert.Equal((keep.Id, (Guid?)copied.Id, copied.HeadVersionId), (moved.SequenceId, moved.ProjectId, moved.ProjectVersionId));
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
}
[Test]
public async Task AProjectTheKeptOneHoldsACopyOfIsNotCopiedAgainAndItsRendersNameTheCopy()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG")]);
var cut = await fx.Service.Projects.CreateAsync(drop.Id, "Night cut", cancellationToken: fx.Ct);
var render = await RenderOfAsync(fx, drop, cut);
// An earlier run of the merge copied it before it stopped.
var held = await new Chronos.Data.Projects.ProjectTransplant().CopyAsync(new Chronos.Data.Projects.ArchiveProjectHome(fx.Db, fx.Service.Projects, drop.Id), cut,
new Chronos.Data.Projects.ArchiveProjectHome(fx.Db, fx.Service.Projects, keep.Id), "Night cut", null, _ => ValueTask.CompletedTask, Ct);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
await engine.StartAsync(Ct);
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
Assert.Empty(plan.Projects);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
var moved = (await fx.Db.Renders.GetAsync(render.Id))!;
var head = (await fx.Db.Projects.GetAsync(held.Id, Ct))!.HeadVersionId;
Assert.Equal((keep.Id, (Guid?)held.Id, head), (moved.SequenceId, moved.ProjectId, moved.ProjectVersionId));
}
/// <summary>A render of <paramref name="project"/>'s head, its file outside the sequence's folder.</summary>
private static async Task<Render> RenderOfAsync(ArchiveFixture fx, Sequence sequence, SequenceProject project)
{
var bytes = RandomNumberGenerator.GetBytes(2048);
var file = Path.Combine(fx.Paths.Root, $"{project.Id:N}.mp4");
await File.WriteAllBytesAsync(file, bytes, fx.Ct);
var render = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = sequence.Id,
PresetName = "1080p",
OutputPath = file,
Bytes = bytes.Length,
Sha256 = Sha(bytes),
Created = DateTimeOffset.UtcNow,
ProjectId = project.Id,
ProjectVersionId = project.HeadVersionId,
};
await fx.Db.Renders.InsertAsync(render, fx.Ct);
return render;
}
[Test]
public async Task AJobWhosePlanMovedSinceTheDialogRefusesAndChangesNothing()
{
var fx = await FixtureAsync();
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var copy = await PairAsync(fx, keep.Id, drop.Id);
var merger = Merger(fx);
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
await SequenceAsync(fx, "drop", [(3, 20_000, "IMG_0003.JPG")]); // the session key files it into the other
await engine.StartAsync(Ct);
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.Count(1, await fx.Db.Frames.ListBySequenceAsync(keep.Id));
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
}
private sealed class FakeProjects : IProjectCopier
{
public List<(Guid Source, string Name)> Copied { get; } = [];
public async Task<SequenceProject> CopyAsync(Chronos.Data.Projects.IProjectHome from, SequenceProject source, Chronos.Data.Projects.IProjectHome to,
string name, TransplantStep? journal, Func<TransplantStep, ValueTask> save, CancellationToken ct)
{
await save(new TransplantStep(name, DateTimeOffset.UtcNow, null, null, 0, []));
lock (Copied) Copied.Add((source.Id, name));
return source with { Id = Guid.CreateVersion7(), Name = name };
}
}
private sealed class FakeWorkspace : IWorkspaceGuard
{
public HashSet<Guid> Unsent { get; } = [];
public Task<bool> HasUnsentAsync(Guid sequenceId, CancellationToken ct) => Task.FromResult(Unsent.Contains(sequenceId));
}
}
public sealed class SqliteSequenceMergerTests(TestContext context) : SequenceMergerTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresSequenceMergerTests(TestContext context) : SequenceMergerTests(context)
{
protected override Backend Backend => Backend.Postgres;
}