Files
Chronos/tests/Chronos.Data.Tests/Copies/IntegrateTests.cs
T
Alby96andClaude Opus 5.5 461dfcf4a3 feat(copies): integrate local and server, filtered transfers, project transplant (owner review 6, P3)
A local sequence and its server twin can now be integrated into one linked
pair by a resumable job instead of by hand. The planner runs the dry run of
design 5c: photo steps per policy (the wins policies decide only the Different
class; Only here and Only there always move), left out and AutoPhotos, last
copies, the one portable payload, projects copied across under free names,
space, rights and every block, and a fingerprint the job checks again.

The Integrate job keeps an atomic journal and resumes at each stage: link,
photos both ways through filtered transfers that never create or retarget a
server sequence (a 404 marks the pair remote gone), trash only once the winner
is present (locally through the photo trash inside the job, on the server at
the version its dry run read), the agreed details, then projects through
ProjectTransplant. Refusals are kept and reported; a failure or a cancel sets
the pair back to Linked or its previous state.

Linked uploads now file into their target when the server announces
upload-into. The plan context travels with the job so rights and names give
the same plan on the job side.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 11:24:18 +01:00

575 lines
32 KiB
C#

using System.Security.Cryptography;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Copies;
using Chronos.Data.Jobs;
using Chronos.Data.Projects;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Archiving;
using Chronos.Data.Tests.Sync;
using Chronos.Protocol;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Copies;
/// <summary>
/// Integrating a local sequence with a server sequence (owner review 6, #73; design §5c): the dry run (photos by policy, left
/// out, last copies, blocks, fingerprint) and the job that applies it: link, photos both ways, trash once the winner is in
/// place, the one payload on both sides, resumable at every stage.
/// </summary>
[Category("sync")]
public sealed class IntegrateTests(TestContext context)
{
private static CopyPlanContext PlanContext(bool canUpload = true, bool canFetch = true, bool canDelete = true, bool keepsProjects = false) =>
new("STUDIO-PC", "NAS", canUpload, canFetch, true, canDelete, canDelete, keepsProjects, null, new HashSet<Guid>(), new HashSet<Guid>());
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private static byte[] Jpeg(int seed, int ms) => Photos.Jpeg(Start.AddMilliseconds(ms), seed);
private static string Sha(int seed, int ms) => Sha(Jpeg(seed, ms));
private static async Task<SyncFixture> FixtureAsync(TestContext context)
{
var f = await SyncFixture.CreateAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
return f;
}
/// <summary>A local sequence filed photo by photo: (seed, capture offset in ms, file name).</summary>
private static async Task<Sequence> LocalAsync(SyncFixture f, string title, IReadOnlyList<(int Seed, int Ms, string Name)> photos)
{
Guid? id = null;
var staging = Path.Combine(f.Archive.Paths.StagingDirectory, "test");
Directory.CreateDirectory(staging);
foreach (var (seed, ms, name) in photos)
{
var bytes = Jpeg(seed, ms);
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
await File.WriteAllBytesAsync(path, bytes, f.Ct);
var result = await f.Archive.Service.Ingest.CommitAsync(new IngestRequest(f.Archive.Archive.Id, id, id is null ? title : null, null, path, name,
bytes.Length, Sha(bytes), null, null), f.Ct);
id = result.SequenceId;
}
return (await f.Db.Sequences.GetAsync(id!.Value, f.Ct))!;
}
private static FakeSequence Remote(SyncFixture f, string title, IReadOnlyList<(int Seed, int Ms, string Name)> photos)
{
var sequence = f.Remote.Add(title, []);
foreach (var (seed, ms, name) in photos) sequence.AddFrame(Jpeg(seed, ms), name);
return sequence;
}
private static Task<FoundCopy> PairAsync(SyncFixture f, Guid localId, Guid remoteId) =>
f.Sync.Copies.UpsertFoundAsync(new FoundCopy
{
Id = Guid.NewGuid(),
Kind = CopyKind.LocalServer,
LocalId = localId,
Server = f.Server,
RemoteId = remoteId,
Evidence = CopyEvidence.Hashes,
Fingerprint = "f",
}, f.Ct);
/// <summary>Choices with every detail line on this PC's side but those in <paramref name="other"/>.</summary>
private static async Task<IntegrateChoices> ChoicesAsync(SyncFixture f, Guid localId, Guid remoteId, CopyPhotoPolicy policy,
IReadOnlyDictionary<string, PhotoChoice>? perPhoto = null, params string[] other)
{
var sides = new CopySides(f.Db);
var here = await sides.LocalAsync(localId, f.Ct);
var there = await sides.RemoteAsync(f.Remote, remoteId, f.Ct);
var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
var local = (await f.Db.Sequences.GetAsync(localId, f.Ct))!;
var manifest = await f.Remote.GetManifestAsync(remoteId, f.Ct);
var lines = CopyPayload.Lines(SyncPayload.Of(local), PhotosOf(here, tombstones),
SyncPayload.Of(await f.Remote.GetSequenceAsync(remoteId, f.Ct), manifest.SettingsJson), PhotosOf(there, tombstones));
var details = lines.Select(l => l.Path).Distinct().ToDictionary(p => p, p => other.Contains(p) ? CopySide.Other : CopySide.ThisPc);
return new IntegrateChoices(policy, perPhoto ?? new Dictionary<string, PhotoChoice>(), details, CopyProjects.CopyAcross);
static FramePhotos PhotosOf(CopySideInput side, SyncTombstones tombstones) =>
new(side.Photos.Select(p => (p.Index, tombstones.Canonical(new FramePhoto(p.Sha256, p.Size, p.CapturedAt)))));
}
private static IntegratePlanner Planner(SyncFixture f) => new(f.Sync, new CopySides(f.Db), f.Archive.Service.Projects);
private static (JobEngine Engine, IntegrateJobHandler Handler) Engine(SyncFixture f)
{
var connector = new SyncFixture.Connector(f.Remote);
var handler = (IntegrateJobHandler)CopyJobs.CreateIntegrateHandler(f.Sync, f.Archive.Service, connector, f.Sync.Copies);
var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
return (engine, handler);
}
private static async Task<HashSet<string>> LocalHashesAsync(SyncFixture f, Guid id) =>
(await f.Db.Frames.ListBySequenceAsync(id, cancellationToken: f.Ct)).Select(fr => fr.Sha256!).ToHashSet(StringComparer.Ordinal);
private static HashSet<string> RemoteHashes(FakeSequence remote) => remote.Frames.Select(fr => fr.Sha256).ToHashSet(StringComparer.Ordinal);
/// <summary>
/// A pair with every kind of photo, seeds from <paramref name="n"/>: photo 1 on both sides, 2 only here, 3 only there, 4
/// here and 5 there at one sub-second instant under one name (different), and different titles.
/// </summary>
private static async Task<(Sequence Local, FakeSequence Remote, FoundCopy Pair)> ScenarioAsync(SyncFixture f, int n = 0)
{
var b = 100 * n;
var local = await LocalAsync(f, $"Comet {n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 2, 10_000, "IMG_0002.JPG"), (b + 4, 30_250, "IMG_0004.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, $"Comet {n} NAS", [(b + 1, 0, names[0]), (b + 3, 20_000, "IMG_0003.JPG"), (b + 5, 30_250, names[2])]);
return (local, remote, await PairAsync(f, local.Id, remote.Id));
}
/// <summary>The names the archive filed the local photos under (a Differs pair needs one name on both sides).</summary>
private static async Task<List<string>> NamesAsync(SyncFixture f, Guid localId) =>
[.. (await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct)).OrderBy(fr => fr.Index).Select(fr => fr.FileName)];
[Test]
public async Task EachPolicyPlansItsStepsAndOneSideWinsStillBringsWhatEachLacks()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var planner = Planner(f);
var (here, there, differs) = (Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250));
async Task<IntegratePlan> PlanAsync(CopyPhotoPolicy policy, IReadOnlyDictionary<string, PhotoChoice>? perPhoto = null) =>
await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, policy, perPhoto), PlanContext(), f.Ct);
static List<string> Steps(IntegratePlan plan) => [.. plan.Photos.Select(p => $"{p.Action}:{p.Sha256}").Order(StringComparer.Ordinal)];
var union = await PlanAsync(CopyPhotoPolicy.Union);
Assert.True(union.CanApply, string.Join(", ", union.Blocks));
Assert.SequenceEqual(new[] { $"Fetch:{there}", $"FetchVersion:{Sha(5, 30_250)}", $"Send:{here}", $"SendVersion:{differs}" }.Order(StringComparer.Ordinal), Steps(union));
Assert.Equal((0, true, false), (union.LeftOut, union.AutoPhotos, union.Destructive));
Assert.NotNull(union.Payload);
Assert.Empty(union.LastCopies);
var mine = await PlanAsync(CopyPhotoPolicy.ThisPcWins);
Assert.SequenceEqual(new[] { $"Fetch:{there}", $"Send:{here}", $"SendVersion:{differs}", $"TrashThere:{Sha(5, 30_250)}" }.Order(StringComparer.Ordinal), Steps(mine));
Assert.True(mine.Destructive);
Assert.Equal((Sha(5, 30_250), CopySide.Other), (mine.LastCopies.Single().Key, mine.LastCopies.Single().Side));
var theirs = await PlanAsync(CopyPhotoPolicy.OtherWins);
Assert.SequenceEqual(new[] { $"Fetch:{there}", $"FetchVersion:{Sha(5, 30_250)}", $"Send:{here}", $"TrashHere:{differs}" }.Order(StringComparer.Ordinal), Steps(theirs));
Assert.Equal((differs, CopySide.ThisPc), (theirs.LastCopies.Single().Key, theirs.LastCopies.Single().Side));
Assert.NotEqual(union.Fingerprint, mine.Fingerprint);
Assert.NotEqual(mine.Fingerprint, theirs.Fingerprint);
// Again: the same plan, the same fingerprint.
Assert.Equal(mine.Fingerprint, (await PlanAsync(CopyPhotoPolicy.ThisPcWins)).Fingerprint);
}
[Test]
public async Task WhatIsLeftOutIsCountedAndTurnsAutoPhotosOff()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var planner = Planner(f);
var nothing = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ChoosePerPhoto), PlanContext(), f.Ct);
Assert.Empty(nothing.Photos);
Assert.Equal((3, false), (nothing.LeftOut, nothing.AutoPhotos));
var one = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ChoosePerPhoto,
new Dictionary<string, PhotoChoice> { [Sha(2, 10_000)] = PhotoChoice.Bring, [Sha(4, 30_250)] = PhotoChoice.KeepBoth }), PlanContext(), f.Ct);
Assert.Equal(3, one.Photos.Count);
Assert.Equal(1, one.LeftOut);
// Without the Delete right the loser on the server stays: left out, and no trash step.
var kept = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(canDelete: false), f.Ct);
Assert.False(kept.Destructive);
Assert.Equal((1, false), (kept.LeftOut, kept.AutoPhotos));
Assert.True(kept.CanApply);
}
[Test]
public async Task ALoserHeldElsewhereIsNotALastCopy()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
// Another sequence of this index holds the server's losing version, and the server holds this PC's elsewhere.
await LocalAsync(f, "Elsewhere", [(5, 30_250, "IMG_0004.JPG")]);
f.Remote.Add("Other", [Jpeg(4, 30_250)]);
var planner = Planner(f);
var mine = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
Assert.True(mine.Destructive);
Assert.Empty(mine.LastCopies);
}
[Test]
public async Task BlocksSayWhatStopsIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var planner = Planner(f);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union);
// An older server.
var plan = await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
Assert.Contains(CopyBlock.NeedsNewerServer, plan.Blocks);
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
Assert.True((await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).CanApply);
// Rights.
plan = await planner.PlanAsync(f.Remote, pair, choices, PlanContext(canUpload: false, canFetch: false), f.Ct);
Assert.Contains(CopyBlock.NoUploadRight, plan.Blocks);
Assert.Contains(CopyBlock.NoFetchRight, plan.Blocks);
// A detail without a side.
plan = await planner.PlanAsync(f.Remote, pair, choices with { Details = new Dictionary<string, CopySide>() }, PlanContext(), f.Ct);
Assert.Contains(CopyBlock.DetailsUnchosen, plan.Blocks);
// A job on the local copy.
var running = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, TargetIds = [local.Id], State = JobState.Running, Created = DateTimeOffset.UtcNow };
await f.Db.Jobs.EnqueueAsync(running, f.Ct);
Assert.Contains(CopyBlock.Busy, (await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).Blocks);
await f.Db.Jobs.SetStateAsync(running.Id, JobState.Done, null, f.Ct);
// The local copy linked to another sequence of the server.
var other = f.Remote.Add("Other", f.NewPhotos(1));
await f.Sync.LinkAsync(f.Server, other.Id, local.Id, f.Ct);
Assert.Contains(CopyBlock.LinkTaken, (await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).Blocks);
}
[Test]
public async Task WithNothingButALoserThatStaysTheDeleteRightBlocks()
{
await using var f = await FixtureAsync(context);
var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, "Comet", [(1, 0, names[0]), (5, 30_250, names[1])]);
var pair = await PairAsync(f, local.Id, remote.Id);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins);
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(canDelete: false), f.Ct);
Assert.Contains(CopyBlock.NoDeleteRight, plan.Blocks);
}
[Test]
public async Task UnionBringsEveryPhotoBothWaysAndIntegratesThePair()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union, null, "sequence.title");
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
Assert.Equal(CopyState.Integrating, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var all = new[] { Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250), Sha(5, 30_250) };
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals(all));
Assert.True(RemoteHashes(remote).SetEquals(all));
Assert.Equal(2, f.Remote.IntoUploads);
var stored = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
Assert.Equal(("Comet 0 NAS", "Comet 0 NAS"), (stored.Title, remote.Title));
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal((CopyState.Integrated, true, (Guid?)null), (after.State, after.AutoPhotos, after.JobId));
Assert.Equal(local.Id, (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))!.LocalId);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
}
[Test]
public async Task AJobWhosePlanMovedRefusesAndSetsThePairBack()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var (engine, _) = Engine(f);
await using var _ = engine;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
remote.AddFrame(Jpeg(9, 50_000), "IMG_0009.JPG");
await engine.StartAsync(f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.Equal(0, f.Remote.IntoUploads);
Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal((CopyState.Candidate, (Guid?)null), (after.State, after.JobId));
Assert.Null(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct));
}
[Test]
public async Task ACrashAtEachStageResumesWhereItStopped()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var stages = Enum.GetValues<IntegrateStage>().Where(s => s != IntegrateStage.Done).ToList();
for (var i = 0; i < stages.Count; i++)
{
var n = i + 1;
var (local, remote, pair) = await ScenarioAsync(f, n);
var at = stages[i];
var crashed = false;
handler.BeforeStage = stage =>
{
if (stage != at || crashed) return Task.CompletedTask;
crashed = true;
throw new IOException($"crash before {at}");
};
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins, null, "sequence.title");
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(at == IntegrateStage.Link ? CopyState.Candidate : CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State, at.ToString());
await engine.ResumeAsync(job.Id, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var b = 100 * n;
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
Assert.True(RemoteHashes(remote).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
Assert.Equal(CopyState.Integrated, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State, at.ToString());
Assert.Equal($"Comet {n} NAS", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title, at.ToString());
await WaitIdleAsync(f, local.Id);
}
}
[Test]
public async Task TheTrashStepInsideTheJobIsNotBusyWithItsOwnJobAndWaitsForTheWinner()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
// The server's version wins: this PC's goes to its photo trash, inside the job that targets the sequence.
var (local, remote, pair) = await ScenarioAsync(f, 1);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.DoesNotContain(Sha(104, 30_250), await LocalHashesAsync(f, local.Id));
Assert.Contains(Sha(105, 30_250), await LocalHashesAsync(f, local.Id));
await WaitIdleAsync(f, local.Id);
// The winner taken away before the trash: the loser stays and the job says so.
(local, remote, pair) = await ScenarioAsync(f, 2);
var localId = local.Id;
handler.BeforeStage = async stage =>
{
if (stage != IntegrateStage.Trash) return;
var winner = (await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(205, 30_250));
File.WriteAllBytes(f.Archive.Paths.FromRelative(winner.RelativePath), [1, 2, 3]);
};
plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins), PlanContext(), f.Ct);
job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.Plural("Errors.Copies.Kept", 1), failed.Error);
Assert.Contains(Sha(204, 30_250), await LocalHashesAsync(f, local.Id));
Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task TheServersLoserGoesToItsTrashAtTheVersionItsDryRunRead()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(Sha(5, 30_250), f.Remote.Trashed.Single().Frame.Sha256);
Assert.True(RemoteHashes(remote).SetEquals([Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250)]));
Assert.True(remote.Removed.Any(r => r.Sha256 == Sha(5, 30_250)));
// Deleted on the server: not fetched back, nor is this PC's copy of it there.
Assert.DoesNotContain(Sha(5, 30_250), await LocalHashesAsync(f, local.Id));
}
[Test]
public async Task AServerRefusalKeepsItsLoserAndTheRestIsDone()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
f.Remote.ForbidFrameDeletes = true;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.Plural("Errors.Copies.Kept", 1), failed.Error);
Assert.Contains(Sha(5, 30_250), RemoteHashes(remote));
Assert.Contains(Sha(4, 30_250), RemoteHashes(remote));
Assert.Contains(Sha(3, 20_000), await LocalHashesAsync(f, local.Id));
Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task TheServersOwnAnalysisMeanwhileIsAbsorbedIntoTheDetails()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
var pair = await PairAsync(f, local.Id, remote.Id);
handler.BeforeStage = stage =>
{
if (stage == IntegrateStage.Details) f.Remote.EditSettings(remote.Id, SyncFixture.Window41);
return Task.CompletedTask;
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var stored = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
Assert.Equal(("Comet", "Comet"), (stored.Title, remote.Title));
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson));
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, stored.SettingsJson));
}
[Test]
public async Task AnEditOnThisPcMeanwhileStopsTheDetailsAndKeepsThePhotos()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
var pair = await PairAsync(f, local.Id, remote.Id);
handler.BeforeStage = async stage =>
{
if (stage == IntegrateStage.Details) await f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41);
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals(photos.Select(Sha)));
Assert.True(RemoteHashes(remote).SetEquals(photos.Select(Sha)));
Assert.Equal("Comet NAS", remote.Title);
Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task ACancelledQueuedJobSetsThePairBack()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var (engine, _) = Engine(f);
await using var _ = engine;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
Assert.Equal(job.Id, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.JobId);
await engine.CancelAsync(job.Id, f.Ct);
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal((CopyState.Candidate, (Guid?)null), (after.State, after.JobId));
Assert.Equal(0, f.Remote.IntoUploads);
// Asked again, it can be queued again.
Assert.True((await Planner(f).PlanAsync(f.Remote, after, plan.Choices, PlanContext(), f.Ct)).CanApply);
}
[Test]
public async Task ProjectsAreCopiedAcrossBothWaysUnderFreeNames()
{
await using var f = await FixtureAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.Projects);
var (local, remote, pair) = await ScenarioAsync(f);
// The server's projects: a home of another sequence of this index stands in for it.
var standIn = await LocalAsync(f, "Stand-in", [(50, 0, "IMG_0050.JPG"), (51, 5_000, "IMG_0051.JPG")]);
var projects = f.Archive.Service.Projects;
var serverHome = new ArchiveProjectHome(f.Db, projects, standIn.Id);
var localFrames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
var standInFrames = await f.Db.Frames.ListBySequenceAsync(standIn.Id, cancellationToken: f.Ct);
async Task<SequenceProject> ProjectAsync(Guid sequenceId, string name, ProjectSettings settings, IEnumerable<Frame> frames)
{
var project = await projects.CreateAsync(sequenceId, name, cancellationToken: f.Ct);
var head = await projects.HeadAsync(project.Id, f.Ct);
await projects.PushAsync(project.Id, Projects.ProjectServiceTests.Draft(head, settings, frames), f.Ct);
return project;
}
var mine = await ProjectAsync(local.Id, "Vertical", new ProjectSettings { ExcludedFrames = [1] }, localFrames);
await ProjectAsync(standIn.Id, "Vertical", new ProjectSettings { ExcludedFrames = [0] }, standInFrames);
await ProjectAsync(standIn.Id, "Wide", new ProjectSettings { HeroFrame = 1 }, standInFrames);
var planner = new IntegratePlanner(f.Sync, new CopySides(f.Db), projects, (_, _) => serverHome);
var plan = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(keepsProjects: true), f.Ct);
Assert.SequenceEqual(["ThisPc:Vertical (STUDIO-PC)", "Other:Vertical (NAS)", "Other:Wide"], plan.Projects.Select(p => $"{p.From}:{p.TargetName}"));
// Without the projects choice, nothing.
Assert.Empty((await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct)).Projects);
var connector = new SyncFixture.Connector(f.Remote);
var handler = new IntegrateJobHandler(f.Sync, planner, new ProjectTransplant(), connector, f.Sync.Copies, f.Archive.Service);
await using var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync(f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
static List<string> Names(IEnumerable<SequenceProject> list) => [.. list.Where(p => !p.IsMain).Select(p => p.Name).Order(StringComparer.Ordinal)];
Assert.SequenceEqual(["Vertical", "Vertical (NAS)", "Wide"], Names(await projects.ListAsync(local.Id, f.Ct)));
Assert.SequenceEqual(["Vertical", "Vertical (STUDIO-PC)", "Wide"], Names(await projects.ListAsync(standIn.Id, f.Ct)));
var copied = (await projects.ListAsync(standIn.Id, f.Ct)).Single(p => p.Name == "Vertical (STUDIO-PC)");
Assert.Equal((await projects.HeadAsync(mine.Id, f.Ct)).ContentHash, (await projects.HeadAsync(copied.Id, f.Ct)).ContentHash);
}
[Test]
public async Task TombstonesAreHonouredBothWays()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
// The server deleted photo 2 (only here otherwise); this PC deleted photo 3 (only there otherwise).
remote.Removed.Add(new RemovedPhotoDto(Sha(2, 10_000), null, "deleted", DateTimeOffset.UtcNow));
var three = Jpeg(3, 20_000);
await f.AddLocalAsync(local.Id, three);
var frame = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(three));
var trash = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [frame.Id], null, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(trash, null, null, f.Ct);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
Assert.False(plan.Photos.Any(p => p.Action is PhotoAction.Send && p.Sha256 == Sha(2, 10_000)));
Assert.False(plan.Photos.Any(p => p.Action is PhotoAction.Fetch && p.Sha256 == Sha(3, 20_000)));
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.DoesNotContain(Sha(2, 10_000), RemoteHashes(remote));
Assert.DoesNotContain(Sha(3, 20_000), await LocalHashesAsync(f, local.Id));
Assert.Contains(Sha(5, 30_250), await LocalHashesAsync(f, local.Id));
}
/// <summary>Waits until no job works on <paramref name="sequenceId"/> (the passes an Integrate queues after its photos).</summary>
private static async Task WaitIdleAsync(SyncFixture f, Guid sequenceId)
{
var until = DateTime.UtcNow.AddSeconds(30);
while (DateTime.UtcNow < until)
{
var held = await f.Db.Jobs.ListPendingTargetsAsync(JobKind.Thumbs, f.Ct);
held.UnionWith(await f.Db.Jobs.ListPendingTargetsAsync(JobKind.LightAnalyse, f.Ct));
if (!held.Contains(sequenceId)) return;
await Task.Delay(20, f.Ct);
}
}
}