Files
Alby96andClaude Opus 5.5 f33bcac32e
CI / Build and test (push) Successful in 13m44s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 16s
Release / Build, test and publish (push) Successful in 26m34s
CI / PostgreSQL 16 integration (push) Failing after 17m36s
CI / PostgreSQL 17 integration (push) Failing after 15m36s
CI / Docker image (push) Successful in 1m12s
Release / Server image (push) Failing after 15s
feat(copies): matches between this PC and the server, Integrate and merge (owner review 6, v1.2.0) (#74)
The same time-lapse on a server and on this PC under different ids, or twice on this PC, is recognised,
compared and settled as the owner asked in owner review 6.

Matches are found after each sync refresh, through the server's new photo lookup, and after the jobs that
change photos. The default is to ask every time. Server › Matches offers:
- Keep apart, which is remembered;
- Link only and Unlink;
- the Compare dialog: photos, details line by line, projects and the plan.

Integrate completes both sides. Photos default to Union, projects are copied across, and a losing version
goes to the trash only after a hash check. Afterwards new photos move by themselves, and deletions wait for
the user.

A merge of two local sequences verifies every photo before the other sequence goes to its archive's trash.
Upload all local skips suspected matches and flags them.

Migration 0012 adds the found_copies and copy_probes tables. Jobs 25 (Integrate) and 26 (MergeSequences) are
new, and so are the server capabilities photo-lookup and upload-into.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-11-18 15:20:29 +01:00

1127 lines
66 KiB
C#

using System.Security.Cryptography;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Copies;
using Chronos.Data.Jobs;
using Chronos.Data.Projects;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Archiving;
using Chronos.Data.Tests.Sync;
using Chronos.Protocol;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Copies;
/// <summary>
/// Integrating a local sequence with a server sequence (owner review 6, #73; design §5c): the dry run (photos by policy, left
/// out, last copies, blocks, fingerprint) and the job that applies it: link, photos both ways, trash once the winner is in
/// place, the one payload on both sides, resumable at every stage.
/// </summary>
[Category("sync")]
public sealed class IntegrateTests(TestContext context)
{
private static CopyPlanContext PlanContext(bool canUpload = true, bool canFetch = true, bool canDelete = true, bool keepsProjects = false) =>
new("STUDIO-PC", "NAS", canUpload, canFetch, true, canDelete, canDelete, keepsProjects, null, new HashSet<Guid>(), new HashSet<Guid>());
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private static byte[] Jpeg(int seed, int ms) => Photos.Jpeg(Start.AddMilliseconds(ms), seed);
private static string Sha(int seed, int ms) => Sha(Jpeg(seed, ms));
private static async Task<SyncFixture> FixtureAsync(TestContext context)
{
var f = await SyncFixture.CreateAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
return f;
}
/// <summary>A local sequence filed photo by photo: (seed, capture offset in ms, file name).</summary>
private static async Task<Sequence> LocalAsync(SyncFixture f, string title, IReadOnlyList<(int Seed, int Ms, string Name)> photos)
{
Guid? id = null;
var staging = Path.Combine(f.Archive.Paths.StagingDirectory, "test");
Directory.CreateDirectory(staging);
foreach (var (seed, ms, name) in photos)
{
var bytes = Jpeg(seed, ms);
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
await File.WriteAllBytesAsync(path, bytes, f.Ct);
var result = await f.Archive.Service.Ingest.CommitAsync(new IngestRequest(f.Archive.Archive.Id, id, id is null ? title : null, null, path, name,
bytes.Length, Sha(bytes), null, null), f.Ct);
id = result.SequenceId;
}
return (await f.Db.Sequences.GetAsync(id!.Value, f.Ct))!;
}
private static FakeSequence Remote(SyncFixture f, string title, IReadOnlyList<(int Seed, int Ms, string Name)> photos)
{
var sequence = f.Remote.Add(title, []);
foreach (var (seed, ms, name) in photos) sequence.AddFrame(Jpeg(seed, ms), name);
return sequence;
}
private static Task<FoundCopy> PairAsync(SyncFixture f, Guid localId, Guid remoteId) =>
f.Sync.Copies.UpsertFoundAsync(new FoundCopy
{
Id = Guid.NewGuid(),
Kind = CopyKind.LocalServer,
LocalId = localId,
Server = f.Server,
RemoteId = remoteId,
Evidence = CopyEvidence.Hashes,
Fingerprint = "f",
}, f.Ct);
/// <summary>Choices with every detail line on this PC's side but those in <paramref name="other"/>.</summary>
private static async Task<IntegrateChoices> ChoicesAsync(SyncFixture f, Guid localId, Guid remoteId, CopyPhotoPolicy policy,
IReadOnlyDictionary<string, PhotoChoice>? perPhoto = null, params string[] other)
{
var sides = new CopySides(f.Db);
var here = await sides.LocalAsync(localId, f.Ct);
var there = await sides.RemoteAsync(f.Remote, remoteId, f.Ct);
var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
var local = (await f.Db.Sequences.GetAsync(localId, f.Ct))!;
var manifest = await f.Remote.GetManifestAsync(remoteId, f.Ct);
var lines = CopyPayload.Lines(SyncPayload.Of(local), PhotosOf(here, tombstones),
SyncPayload.Of(await f.Remote.GetSequenceAsync(remoteId, f.Ct), manifest.SettingsJson), PhotosOf(there, tombstones));
var details = lines.Select(l => l.Path).Distinct().ToDictionary(p => p, p => other.Contains(p) ? CopySide.Other : CopySide.ThisPc);
return new IntegrateChoices(policy, perPhoto ?? new Dictionary<string, PhotoChoice>(), details, CopyProjects.CopyAcross);
static FramePhotos PhotosOf(CopySideInput side, SyncTombstones tombstones) =>
new(side.Photos.Select(p => (p.Index, tombstones.Canonical(new FramePhoto(p.Sha256, p.Size, p.CapturedAt)))));
}
private static IntegratePlanner Planner(SyncFixture f) => new(f.Sync, new CopySides(f.Db), f.Archive.Service.Projects);
private static (JobEngine Engine, IntegrateJobHandler Handler) Engine(SyncFixture f)
{
var connector = new SyncFixture.Connector(f.Remote);
var handler = (IntegrateJobHandler)CopyJobs.CreateIntegrateHandler(f.Sync, f.Archive.Service, connector, f.Sync.Copies);
var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
return (engine, handler);
}
private static async Task<HashSet<string>> LocalHashesAsync(SyncFixture f, Guid id) =>
(await f.Db.Frames.ListBySequenceAsync(id, cancellationToken: f.Ct)).Select(fr => fr.Sha256!).ToHashSet(StringComparer.Ordinal);
private static HashSet<string> RemoteHashes(FakeSequence remote) => remote.Frames.Select(fr => fr.Sha256).ToHashSet(StringComparer.Ordinal);
/// <summary>
/// A pair with every kind of photo, seeds from <paramref name="n"/>: photo 1 on both sides, 2 only here, 3 only there, 4
/// here and 5 there at one sub-second instant under one name (different), and different titles.
/// </summary>
private static async Task<(Sequence Local, FakeSequence Remote, FoundCopy Pair)> ScenarioAsync(SyncFixture f, int n = 0)
{
var b = 100 * n;
var local = await LocalAsync(f, $"Comet {n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 2, 10_000, "IMG_0002.JPG"), (b + 4, 30_250, "IMG_0004.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, $"Comet {n} NAS", [(b + 1, 0, names[0]), (b + 3, 20_000, "IMG_0003.JPG"), (b + 5, 30_250, names[2])]);
return (local, remote, await PairAsync(f, local.Id, remote.Id));
}
/// <summary>The names the archive filed the local photos under (a Differs pair needs one name on both sides).</summary>
private static async Task<List<string>> NamesAsync(SyncFixture f, Guid localId) =>
[.. (await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct)).OrderBy(fr => fr.Index).Select(fr => fr.FileName)];
[Test]
public async Task EachPolicyPlansItsStepsAndOneSideWinsStillBringsWhatEachLacks()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var planner = Planner(f);
var (here, there, differs) = (Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250));
async Task<IntegratePlan> PlanAsync(CopyPhotoPolicy policy, IReadOnlyDictionary<string, PhotoChoice>? perPhoto = null) =>
await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, policy, perPhoto), PlanContext(), f.Ct);
static List<string> Steps(IntegratePlan plan) => [.. plan.Photos.Select(p => $"{p.Action}:{p.Sha256}").Order(StringComparer.Ordinal)];
var union = await PlanAsync(CopyPhotoPolicy.Union);
Assert.True(union.CanApply, string.Join(", ", union.Blocks));
Assert.SequenceEqual(new[] { $"Fetch:{there}", $"FetchVersion:{Sha(5, 30_250)}", $"Send:{here}", $"SendVersion:{differs}" }.Order(StringComparer.Ordinal), Steps(union));
Assert.Equal((0, true, false), (union.LeftOut, union.AutoPhotos, union.Destructive));
Assert.NotNull(union.Payload);
Assert.Empty(union.LastCopies);
var mine = await PlanAsync(CopyPhotoPolicy.ThisPcWins);
Assert.SequenceEqual(new[] { $"Fetch:{there}", $"Send:{here}", $"SendVersion:{differs}", $"TrashThere:{Sha(5, 30_250)}" }.Order(StringComparer.Ordinal), Steps(mine));
Assert.True(mine.Destructive);
Assert.Equal((Sha(5, 30_250), CopySide.Other), (mine.LastCopies.Single().Key, mine.LastCopies.Single().Side));
var theirs = await PlanAsync(CopyPhotoPolicy.OtherWins);
Assert.SequenceEqual(new[] { $"Fetch:{there}", $"FetchVersion:{Sha(5, 30_250)}", $"Send:{here}", $"TrashHere:{differs}" }.Order(StringComparer.Ordinal), Steps(theirs));
Assert.Equal((differs, CopySide.ThisPc), (theirs.LastCopies.Single().Key, theirs.LastCopies.Single().Side));
Assert.NotEqual(union.Fingerprint, mine.Fingerprint);
Assert.NotEqual(mine.Fingerprint, theirs.Fingerprint);
// Again: the same plan, the same fingerprint.
Assert.Equal(mine.Fingerprint, (await PlanAsync(CopyPhotoPolicy.ThisPcWins)).Fingerprint);
}
[Test]
public async Task WhatIsLeftOutIsCountedAndTurnsAutoPhotosOff()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var planner = Planner(f);
var nothing = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ChoosePerPhoto), PlanContext(), f.Ct);
Assert.Empty(nothing.Photos);
Assert.Equal((3, false), (nothing.LeftOut, nothing.AutoPhotos));
var one = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ChoosePerPhoto,
new Dictionary<string, PhotoChoice> { [Sha(2, 10_000)] = PhotoChoice.Bring, [Sha(4, 30_250)] = PhotoChoice.KeepBoth }), PlanContext(), f.Ct);
Assert.Equal(3, one.Photos.Count);
Assert.Equal(1, one.LeftOut);
// Without the Delete right the loser on the server stays: left out, and no trash step.
var kept = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(canDelete: false), f.Ct);
Assert.False(kept.Destructive);
Assert.Equal((1, false), (kept.LeftOut, kept.AutoPhotos));
Assert.True(kept.CanApply);
}
[Test]
public async Task ALoserHeldElsewhereIsNotALastCopy()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
// Another sequence of this index holds the server's losing version, and the server holds this PC's elsewhere.
await LocalAsync(f, "Elsewhere", [(5, 30_250, "IMG_0004.JPG")]);
f.Remote.Add("Other", [Jpeg(4, 30_250)]);
var planner = Planner(f);
var mine = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
Assert.True(mine.Destructive);
Assert.Empty(mine.LastCopies);
}
[Test]
public async Task BlocksSayWhatStopsIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var planner = Planner(f);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union);
// An older server.
var plan = await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
Assert.Contains(CopyBlock.NeedsNewerServer, plan.Blocks);
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
Assert.True((await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).CanApply);
// Rights.
plan = await planner.PlanAsync(f.Remote, pair, choices, PlanContext(canUpload: false, canFetch: false), f.Ct);
Assert.Contains(CopyBlock.NoUploadRight, plan.Blocks);
Assert.Contains(CopyBlock.NoFetchRight, plan.Blocks);
// Photos sent into a server sequence the account neither owns nor may edit: the server would refuse them (403).
var readUpload = PlanContext(canDelete: false) with { CanEdit = false, UserId = Guid.NewGuid() };
Assert.Contains(CopyBlock.NoAddRight, (await planner.PlanAsync(f.Remote, pair, choices, readUpload, f.Ct)).Blocks);
Assert.Equal(Loc.L("Errors.Copies.NoAddRight"), IntegratePlanner.BlockText(plan, CopyBlock.NoAddRight, "Comet", "NAS"));
remote.OwnerUserId = readUpload.UserId;
Assert.DoesNotContain(CopyBlock.NoAddRight, (await planner.PlanAsync(f.Remote, pair, choices, readUpload, f.Ct)).Blocks);
Assert.DoesNotContain(CopyBlock.NoAddRight, (await planner.PlanAsync(f.Remote, pair, choices, PlanContext(canDelete: false), f.Ct)).Blocks);
remote.OwnerUserId = null;
// A detail without a side.
plan = await planner.PlanAsync(f.Remote, pair, choices with { Details = new Dictionary<string, CopySide>() }, PlanContext(), f.Ct);
Assert.Contains(CopyBlock.DetailsUnchosen, plan.Blocks);
// A job on the local copy.
var running = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, TargetIds = [local.Id], State = JobState.Running, Created = DateTimeOffset.UtcNow };
await f.Db.Jobs.EnqueueAsync(running, f.Ct);
Assert.Contains(CopyBlock.Busy, (await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).Blocks);
await f.Db.Jobs.SetStateAsync(running.Id, JobState.Done, null, f.Ct);
// The local copy linked to another sequence of the server.
var other = f.Remote.Add("Other", f.NewPhotos(1));
await f.Sync.LinkAsync(f.Server, other.Id, local.Id, f.Ct);
Assert.Contains(CopyBlock.LinkTaken, (await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).Blocks);
}
[Test]
public async Task WithNothingButALoserThatStaysTheDeleteRightBlocks()
{
await using var f = await FixtureAsync(context);
var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, "Comet", [(1, 0, names[0]), (5, 30_250, names[1])]);
var pair = await PairAsync(f, local.Id, remote.Id);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins);
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(canDelete: false), f.Ct);
Assert.Contains(CopyBlock.NoDeleteRight, plan.Blocks);
}
[Test]
public async Task AnEditorMayNotTrashTheVersionOfASequenceSomeoneElseOwns()
{
await using var f = await FixtureAsync(context);
var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, "Comet", [(1, 0, names[0]), (5, 30_250, names[1])]);
remote.OwnerUserId = Guid.NewGuid();
var pair = await PairAsync(f, local.Id, remote.Id);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins);
// An Editor: the Delete right, but not the admin role, and the server sequence is someone else's.
var editor = PlanContext() with { OwnsOrAdmin = false, UserId = Guid.NewGuid() };
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, editor, f.Ct);
Assert.False(plan.Photos.Any(p => p.Action == PhotoAction.TrashThere), "no trash step on the server");
// It stays, out of the server's Main, which uses this PC's version instead.
Assert.Equal((1, 1, 1), (plan.LeftOut, plan.StaysForDelete, plan.ServerUsesOurs));
Assert.Contains(CopyBlock.NoDeleteRight, plan.Blocks);
Assert.False(plan.Destructive);
// The owner may: the same plan trashes the server's version, and the server uses this PC's.
var owner = editor with { UserId = remote.OwnerUserId };
var owned = await Planner(f).PlanAsync(f.Remote, pair, choices, owner, f.Ct);
Assert.Contains(owned.Photos, p => p.Action == PhotoAction.TrashThere);
Assert.Equal((0, 1), (owned.StaysForDelete, owned.ServerUsesOurs));
// Compare offers Delete on server only where the server would allow it.
var builder = new CopyReviewBuilder(f.Db, f.Archive.Service.Projects, new CopySides(f.Db));
Assert.False((await builder.BuildAsync(pair, f.Remote, editor, f.Ct)).Can.DeleteThere);
Assert.True((await builder.BuildAsync(pair, f.Remote, owner, f.Ct)).Can.DeleteThere);
}
[Test]
public async Task TheJobOfAnOwnerWhoIsNotAnAdminPlansWhatTheDialogPlanned()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, "Comet", [(1, 0, names[0]), (5, 30_250, names[1])]);
remote.OwnerUserId = Guid.NewGuid();
var pair = await PairAsync(f, local.Id, remote.Id);
// An Editor who owns the server sequence: the Delete right, not the admin role.
var owner = PlanContext() with { OwnsOrAdmin = false, UserId = remote.OwnerUserId };
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), owner, f.Ct);
Assert.Contains(plan.Photos, p => p.Action == PhotoAction.TrashThere);
// The context the job plans with gives the same plan.
var queued = IntegrateJobContext.From(owner);
Assert.Equal(owner.UserId, queued.ToPlanContext().UserId);
Assert.Equal(plan.Fingerprint, (await Planner(f).PlanAsync(f.Remote, pair, plan.Choices, queued.ToPlanContext(), f.Ct)).Fingerprint);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(Sha(5, 30_250), f.Remote.Trashed.Single().Frame.Sha256);
Assert.True(RemoteHashes(remote).SetEquals([Sha(1, 0), Sha(4, 30_250)]));
}
[Test]
public async Task TheJobOfAnOwnerWithoutTheEditRightSendsWhatTheDialogAllowed()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, "Comet", [(1, 0, names[0])]);
remote.OwnerUserId = Guid.NewGuid();
var pair = await PairAsync(f, local.Id, remote.Id);
// Upload but not Edit: photos may go into a sequence the account owns.
var owner = PlanContext(canDelete: false) with { CanEdit = false, UserId = remote.OwnerUserId };
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union);
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, owner, f.Ct);
Assert.Contains(plan.Photos, p => p.Action is PhotoAction.Send or PhotoAction.SendVersion);
Assert.DoesNotContain(CopyBlock.NoAddRight, plan.Blocks);
Assert.Equal(plan.Fingerprint, (await Planner(f).PlanAsync(f.Remote, pair, choices, IntegrateJobContext.From(owner).ToPlanContext(), f.Ct)).Fingerprint);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(RemoteHashes(remote).SetEquals([Sha(1, 0), Sha(2, 10_000)]));
}
[Test]
public async Task WithoutTheDeleteRightTheServersVersionStaysOutOfItsMainAndTheRestRuns()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins);
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(canDelete: false), f.Ct);
// The server's losing version stays (it needs the Delete right), but its Main uses this PC's version.
Assert.False(plan.Photos.Any(p => p.Action == PhotoAction.TrashThere), "no trash step on the server");
Assert.Equal((1, 1), (plan.StaysForDelete, plan.ServerUsesOurs));
var pending = ProjectSettings.FromJson(SyncRules.NormalizeSettings(Assert.NotNull(plan.Payload).SettingsJson)).PendingFrames ?? [];
Assert.Contains(pending, p => p.Sha256 == Sha(5, 30_250) && p.Excluded);
// Other steps remain: the plan runs, only the delete waits.
Assert.DoesNotContain(CopyBlock.NoDeleteRight, plan.Blocks);
Assert.True(plan.CanApply);
}
[Test]
public async Task UnionBringsEveryPhotoBothWaysAndIntegratesThePair()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union, null, "sequence.title");
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
Assert.Equal(CopyState.Integrating, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var all = new[] { Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250), Sha(5, 30_250) };
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals(all));
Assert.True(RemoteHashes(remote).SetEquals(all));
Assert.Equal(2, f.Remote.IntoUploads);
var stored = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
Assert.Equal(("Comet 0 NAS", "Comet 0 NAS"), (stored.Title, remote.Title));
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal((CopyState.Integrated, true, (Guid?)null), (after.State, after.AutoPhotos, after.JobId));
Assert.Equal(local.Id, (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))!.LocalId);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
}
[Test]
public async Task ACorruptPhotoOnThisPcIsRepairedFromTheServersCopy()
{
await using var f = await FixtureAsync(context);
var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, "Comet", [(1, 0, names[0]), (2, 10_000, names[1])]);
var pair = await PairAsync(f, local.Id, remote.Id);
// A verify finds the second photo damaged on this PC (design §5c: an unusable photo the other side holds is a repair).
var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(2, 10_000));
var file = f.Archive.Paths.FromRelative(damaged.RelativePath);
var bytes = await File.ReadAllBytesAsync(file, f.Ct);
bytes[^10] ^= 0xFF;
await File.WriteAllBytesAsync(file, bytes, f.Ct);
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
Assert.Equal(FrameFlags.Corrupt, (await f.Db.Frames.GetAsync(damaged.Id, f.Ct))!.Flags);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, Sha(2, 10_000))], plan.Photos.Select(p => (p.Action, p.Sha256!)));
Assert.Equal(0, plan.LeftOut);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Count(2, frames);
var repaired = frames.Single(fr => fr.Sha256 == Sha(2, 10_000));
Assert.Equal(FrameFlags.None, repaired.Flags & (FrameFlags.Corrupt | FrameFlags.Missing));
var repairedFile = f.Archive.Paths.FromRelative(repaired.RelativePath);
Assert.Equal(Sha(2, 10_000), Sha(await File.ReadAllBytesAsync(repairedFile, f.Ct)));
// The damaged file is kept beside it, never deleted.
var aside = Directory.EnumerateFiles(Path.GetDirectoryName(repairedFile)!, Path.GetFileName(repairedFile) + ".damaged-*").ToList();
Assert.Count(1, aside);
Assert.True((await File.ReadAllBytesAsync(aside[0], f.Ct)).AsSpan().SequenceEqual(bytes), "the damaged file moved aside");
Assert.Equal(0, f.Remote.IntoUploads);
Assert.Equal(CopyState.Integrated, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task AJobWhosePlanMovedRefusesAndSetsThePairBack()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var (engine, _) = Engine(f);
await using var _ = engine;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
remote.AddFrame(Jpeg(9, 50_000), "IMG_0009.JPG");
await engine.StartAsync(f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.Equal(0, f.Remote.IntoUploads);
Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal((CopyState.Candidate, (Guid?)null), (after.State, after.JobId));
Assert.Null(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct));
}
[Test]
public async Task ACrashAtEachStageResumesWhereItStopped()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var stages = Enum.GetValues<IntegrateStage>().Where(s => s != IntegrateStage.Done).ToList();
for (var i = 0; i < stages.Count; i++)
{
var n = i + 1;
var (local, remote, pair) = await ScenarioAsync(f, n);
var at = stages[i];
var crashed = false;
handler.BeforeStage = stage =>
{
if (stage != at || crashed) return Task.CompletedTask;
crashed = true;
throw new IOException($"crash before {at}");
};
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins, null, "sequence.title");
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(at == IntegrateStage.Link ? CopyState.Candidate : CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State, at.ToString());
await engine.ResumeAsync(job.Id, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var b = 100 * n;
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
Assert.True(RemoteHashes(remote).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
Assert.Equal(CopyState.Integrated, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State, at.ToString());
Assert.Equal($"Comet {n} NAS", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title, at.ToString());
await WaitIdleAsync(f, local.Id);
}
}
[Test]
public async Task TheTrashStepInsideTheJobIsNotBusyWithItsOwnJobAndWaitsForTheWinner()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
// The server's version wins: this PC's goes to its photo trash, inside the job that targets the sequence.
var (local, remote, pair) = await ScenarioAsync(f, 1);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.DoesNotContain(Sha(104, 30_250), await LocalHashesAsync(f, local.Id));
Assert.Contains(Sha(105, 30_250), await LocalHashesAsync(f, local.Id));
await WaitIdleAsync(f, local.Id);
// The winner taken away before the trash: the loser stays and the job says so.
(local, remote, pair) = await ScenarioAsync(f, 2);
var localId = local.Id;
handler.BeforeStage = async stage =>
{
if (stage != IntegrateStage.Trash) return;
var winner = (await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(205, 30_250));
File.WriteAllBytes(f.Archive.Paths.FromRelative(winner.RelativePath), [1, 2, 3]);
};
plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins), PlanContext(), f.Ct);
job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.Plural("Errors.Copies.Kept", 1), failed.Error);
Assert.Contains(Sha(204, 30_250), await LocalHashesAsync(f, local.Id));
Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task ALoserWhoseOtherHolderGoesBeforeTheTrashIsNeverTrashedUnlisted()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
// This PC's losing version (the server wins) and the server's (this PC wins), each held by a third local sequence
// when planned: neither is a last copy. That sequence's copy is flagged corrupt while the photos move.
foreach (var (n, policy, loser) in new[] { (1, CopyPhotoPolicy.OtherWins, 104), (2, CopyPhotoPolicy.ThisPcWins, 205) })
{
var (local, remote, pair) = await ScenarioAsync(f, n);
var third = await LocalAsync(f, $"Third {n}", [(loser, 30_250, "IMG_0004.JPG")]);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, policy), PlanContext(), f.Ct);
Assert.True(plan.Destructive, policy.ToString());
Assert.Empty(plan.LastCopies);
handler.BeforeStage = async stage =>
{
if (stage != IntegrateStage.Trash) return;
var held = (await f.Db.Frames.ListBySequenceAsync(third.Id, cancellationToken: f.Ct)).Single();
await f.Db.Frames.UpdateFlagsAsync([held.Id], FrameFlags.Corrupt, FrameFlags.None, f.Ct);
};
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
// Its only copy now: it stays where it was (nothing in either trash), and the journal goes so a retry plans again.
Assert.Equal(Loc.L("Errors.Copies.StalePlan"), failed.Error);
if (policy == CopyPhotoPolicy.OtherWins) Assert.Contains(Sha(loser, 30_250), await LocalHashesAsync(f, local.Id));
else Assert.Contains(Sha(loser, 30_250), RemoteHashes(remote));
Assert.Empty(f.Remote.Trashed);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
handler.BeforeStage = null;
await WaitIdleAsync(f, local.Id);
}
}
[Test]
public async Task TheServersLoserGoesToItsTrashAtTheVersionItsDryRunRead()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(Sha(5, 30_250), f.Remote.Trashed.Single().Frame.Sha256);
Assert.True(RemoteHashes(remote).SetEquals([Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250)]));
Assert.True(remote.Removed.Any(r => r.Sha256 == Sha(5, 30_250)));
// Deleted on the server: not fetched back, nor is this PC's copy of it there.
Assert.DoesNotContain(Sha(5, 30_250), await LocalHashesAsync(f, local.Id));
}
[Test]
public async Task AServerSequenceChangedAfterTheWinnerCheckKeepsItsLoser()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
// Another client changes the server sequence after the job read the manifest that showed the winner in place.
f.Remote.OnceBeforeDelete = () => remote.FramesVersion++;
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.Plural("Errors.Copies.Kept", 1), failed.Error);
Assert.Empty(f.Remote.Trashed);
Assert.Contains(Sha(5, 30_250), RemoteHashes(remote));
}
[Test]
public async Task AServerRefusalKeepsItsLoserAndTheRestIsDone()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
f.Remote.ForbidFrameDeletes = true;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.Plural("Errors.Copies.Kept", 1), failed.Error);
// The log says why in the UI language: "IMG_0004.JPG: older version kept, the server refused it (…)".
var log = Assert.NotNull(failed.LogTail);
Assert.Contains(Between(Loc.L("Archive.Log.CopyKept", "\u0001", "\u0002"), '\u0001', '\u0002'), log);
var refused = Loc.L("Archive.Log.KeptReason.Refused", "\u0001");
Assert.Contains(refused[..refused.IndexOf('\u0001', StringComparison.Ordinal)], log);
Assert.Contains(Sha(5, 30_250), RemoteHashes(remote));
Assert.Contains(Sha(4, 30_250), RemoteHashes(remote));
Assert.Contains(Sha(3, 20_000), await LocalHashesAsync(f, local.Id));
Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task TheServersOwnAnalysisMeanwhileIsAbsorbedIntoTheDetails()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
var pair = await PairAsync(f, local.Id, remote.Id);
handler.BeforeStage = stage =>
{
if (stage == IntegrateStage.Details) f.Remote.EditSettings(remote.Id, SyncFixture.Window41);
return Task.CompletedTask;
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var stored = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
Assert.Equal(("Comet", "Comet"), (stored.Title, remote.Title));
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson));
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, stored.SettingsJson));
}
[Test]
public async Task AnEditOnThisPcMeanwhileStopsTheDetailsAndKeepsThePhotos()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
var pair = await PairAsync(f, local.Id, remote.Id);
handler.BeforeStage = async stage =>
{
if (stage == IntegrateStage.Details) await f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41);
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals(photos.Select(Sha)));
Assert.True(RemoteHashes(remote).SetEquals(photos.Select(Sha)));
Assert.Equal("Comet NAS", remote.Title);
Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task ARetryAfterTheDetailsChangedPlansAgainInsteadOfFailingTheSameWay()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
var pair = await PairAsync(f, local.Id, remote.Id);
handler.BeforeStage = async stage =>
{
if (stage == IntegrateStage.Details) await f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41);
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)), "the next run plans again");
// The edit never goes back: the Retry plans again against both sides as they are now and names the new difference.
handler.BeforeStage = null;
await engine.ResumeAsync(job.Id, f.Ct);
failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.Plural("Errors.Copies.DetailsUnchosen", 1), failed.Error);
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal(CopyState.Linked, after.State);
// Compared again, the photos moved are the same on both sides and only the details differ.
var again = await Planner(f).PlanAsync(f.Remote, after, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
Assert.Empty(again.Photos);
}
[Test]
public async Task AServerOutOfReachDuringTheProjectsFailsTheJobAndTheRetryFinishesTheSameCopy()
{
await using var f = await FixtureAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.Projects);
var (local, remote, pair) = await ScenarioAsync(f);
var standIn = await LocalAsync(f, "Stand-in", [(50, 0, "IMG_0050.JPG"), (51, 5_000, "IMG_0051.JPG")]);
var projects = f.Archive.Service.Projects;
var serverHome = new ArchiveProjectHome(f.Db, projects, standIn.Id);
var localFrames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
var project = await projects.CreateAsync(local.Id, "Vertical", cancellationToken: f.Ct);
await projects.PushAsync(project.Id, Projects.ProjectServiceTests.Draft(await projects.HeadAsync(project.Id, f.Ct),
new ProjectSettings { ExcludedFrames = [1] }, localFrames), f.Ct);
var planner = new IntegratePlanner(f.Sync, new CopySides(f.Db), projects, (_, _) => serverHome);
var plan = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(keepsProjects: true), f.Ct);
Assert.SequenceEqual(["ThisPc:Vertical"], plan.Projects.Select(p => $"{p.From}:{p.TargetName}"));
var copier = new FlakyCopier(new ProjectTransplant());
var connector = new SyncFixture.Connector(f.Remote);
var handler = new IntegrateJobHandler(f.Sync, planner, copier, connector, f.Sync.Copies, f.Archive.Service);
await using var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync(f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
// Not a project kept for good: the journal and the pair's job stay for the retry.
Assert.True(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)), "a retry resumes the copy");
Assert.Equal(job.Id, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.JobId);
await engine.ResumeAsync(job.Id, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(CopyState.Integrated, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.SequenceEqual(["Vertical"], (await projects.ListAsync(standIn.Id, f.Ct)).Where(p => !p.IsMain).Select(p => p.Name));
Assert.Equal(2, copier.Calls);
}
/// <summary>A project copier whose first copy meets a server out of reach (a network error, gone a moment later).</summary>
private sealed class FlakyCopier(IProjectCopier inner) : IProjectCopier
{
public int Calls { get; private set; }
public async Task<SequenceProject> CopyAsync(IProjectHome from, SequenceProject source, IProjectHome to, string name, TransplantStep? journal,
Func<TransplantStep, ValueTask> save, CancellationToken ct)
{
if (++Calls == 1) throw new HomeUnavailableException("The server cannot be reached: connection reset");
return await inner.CopyAsync(from, source, to, name, journal, save, ct);
}
}
[Test]
public async Task AServerTitleEditMeanwhileOverAPlannedLineThisPcWinsStopsTheDetails()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
var pair = await PairAsync(f, local.Id, remote.Id);
// The title differs and the user keeps this PC's; someone renames the server's while the photos travel.
handler.BeforeStage = stage =>
{
if (stage == IntegrateStage.Photos)
{
remote.Title = "Comet (night)";
remote.SettingsVersion++;
remote.UpdatedAt = DateTimeOffset.UtcNow;
}
return Task.CompletedTask;
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
// Not absorbed over the user's choice: the Details stage stops and neither title is overwritten.
Assert.Equal(Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
Assert.Equal("Comet", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title);
Assert.Equal("Comet (night)", remote.Title);
}
/// <summary>
/// A linked pair whose details were agreed, then edited on this PC (Window41, not sent yet), with every detail line
/// decided as the server's (Window33): the plan of an Integrate the user confirmed.
/// </summary>
private static async Task<(Sequence Local, FakeSequence Remote, FoundCopy Pair, IntegratePlan Plan)> EditedHereServerChosenAsync(SyncFixture f)
{
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet", [photos[0], photos[2]], SyncFixture.Window33);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.Sync.RefreshSequencesAsync(f.Remote, [remote.Id], new SyncPolicy(AutoSyncSettings: false), f.Ct);
Assert.Equal(local.SettingsVersion, (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))!.BaseLocalSettings);
local = await f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41);
var pair = (await f.Sync.Copies.FindRemoteAsync(local.Id, f.Server, remote.Id, f.Ct))!;
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union);
Assert.NotEmpty(choices.Details);
choices = choices with { Details = choices.Details.ToDictionary(d => d.Key, _ => CopySide.Other) };
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
return (local, remote, pair, plan);
}
[Test]
public async Task ABackgroundSyncDuringTheIntegrateLeavesTheDetailsToTheUsersChoice()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, _, plan) = await EditedHereServerChosenAsync(f);
// The server fires sequenceChanged as the photos land (or the periodic refresh runs): the agent compares the pair.
handler.BeforeStage = async stage =>
{
if (stage == IntegrateStage.Details) await f.Sync.RefreshSequencesAsync(f.Remote, [remote.Id], new SyncPolicy(), f.Ct);
};
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The server's settings the user kept are on both sides: this PC's edit was not pushed over them meanwhile.
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, remote.SettingsJson), remote.SettingsJson);
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson));
}
[Test]
public async Task AServerValueThatIsThisPcsOwnOverALineDecidedAsTheServersStopsTheDetails()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair, plan) = await EditedHereServerChosenAsync(f);
// This PC's planned value reaches the server while the photos travel (a push the job did not make).
handler.BeforeStage = stage =>
{
if (stage == IntegrateStage.Photos) f.Remote.EditSettings(remote.Id, SyncFixture.Window41);
return Task.CompletedTask;
};
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
// Not absorbed as a newer edit, which would reverse the user's choice on both sides: the details stop, the photos stay.
Assert.Equal(Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.Equal(3, RemoteHashes(remote).Count);
}
[Test]
public async Task ACancelledQueuedJobSetsThePairBack()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var (engine, _) = Engine(f);
await using var _ = engine;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
Assert.Equal(job.Id, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.JobId);
await engine.CancelAsync(job.Id, f.Ct);
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal((CopyState.Candidate, (Guid?)null), (after.State, after.JobId));
Assert.Equal(0, f.Remote.IntoUploads);
// Asked again, it can be queued again.
Assert.True((await Planner(f).PlanAsync(f.Remote, after, plan.Choices, PlanContext(), f.Ct)).CanApply);
}
[Test]
public async Task ARetryAfterThePairWasAnsweredOtherwiseRefusesAndTouchesNothing()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
handler.BeforeStage = stage => stage == IntegrateStage.Photos ? throw new IOException("crash before Photos") : Task.CompletedTask;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
// The failed job still names the pair, so a retry may resume it while nobody answered otherwise.
var held = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal((CopyState.Linked, (Guid?)job.Id), (held.State, held.JobId));
// Kept apart meanwhile: the retry refuses, sends nothing and forgets its journal.
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, null, CopyState.KeptApart, new CopyAnswer(pair.Fingerprint, null, null, false, DateTimeOffset.UtcNow), f.Ct));
handler.BeforeStage = null;
var uploads = f.Remote.IntoUploads;
await engine.ResumeAsync(job.Id, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.Equal(uploads, f.Remote.IntoUploads);
Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
}
[Test]
public async Task ALinkTheUserGaveTheServerSequenceAfterTheJournalIsNeverUndoneOnResume()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
var other = await LocalAsync(f, "Comet copy", [(1, 0, "IMG_0001.JPG")]);
handler.BeforeStage = stage => stage == IntegrateStage.Link ? throw new IOException("crash before Link") : Task.CompletedTask;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.True(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
// Meanwhile the user links the server sequence to another copy of this PC.
await f.Sync.LinkAsync(f.Server, remote.Id, other.Id, f.Ct);
handler.BeforeStage = null;
var uploads = f.Remote.IntoUploads;
await engine.ResumeAsync(job.Id, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.Equal(other.Id, (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))!.LocalId);
Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct));
Assert.Equal(uploads, f.Remote.IntoUploads);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
Assert.NotEqual(CopyState.Integrating, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task ARetryAfterAFailureBeforeTheJournalRunsTheIntegrate()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
var (local, remote, pair) = await ScenarioAsync(f);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
// The plan made again is blocked: the job fails before it writes its journal.
f.Remote.Capabilities.Remove(ServerCapabilities.UploadInto);
await engine.StartAsync(f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
Assert.Equal(CopyState.Candidate, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
await engine.ResumeAsync(job.Id, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(CopyState.Integrated, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.Contains(Sha(3, 20_000), await LocalHashesAsync(f, local.Id));
}
[Test]
public async Task ProjectsAreLeftWhereTheyAreWhenTheChoiceSaysSo()
{
await using var f = await FixtureAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.Projects);
var (local, remote, pair) = await ScenarioAsync(f);
var standIn = await LocalAsync(f, "Stand-in", [(50, 0, "IMG_0050.JPG")]);
var projects = f.Archive.Service.Projects;
var serverHome = new ArchiveProjectHome(f.Db, projects, standIn.Id);
await projects.CreateAsync(local.Id, "Vertical", cancellationToken: f.Ct);
await projects.CreateAsync(standIn.Id, "Wide", cancellationToken: f.Ct);
var planner = new IntegratePlanner(f.Sync, new CopySides(f.Db), projects, (_, _) => serverHome);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union) with { Projects = CopyProjects.Leave };
// Even where the user keeps projects, Leave plans no project step.
var plan = await planner.PlanAsync(f.Remote, pair, choices, PlanContext(keepsProjects: true), f.Ct);
Assert.True(plan.CanApply);
Assert.Empty(plan.Projects);
var connector = new SyncFixture.Connector(f.Remote);
var handler = new IntegrateJobHandler(f.Sync, planner, new ProjectTransplant(), connector, f.Sync.Copies, f.Archive.Service);
await using var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync(f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
static List<string> Names(IEnumerable<SequenceProject> list) => [.. list.Where(p => !p.IsMain).Select(p => p.Name).Order(StringComparer.Ordinal)];
Assert.SequenceEqual(["Vertical"], Names(await projects.ListAsync(local.Id, f.Ct)));
Assert.SequenceEqual(["Wide"], Names(await projects.ListAsync(standIn.Id, f.Ct)));
}
[Test]
public async Task ACancelAfterTheLinkLeavesThePairLinkedAndFreeOfTheJob()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
var running = new TaskCompletionSource<Guid>(TaskCreationOptions.RunContinuationsAsynchronously);
handler.BeforeStage = async stage =>
{
if (stage == IntegrateStage.Photos) await engine.CancelAsync(await running.Task, CancellationToken.None);
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
running.SetResult(job.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Cancelled);
var until = DateTime.UtcNow.AddSeconds(10);
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
while ((after.State != CopyState.Linked || after.JobId is not null) && DateTime.UtcNow < until)
{
await Task.Delay(20, f.Ct);
after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
}
// The link the job made stays (both sides are joined); the pair no longer waits for the cancelled job.
Assert.Equal((CopyState.Linked, (Guid?)null), (after.State, after.JobId));
Assert.Equal(local.Id, (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))?.LocalId);
}
[Test]
public async Task ProjectsAreCopiedAcrossBothWaysUnderFreeNames()
{
await using var f = await FixtureAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.Projects);
var (local, remote, pair) = await ScenarioAsync(f);
// The server's projects: a home of another sequence of this index stands in for it.
var standIn = await LocalAsync(f, "Stand-in", [(50, 0, "IMG_0050.JPG"), (51, 5_000, "IMG_0051.JPG")]);
var projects = f.Archive.Service.Projects;
var serverHome = new ArchiveProjectHome(f.Db, projects, standIn.Id);
var localFrames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
var standInFrames = await f.Db.Frames.ListBySequenceAsync(standIn.Id, cancellationToken: f.Ct);
async Task<SequenceProject> ProjectAsync(Guid sequenceId, string name, ProjectSettings settings, IEnumerable<Frame> frames)
{
var project = await projects.CreateAsync(sequenceId, name, cancellationToken: f.Ct);
var head = await projects.HeadAsync(project.Id, f.Ct);
await projects.PushAsync(project.Id, Projects.ProjectServiceTests.Draft(head, settings, frames), f.Ct);
return project;
}
var mine = await ProjectAsync(local.Id, "Vertical", new ProjectSettings { ExcludedFrames = [1] }, localFrames);
await ProjectAsync(standIn.Id, "Vertical", new ProjectSettings { ExcludedFrames = [0] }, standInFrames);
await ProjectAsync(standIn.Id, "Wide", new ProjectSettings { HeroFrame = 1 }, standInFrames);
var planner = new IntegratePlanner(f.Sync, new CopySides(f.Db), projects, (_, _) => serverHome);
var plan = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(keepsProjects: true), f.Ct);
Assert.SequenceEqual(["ThisPc:Vertical (STUDIO-PC)", "Other:Vertical (NAS)", "Other:Wide"], plan.Projects.Select(p => $"{p.From}:{p.TargetName}"));
// Without the projects choice, nothing.
Assert.Empty((await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct)).Projects);
var connector = new SyncFixture.Connector(f.Remote);
var handler = new IntegrateJobHandler(f.Sync, planner, new ProjectTransplant(), connector, f.Sync.Copies, f.Archive.Service);
await using var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync(f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
static List<string> Names(IEnumerable<SequenceProject> list) => [.. list.Where(p => !p.IsMain).Select(p => p.Name).Order(StringComparer.Ordinal)];
Assert.SequenceEqual(["Vertical", "Vertical (NAS)", "Wide"], Names(await projects.ListAsync(local.Id, f.Ct)));
Assert.SequenceEqual(["Vertical", "Vertical (STUDIO-PC)", "Wide"], Names(await projects.ListAsync(standIn.Id, f.Ct)));
var copied = (await projects.ListAsync(standIn.Id, f.Ct)).Single(p => p.Name == "Vertical (STUDIO-PC)");
Assert.Equal((await projects.HeadAsync(mine.Id, f.Ct)).ContentHash, (await projects.HeadAsync(copied.Id, f.Ct)).ContentHash);
}
[Test]
public async Task TombstonesAreHonouredBothWays()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
// The server deleted photo 2 (only here otherwise); this PC deleted photo 3 (only there otherwise).
remote.Removed.Add(new RemovedPhotoDto(Sha(2, 10_000), null, "deleted", DateTimeOffset.UtcNow));
var three = Jpeg(3, 20_000);
await f.AddLocalAsync(local.Id, three);
var frame = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(three));
var trash = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [frame.Id], null, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(trash, null, null, f.Ct);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
Assert.False(plan.Photos.Any(p => p.Action is PhotoAction.Send && p.Sha256 == Sha(2, 10_000)));
Assert.False(plan.Photos.Any(p => p.Action is PhotoAction.Fetch && p.Sha256 == Sha(3, 20_000)));
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.DoesNotContain(Sha(2, 10_000), RemoteHashes(remote));
Assert.DoesNotContain(Sha(3, 20_000), await LocalHashesAsync(f, local.Id));
Assert.Contains(Sha(5, 30_250), await LocalHashesAsync(f, local.Id));
}
/// <summary>Waits until no job works on <paramref name="sequenceId"/> (the passes an Integrate queues after its photos).</summary>
/// <summary>The fixed text of a localized line between two placeholders' markers.</summary>
private static string Between(string text, char from, char to)
{
var start = text.IndexOf(from, StringComparison.Ordinal) + 1;
return text[start..text.IndexOf(to, start)];
}
private static async Task WaitIdleAsync(SyncFixture f, Guid sequenceId)
{
var until = DateTime.UtcNow.AddSeconds(30);
while (DateTime.UtcNow < until)
{
var held = await f.Db.Jobs.ListPendingTargetsAsync(JobKind.Thumbs, f.Ct);
held.UnionWith(await f.Db.Jobs.ListPendingTargetsAsync(JobKind.LightAnalyse, f.Ct));
if (!held.Contains(sequenceId)) return;
await Task.Delay(20, f.Ct);
}
}
}