Files
Chronos/tests/Chronos.Data.Tests/Copies/IntegrateTests.cs
T
Alby96andClaude Opus 5.5 e002c583d6 fix(copies): close the tenth verification round of owner review 6
- A merge verify that finds a hashed photo the plan did not copy (held by
  a third sequence when planned) held by nothing now drops its journal and
  fails as a stale plan, so the retry plans again and lists the photo as a
  last copy instead of failing with "did not match after copying" forever.
- A comparison settles no details of a pair being integrated, and the
  Integrate's Details stage refuses a line decided as the server's whose
  server value is now this PC's own, so a background push or pull no longer
  reverses the user's line-by-line choice (owner Q4).
- A 404 for a server sequence no local copy is linked to deletes only its
  Candidate pairs; a Kept apart answer counts the hit as a linked pair does
  (two 404s GoneRecheck apart), so a moment's 404 neither re-asks nor
  auto-links a pair the user kept separate (owner Q1). Upload all local
  sends the local sequence once the deletion is confirmed.
- Download originals into a local copy the server key names and that is
  paired with the server sequence links it first and fills it as a linked
  copy: its tombstones hold and its details are compared, not overwritten.
- An upload of a sequence with a Candidate pair goes under a key of the
  job, and one that joins a server sequence this PC already knows compares
  its details instead of sending them all over the server's.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 13:17:23 +01:00

1127 lines
66 KiB
C#

using System.Security.Cryptography;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Copies;
using Chronos.Data.Jobs;
using Chronos.Data.Projects;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Archiving;
using Chronos.Data.Tests.Sync;
using Chronos.Protocol;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Copies;
/// <summary>
/// Integrating a local sequence with a server sequence (owner review 6, #73; design §5c): the dry run (photos by policy, left
/// out, last copies, blocks, fingerprint) and the job that applies it: link, photos both ways, trash once the winner is in
/// place, the one payload on both sides, resumable at every stage.
/// </summary>
[Category("sync")]
public sealed class IntegrateTests(TestContext context)
{
private static CopyPlanContext PlanContext(bool canUpload = true, bool canFetch = true, bool canDelete = true, bool keepsProjects = false) =>
new("STUDIO-PC", "NAS", canUpload, canFetch, true, canDelete, canDelete, keepsProjects, null, new HashSet<Guid>(), new HashSet<Guid>());
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private static byte[] Jpeg(int seed, int ms) => Photos.Jpeg(Start.AddMilliseconds(ms), seed);
private static string Sha(int seed, int ms) => Sha(Jpeg(seed, ms));
private static async Task<SyncFixture> FixtureAsync(TestContext context)
{
var f = await SyncFixture.CreateAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
return f;
}
/// <summary>A local sequence filed photo by photo: (seed, capture offset in ms, file name).</summary>
private static async Task<Sequence> LocalAsync(SyncFixture f, string title, IReadOnlyList<(int Seed, int Ms, string Name)> photos)
{
Guid? id = null;
var staging = Path.Combine(f.Archive.Paths.StagingDirectory, "test");
Directory.CreateDirectory(staging);
foreach (var (seed, ms, name) in photos)
{
var bytes = Jpeg(seed, ms);
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
await File.WriteAllBytesAsync(path, bytes, f.Ct);
var result = await f.Archive.Service.Ingest.CommitAsync(new IngestRequest(f.Archive.Archive.Id, id, id is null ? title : null, null, path, name,
bytes.Length, Sha(bytes), null, null), f.Ct);
id = result.SequenceId;
}
return (await f.Db.Sequences.GetAsync(id!.Value, f.Ct))!;
}
private static FakeSequence Remote(SyncFixture f, string title, IReadOnlyList<(int Seed, int Ms, string Name)> photos)
{
var sequence = f.Remote.Add(title, []);
foreach (var (seed, ms, name) in photos) sequence.AddFrame(Jpeg(seed, ms), name);
return sequence;
}
private static Task<FoundCopy> PairAsync(SyncFixture f, Guid localId, Guid remoteId) =>
f.Sync.Copies.UpsertFoundAsync(new FoundCopy
{
Id = Guid.NewGuid(),
Kind = CopyKind.LocalServer,
LocalId = localId,
Server = f.Server,
RemoteId = remoteId,
Evidence = CopyEvidence.Hashes,
Fingerprint = "f",
}, f.Ct);
/// <summary>Choices with every detail line on this PC's side but those in <paramref name="other"/>.</summary>
private static async Task<IntegrateChoices> ChoicesAsync(SyncFixture f, Guid localId, Guid remoteId, CopyPhotoPolicy policy,
IReadOnlyDictionary<string, PhotoChoice>? perPhoto = null, params string[] other)
{
var sides = new CopySides(f.Db);
var here = await sides.LocalAsync(localId, f.Ct);
var there = await sides.RemoteAsync(f.Remote, remoteId, f.Ct);
var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
var local = (await f.Db.Sequences.GetAsync(localId, f.Ct))!;
var manifest = await f.Remote.GetManifestAsync(remoteId, f.Ct);
var lines = CopyPayload.Lines(SyncPayload.Of(local), PhotosOf(here, tombstones),
SyncPayload.Of(await f.Remote.GetSequenceAsync(remoteId, f.Ct), manifest.SettingsJson), PhotosOf(there, tombstones));
var details = lines.Select(l => l.Path).Distinct().ToDictionary(p => p, p => other.Contains(p) ? CopySide.Other : CopySide.ThisPc);
return new IntegrateChoices(policy, perPhoto ?? new Dictionary<string, PhotoChoice>(), details, CopyProjects.CopyAcross);
static FramePhotos PhotosOf(CopySideInput side, SyncTombstones tombstones) =>
new(side.Photos.Select(p => (p.Index, tombstones.Canonical(new FramePhoto(p.Sha256, p.Size, p.CapturedAt)))));
}
private static IntegratePlanner Planner(SyncFixture f) => new(f.Sync, new CopySides(f.Db), f.Archive.Service.Projects);
private static (JobEngine Engine, IntegrateJobHandler Handler) Engine(SyncFixture f)
{
var connector = new SyncFixture.Connector(f.Remote);
var handler = (IntegrateJobHandler)CopyJobs.CreateIntegrateHandler(f.Sync, f.Archive.Service, connector, f.Sync.Copies);
var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
return (engine, handler);
}
private static async Task<HashSet<string>> LocalHashesAsync(SyncFixture f, Guid id) =>
(await f.Db.Frames.ListBySequenceAsync(id, cancellationToken: f.Ct)).Select(fr => fr.Sha256!).ToHashSet(StringComparer.Ordinal);
private static HashSet<string> RemoteHashes(FakeSequence remote) => remote.Frames.Select(fr => fr.Sha256).ToHashSet(StringComparer.Ordinal);
/// <summary>
/// A pair with every kind of photo, seeds from <paramref name="n"/>: photo 1 on both sides, 2 only here, 3 only there, 4
/// here and 5 there at one sub-second instant under one name (different), and different titles.
/// </summary>
private static async Task<(Sequence Local, FakeSequence Remote, FoundCopy Pair)> ScenarioAsync(SyncFixture f, int n = 0)
{
var b = 100 * n;
var local = await LocalAsync(f, $"Comet {n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 2, 10_000, "IMG_0002.JPG"), (b + 4, 30_250, "IMG_0004.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, $"Comet {n} NAS", [(b + 1, 0, names[0]), (b + 3, 20_000, "IMG_0003.JPG"), (b + 5, 30_250, names[2])]);
return (local, remote, await PairAsync(f, local.Id, remote.Id));
}
/// <summary>The names the archive filed the local photos under (a Differs pair needs one name on both sides).</summary>
private static async Task<List<string>> NamesAsync(SyncFixture f, Guid localId) =>
[.. (await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct)).OrderBy(fr => fr.Index).Select(fr => fr.FileName)];
[Test]
public async Task EachPolicyPlansItsStepsAndOneSideWinsStillBringsWhatEachLacks()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var planner = Planner(f);
var (here, there, differs) = (Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250));
async Task<IntegratePlan> PlanAsync(CopyPhotoPolicy policy, IReadOnlyDictionary<string, PhotoChoice>? perPhoto = null) =>
await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, policy, perPhoto), PlanContext(), f.Ct);
static List<string> Steps(IntegratePlan plan) => [.. plan.Photos.Select(p => $"{p.Action}:{p.Sha256}").Order(StringComparer.Ordinal)];
var union = await PlanAsync(CopyPhotoPolicy.Union);
Assert.True(union.CanApply, string.Join(", ", union.Blocks));
Assert.SequenceEqual(new[] { $"Fetch:{there}", $"FetchVersion:{Sha(5, 30_250)}", $"Send:{here}", $"SendVersion:{differs}" }.Order(StringComparer.Ordinal), Steps(union));
Assert.Equal((0, true, false), (union.LeftOut, union.AutoPhotos, union.Destructive));
Assert.NotNull(union.Payload);
Assert.Empty(union.LastCopies);
var mine = await PlanAsync(CopyPhotoPolicy.ThisPcWins);
Assert.SequenceEqual(new[] { $"Fetch:{there}", $"Send:{here}", $"SendVersion:{differs}", $"TrashThere:{Sha(5, 30_250)}" }.Order(StringComparer.Ordinal), Steps(mine));
Assert.True(mine.Destructive);
Assert.Equal((Sha(5, 30_250), CopySide.Other), (mine.LastCopies.Single().Key, mine.LastCopies.Single().Side));
var theirs = await PlanAsync(CopyPhotoPolicy.OtherWins);
Assert.SequenceEqual(new[] { $"Fetch:{there}", $"FetchVersion:{Sha(5, 30_250)}", $"Send:{here}", $"TrashHere:{differs}" }.Order(StringComparer.Ordinal), Steps(theirs));
Assert.Equal((differs, CopySide.ThisPc), (theirs.LastCopies.Single().Key, theirs.LastCopies.Single().Side));
Assert.NotEqual(union.Fingerprint, mine.Fingerprint);
Assert.NotEqual(mine.Fingerprint, theirs.Fingerprint);
// Again: the same plan, the same fingerprint.
Assert.Equal(mine.Fingerprint, (await PlanAsync(CopyPhotoPolicy.ThisPcWins)).Fingerprint);
}
[Test]
public async Task WhatIsLeftOutIsCountedAndTurnsAutoPhotosOff()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var planner = Planner(f);
var nothing = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ChoosePerPhoto), PlanContext(), f.Ct);
Assert.Empty(nothing.Photos);
Assert.Equal((3, false), (nothing.LeftOut, nothing.AutoPhotos));
var one = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ChoosePerPhoto,
new Dictionary<string, PhotoChoice> { [Sha(2, 10_000)] = PhotoChoice.Bring, [Sha(4, 30_250)] = PhotoChoice.KeepBoth }), PlanContext(), f.Ct);
Assert.Equal(3, one.Photos.Count);
Assert.Equal(1, one.LeftOut);
// Without the Delete right the loser on the server stays: left out, and no trash step.
var kept = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(canDelete: false), f.Ct);
Assert.False(kept.Destructive);
Assert.Equal((1, false), (kept.LeftOut, kept.AutoPhotos));
Assert.True(kept.CanApply);
}
[Test]
public async Task ALoserHeldElsewhereIsNotALastCopy()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
// Another sequence of this index holds the server's losing version, and the server holds this PC's elsewhere.
await LocalAsync(f, "Elsewhere", [(5, 30_250, "IMG_0004.JPG")]);
f.Remote.Add("Other", [Jpeg(4, 30_250)]);
var planner = Planner(f);
var mine = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
Assert.True(mine.Destructive);
Assert.Empty(mine.LastCopies);
}
[Test]
public async Task BlocksSayWhatStopsIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var planner = Planner(f);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union);
// An older server.
var plan = await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
Assert.Contains(CopyBlock.NeedsNewerServer, plan.Blocks);
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
Assert.True((await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).CanApply);
// Rights.
plan = await planner.PlanAsync(f.Remote, pair, choices, PlanContext(canUpload: false, canFetch: false), f.Ct);
Assert.Contains(CopyBlock.NoUploadRight, plan.Blocks);
Assert.Contains(CopyBlock.NoFetchRight, plan.Blocks);
// Photos sent into a server sequence the account neither owns nor may edit: the server would refuse them (403).
var readUpload = PlanContext(canDelete: false) with { CanEdit = false, UserId = Guid.NewGuid() };
Assert.Contains(CopyBlock.NoAddRight, (await planner.PlanAsync(f.Remote, pair, choices, readUpload, f.Ct)).Blocks);
Assert.Equal(Loc.L("Errors.Copies.NoAddRight"), IntegratePlanner.BlockText(plan, CopyBlock.NoAddRight, "Comet", "NAS"));
remote.OwnerUserId = readUpload.UserId;
Assert.DoesNotContain(CopyBlock.NoAddRight, (await planner.PlanAsync(f.Remote, pair, choices, readUpload, f.Ct)).Blocks);
Assert.DoesNotContain(CopyBlock.NoAddRight, (await planner.PlanAsync(f.Remote, pair, choices, PlanContext(canDelete: false), f.Ct)).Blocks);
remote.OwnerUserId = null;
// A detail without a side.
plan = await planner.PlanAsync(f.Remote, pair, choices with { Details = new Dictionary<string, CopySide>() }, PlanContext(), f.Ct);
Assert.Contains(CopyBlock.DetailsUnchosen, plan.Blocks);
// A job on the local copy.
var running = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, TargetIds = [local.Id], State = JobState.Running, Created = DateTimeOffset.UtcNow };
await f.Db.Jobs.EnqueueAsync(running, f.Ct);
Assert.Contains(CopyBlock.Busy, (await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).Blocks);
await f.Db.Jobs.SetStateAsync(running.Id, JobState.Done, null, f.Ct);
// The local copy linked to another sequence of the server.
var other = f.Remote.Add("Other", f.NewPhotos(1));
await f.Sync.LinkAsync(f.Server, other.Id, local.Id, f.Ct);
Assert.Contains(CopyBlock.LinkTaken, (await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).Blocks);
}
[Test]
public async Task WithNothingButALoserThatStaysTheDeleteRightBlocks()
{
await using var f = await FixtureAsync(context);
var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, "Comet", [(1, 0, names[0]), (5, 30_250, names[1])]);
var pair = await PairAsync(f, local.Id, remote.Id);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins);
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(canDelete: false), f.Ct);
Assert.Contains(CopyBlock.NoDeleteRight, plan.Blocks);
}
[Test]
public async Task AnEditorMayNotTrashTheVersionOfASequenceSomeoneElseOwns()
{
await using var f = await FixtureAsync(context);
var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, "Comet", [(1, 0, names[0]), (5, 30_250, names[1])]);
remote.OwnerUserId = Guid.NewGuid();
var pair = await PairAsync(f, local.Id, remote.Id);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins);
// An Editor: the Delete right, but not the admin role, and the server sequence is someone else's.
var editor = PlanContext() with { OwnsOrAdmin = false, UserId = Guid.NewGuid() };
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, editor, f.Ct);
Assert.False(plan.Photos.Any(p => p.Action == PhotoAction.TrashThere), "no trash step on the server");
// It stays, out of the server's Main, which uses this PC's version instead.
Assert.Equal((1, 1, 1), (plan.LeftOut, plan.StaysForDelete, plan.ServerUsesOurs));
Assert.Contains(CopyBlock.NoDeleteRight, plan.Blocks);
Assert.False(plan.Destructive);
// The owner may: the same plan trashes the server's version, and the server uses this PC's.
var owner = editor with { UserId = remote.OwnerUserId };
var owned = await Planner(f).PlanAsync(f.Remote, pair, choices, owner, f.Ct);
Assert.Contains(owned.Photos, p => p.Action == PhotoAction.TrashThere);
Assert.Equal((0, 1), (owned.StaysForDelete, owned.ServerUsesOurs));
// Compare offers Delete on server only where the server would allow it.
var builder = new CopyReviewBuilder(f.Db, f.Archive.Service.Projects, new CopySides(f.Db));
Assert.False((await builder.BuildAsync(pair, f.Remote, editor, f.Ct)).Can.DeleteThere);
Assert.True((await builder.BuildAsync(pair, f.Remote, owner, f.Ct)).Can.DeleteThere);
}
[Test]
public async Task TheJobOfAnOwnerWhoIsNotAnAdminPlansWhatTheDialogPlanned()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, "Comet", [(1, 0, names[0]), (5, 30_250, names[1])]);
remote.OwnerUserId = Guid.NewGuid();
var pair = await PairAsync(f, local.Id, remote.Id);
// An Editor who owns the server sequence: the Delete right, not the admin role.
var owner = PlanContext() with { OwnsOrAdmin = false, UserId = remote.OwnerUserId };
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), owner, f.Ct);
Assert.Contains(plan.Photos, p => p.Action == PhotoAction.TrashThere);
// The context the job plans with gives the same plan.
var queued = IntegrateJobContext.From(owner);
Assert.Equal(owner.UserId, queued.ToPlanContext().UserId);
Assert.Equal(plan.Fingerprint, (await Planner(f).PlanAsync(f.Remote, pair, plan.Choices, queued.ToPlanContext(), f.Ct)).Fingerprint);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(Sha(5, 30_250), f.Remote.Trashed.Single().Frame.Sha256);
Assert.True(RemoteHashes(remote).SetEquals([Sha(1, 0), Sha(4, 30_250)]));
}
[Test]
public async Task TheJobOfAnOwnerWithoutTheEditRightSendsWhatTheDialogAllowed()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, "Comet", [(1, 0, names[0])]);
remote.OwnerUserId = Guid.NewGuid();
var pair = await PairAsync(f, local.Id, remote.Id);
// Upload but not Edit: photos may go into a sequence the account owns.
var owner = PlanContext(canDelete: false) with { CanEdit = false, UserId = remote.OwnerUserId };
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union);
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, owner, f.Ct);
Assert.Contains(plan.Photos, p => p.Action is PhotoAction.Send or PhotoAction.SendVersion);
Assert.DoesNotContain(CopyBlock.NoAddRight, plan.Blocks);
Assert.Equal(plan.Fingerprint, (await Planner(f).PlanAsync(f.Remote, pair, choices, IntegrateJobContext.From(owner).ToPlanContext(), f.Ct)).Fingerprint);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(RemoteHashes(remote).SetEquals([Sha(1, 0), Sha(2, 10_000)]));
}
[Test]
public async Task WithoutTheDeleteRightTheServersVersionStaysOutOfItsMainAndTheRestRuns()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins);
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(canDelete: false), f.Ct);
// The server's losing version stays (it needs the Delete right), but its Main uses this PC's version.
Assert.False(plan.Photos.Any(p => p.Action == PhotoAction.TrashThere), "no trash step on the server");
Assert.Equal((1, 1), (plan.StaysForDelete, plan.ServerUsesOurs));
var pending = ProjectSettings.FromJson(SyncRules.NormalizeSettings(Assert.NotNull(plan.Payload).SettingsJson)).PendingFrames ?? [];
Assert.Contains(pending, p => p.Sha256 == Sha(5, 30_250) && p.Excluded);
// Other steps remain: the plan runs, only the delete waits.
Assert.DoesNotContain(CopyBlock.NoDeleteRight, plan.Blocks);
Assert.True(plan.CanApply);
}
[Test]
public async Task UnionBringsEveryPhotoBothWaysAndIntegratesThePair()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union, null, "sequence.title");
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
Assert.Equal(CopyState.Integrating, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var all = new[] { Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250), Sha(5, 30_250) };
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals(all));
Assert.True(RemoteHashes(remote).SetEquals(all));
Assert.Equal(2, f.Remote.IntoUploads);
var stored = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
Assert.Equal(("Comet 0 NAS", "Comet 0 NAS"), (stored.Title, remote.Title));
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal((CopyState.Integrated, true, (Guid?)null), (after.State, after.AutoPhotos, after.JobId));
Assert.Equal(local.Id, (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))!.LocalId);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
}
[Test]
public async Task ACorruptPhotoOnThisPcIsRepairedFromTheServersCopy()
{
await using var f = await FixtureAsync(context);
var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
var names = await NamesAsync(f, local.Id);
var remote = Remote(f, "Comet", [(1, 0, names[0]), (2, 10_000, names[1])]);
var pair = await PairAsync(f, local.Id, remote.Id);
// A verify finds the second photo damaged on this PC (design §5c: an unusable photo the other side holds is a repair).
var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(2, 10_000));
var file = f.Archive.Paths.FromRelative(damaged.RelativePath);
var bytes = await File.ReadAllBytesAsync(file, f.Ct);
bytes[^10] ^= 0xFF;
await File.WriteAllBytesAsync(file, bytes, f.Ct);
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
Assert.Equal(FrameFlags.Corrupt, (await f.Db.Frames.GetAsync(damaged.Id, f.Ct))!.Flags);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
Assert.SequenceEqual([(PhotoAction.Fetch, Sha(2, 10_000))], plan.Photos.Select(p => (p.Action, p.Sha256!)));
Assert.Equal(0, plan.LeftOut);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Count(2, frames);
var repaired = frames.Single(fr => fr.Sha256 == Sha(2, 10_000));
Assert.Equal(FrameFlags.None, repaired.Flags & (FrameFlags.Corrupt | FrameFlags.Missing));
var repairedFile = f.Archive.Paths.FromRelative(repaired.RelativePath);
Assert.Equal(Sha(2, 10_000), Sha(await File.ReadAllBytesAsync(repairedFile, f.Ct)));
// The damaged file is kept beside it, never deleted.
var aside = Directory.EnumerateFiles(Path.GetDirectoryName(repairedFile)!, Path.GetFileName(repairedFile) + ".damaged-*").ToList();
Assert.Count(1, aside);
Assert.True((await File.ReadAllBytesAsync(aside[0], f.Ct)).AsSpan().SequenceEqual(bytes), "the damaged file moved aside");
Assert.Equal(0, f.Remote.IntoUploads);
Assert.Equal(CopyState.Integrated, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task AJobWhosePlanMovedRefusesAndSetsThePairBack()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var (engine, _) = Engine(f);
await using var _ = engine;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
remote.AddFrame(Jpeg(9, 50_000), "IMG_0009.JPG");
await engine.StartAsync(f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.Equal(0, f.Remote.IntoUploads);
Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal((CopyState.Candidate, (Guid?)null), (after.State, after.JobId));
Assert.Null(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct));
}
[Test]
public async Task ACrashAtEachStageResumesWhereItStopped()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var stages = Enum.GetValues<IntegrateStage>().Where(s => s != IntegrateStage.Done).ToList();
for (var i = 0; i < stages.Count; i++)
{
var n = i + 1;
var (local, remote, pair) = await ScenarioAsync(f, n);
var at = stages[i];
var crashed = false;
handler.BeforeStage = stage =>
{
if (stage != at || crashed) return Task.CompletedTask;
crashed = true;
throw new IOException($"crash before {at}");
};
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins, null, "sequence.title");
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(at == IntegrateStage.Link ? CopyState.Candidate : CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State, at.ToString());
await engine.ResumeAsync(job.Id, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var b = 100 * n;
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
Assert.True(RemoteHashes(remote).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
Assert.Equal(CopyState.Integrated, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State, at.ToString());
Assert.Equal($"Comet {n} NAS", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title, at.ToString());
await WaitIdleAsync(f, local.Id);
}
}
[Test]
public async Task TheTrashStepInsideTheJobIsNotBusyWithItsOwnJobAndWaitsForTheWinner()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
// The server's version wins: this PC's goes to its photo trash, inside the job that targets the sequence.
var (local, remote, pair) = await ScenarioAsync(f, 1);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.DoesNotContain(Sha(104, 30_250), await LocalHashesAsync(f, local.Id));
Assert.Contains(Sha(105, 30_250), await LocalHashesAsync(f, local.Id));
await WaitIdleAsync(f, local.Id);
// The winner taken away before the trash: the loser stays and the job says so.
(local, remote, pair) = await ScenarioAsync(f, 2);
var localId = local.Id;
handler.BeforeStage = async stage =>
{
if (stage != IntegrateStage.Trash) return;
var winner = (await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(205, 30_250));
File.WriteAllBytes(f.Archive.Paths.FromRelative(winner.RelativePath), [1, 2, 3]);
};
plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins), PlanContext(), f.Ct);
job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.Plural("Errors.Copies.Kept", 1), failed.Error);
Assert.Contains(Sha(204, 30_250), await LocalHashesAsync(f, local.Id));
Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task ALoserWhoseOtherHolderGoesBeforeTheTrashIsNeverTrashedUnlisted()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
// This PC's losing version (the server wins) and the server's (this PC wins), each held by a third local sequence
// when planned: neither is a last copy. That sequence's copy is flagged corrupt while the photos move.
foreach (var (n, policy, loser) in new[] { (1, CopyPhotoPolicy.OtherWins, 104), (2, CopyPhotoPolicy.ThisPcWins, 205) })
{
var (local, remote, pair) = await ScenarioAsync(f, n);
var third = await LocalAsync(f, $"Third {n}", [(loser, 30_250, "IMG_0004.JPG")]);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, policy), PlanContext(), f.Ct);
Assert.True(plan.Destructive, policy.ToString());
Assert.Empty(plan.LastCopies);
handler.BeforeStage = async stage =>
{
if (stage != IntegrateStage.Trash) return;
var held = (await f.Db.Frames.ListBySequenceAsync(third.Id, cancellationToken: f.Ct)).Single();
await f.Db.Frames.UpdateFlagsAsync([held.Id], FrameFlags.Corrupt, FrameFlags.None, f.Ct);
};
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
// Its only copy now: it stays where it was (nothing in either trash), and the journal goes so a retry plans again.
Assert.Equal(Loc.L("Errors.Copies.StalePlan"), failed.Error);
if (policy == CopyPhotoPolicy.OtherWins) Assert.Contains(Sha(loser, 30_250), await LocalHashesAsync(f, local.Id));
else Assert.Contains(Sha(loser, 30_250), RemoteHashes(remote));
Assert.Empty(f.Remote.Trashed);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
handler.BeforeStage = null;
await WaitIdleAsync(f, local.Id);
}
}
[Test]
public async Task TheServersLoserGoesToItsTrashAtTheVersionItsDryRunRead()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(Sha(5, 30_250), f.Remote.Trashed.Single().Frame.Sha256);
Assert.True(RemoteHashes(remote).SetEquals([Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250)]));
Assert.True(remote.Removed.Any(r => r.Sha256 == Sha(5, 30_250)));
// Deleted on the server: not fetched back, nor is this PC's copy of it there.
Assert.DoesNotContain(Sha(5, 30_250), await LocalHashesAsync(f, local.Id));
}
[Test]
public async Task AServerSequenceChangedAfterTheWinnerCheckKeepsItsLoser()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
// Another client changes the server sequence after the job read the manifest that showed the winner in place.
f.Remote.OnceBeforeDelete = () => remote.FramesVersion++;
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.Plural("Errors.Copies.Kept", 1), failed.Error);
Assert.Empty(f.Remote.Trashed);
Assert.Contains(Sha(5, 30_250), RemoteHashes(remote));
}
[Test]
public async Task AServerRefusalKeepsItsLoserAndTheRestIsDone()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
f.Remote.ForbidFrameDeletes = true;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.Plural("Errors.Copies.Kept", 1), failed.Error);
// The log says why in the UI language: "IMG_0004.JPG: older version kept, the server refused it (…)".
var log = Assert.NotNull(failed.LogTail);
Assert.Contains(Between(Loc.L("Archive.Log.CopyKept", "\u0001", "\u0002"), '\u0001', '\u0002'), log);
var refused = Loc.L("Archive.Log.KeptReason.Refused", "\u0001");
Assert.Contains(refused[..refused.IndexOf('\u0001', StringComparison.Ordinal)], log);
Assert.Contains(Sha(5, 30_250), RemoteHashes(remote));
Assert.Contains(Sha(4, 30_250), RemoteHashes(remote));
Assert.Contains(Sha(3, 20_000), await LocalHashesAsync(f, local.Id));
Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task TheServersOwnAnalysisMeanwhileIsAbsorbedIntoTheDetails()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
var pair = await PairAsync(f, local.Id, remote.Id);
handler.BeforeStage = stage =>
{
if (stage == IntegrateStage.Details) f.Remote.EditSettings(remote.Id, SyncFixture.Window41);
return Task.CompletedTask;
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var stored = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
Assert.Equal(("Comet", "Comet"), (stored.Title, remote.Title));
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson));
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, stored.SettingsJson));
}
[Test]
public async Task AnEditOnThisPcMeanwhileStopsTheDetailsAndKeepsThePhotos()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
var pair = await PairAsync(f, local.Id, remote.Id);
handler.BeforeStage = async stage =>
{
if (stage == IntegrateStage.Details) await f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41);
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals(photos.Select(Sha)));
Assert.True(RemoteHashes(remote).SetEquals(photos.Select(Sha)));
Assert.Equal("Comet NAS", remote.Title);
Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task ARetryAfterTheDetailsChangedPlansAgainInsteadOfFailingTheSameWay()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
var pair = await PairAsync(f, local.Id, remote.Id);
handler.BeforeStage = async stage =>
{
if (stage == IntegrateStage.Details) await f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41);
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)), "the next run plans again");
// The edit never goes back: the Retry plans again against both sides as they are now and names the new difference.
handler.BeforeStage = null;
await engine.ResumeAsync(job.Id, f.Ct);
failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.Plural("Errors.Copies.DetailsUnchosen", 1), failed.Error);
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal(CopyState.Linked, after.State);
// Compared again, the photos moved are the same on both sides and only the details differ.
var again = await Planner(f).PlanAsync(f.Remote, after, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
Assert.Empty(again.Photos);
}
[Test]
public async Task AServerOutOfReachDuringTheProjectsFailsTheJobAndTheRetryFinishesTheSameCopy()
{
await using var f = await FixtureAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.Projects);
var (local, remote, pair) = await ScenarioAsync(f);
var standIn = await LocalAsync(f, "Stand-in", [(50, 0, "IMG_0050.JPG"), (51, 5_000, "IMG_0051.JPG")]);
var projects = f.Archive.Service.Projects;
var serverHome = new ArchiveProjectHome(f.Db, projects, standIn.Id);
var localFrames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
var project = await projects.CreateAsync(local.Id, "Vertical", cancellationToken: f.Ct);
await projects.PushAsync(project.Id, Projects.ProjectServiceTests.Draft(await projects.HeadAsync(project.Id, f.Ct),
new ProjectSettings { ExcludedFrames = [1] }, localFrames), f.Ct);
var planner = new IntegratePlanner(f.Sync, new CopySides(f.Db), projects, (_, _) => serverHome);
var plan = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(keepsProjects: true), f.Ct);
Assert.SequenceEqual(["ThisPc:Vertical"], plan.Projects.Select(p => $"{p.From}:{p.TargetName}"));
var copier = new FlakyCopier(new ProjectTransplant());
var connector = new SyncFixture.Connector(f.Remote);
var handler = new IntegrateJobHandler(f.Sync, planner, copier, connector, f.Sync.Copies, f.Archive.Service);
await using var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync(f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
// Not a project kept for good: the journal and the pair's job stay for the retry.
Assert.True(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)), "a retry resumes the copy");
Assert.Equal(job.Id, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.JobId);
await engine.ResumeAsync(job.Id, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(CopyState.Integrated, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.SequenceEqual(["Vertical"], (await projects.ListAsync(standIn.Id, f.Ct)).Where(p => !p.IsMain).Select(p => p.Name));
Assert.Equal(2, copier.Calls);
}
/// <summary>A project copier whose first copy meets a server out of reach (a network error, gone a moment later).</summary>
private sealed class FlakyCopier(IProjectCopier inner) : IProjectCopier
{
public int Calls { get; private set; }
public async Task<SequenceProject> CopyAsync(IProjectHome from, SequenceProject source, IProjectHome to, string name, TransplantStep? journal,
Func<TransplantStep, ValueTask> save, CancellationToken ct)
{
if (++Calls == 1) throw new HomeUnavailableException("The server cannot be reached: connection reset");
return await inner.CopyAsync(from, source, to, name, journal, save, ct);
}
}
[Test]
public async Task AServerTitleEditMeanwhileOverAPlannedLineThisPcWinsStopsTheDetails()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
var pair = await PairAsync(f, local.Id, remote.Id);
// The title differs and the user keeps this PC's; someone renames the server's while the photos travel.
handler.BeforeStage = stage =>
{
if (stage == IntegrateStage.Photos)
{
remote.Title = "Comet (night)";
remote.SettingsVersion++;
remote.UpdatedAt = DateTimeOffset.UtcNow;
}
return Task.CompletedTask;
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
// Not absorbed over the user's choice: the Details stage stops and neither title is overwritten.
Assert.Equal(Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
Assert.Equal("Comet", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title);
Assert.Equal("Comet (night)", remote.Title);
}
/// <summary>
/// A linked pair whose details were agreed, then edited on this PC (Window41, not sent yet), with every detail line
/// decided as the server's (Window33): the plan of an Integrate the user confirmed.
/// </summary>
private static async Task<(Sequence Local, FakeSequence Remote, FoundCopy Pair, IntegratePlan Plan)> EditedHereServerChosenAsync(SyncFixture f)
{
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
var remote = f.Remote.Add("Comet", [photos[0], photos[2]], SyncFixture.Window33);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.Sync.RefreshSequencesAsync(f.Remote, [remote.Id], new SyncPolicy(AutoSyncSettings: false), f.Ct);
Assert.Equal(local.SettingsVersion, (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))!.BaseLocalSettings);
local = await f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41);
var pair = (await f.Sync.Copies.FindRemoteAsync(local.Id, f.Server, remote.Id, f.Ct))!;
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union);
Assert.NotEmpty(choices.Details);
choices = choices with { Details = choices.Details.ToDictionary(d => d.Key, _ => CopySide.Other) };
var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
return (local, remote, pair, plan);
}
[Test]
public async Task ABackgroundSyncDuringTheIntegrateLeavesTheDetailsToTheUsersChoice()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, _, plan) = await EditedHereServerChosenAsync(f);
// The server fires sequenceChanged as the photos land (or the periodic refresh runs): the agent compares the pair.
handler.BeforeStage = async stage =>
{
if (stage == IntegrateStage.Details) await f.Sync.RefreshSequencesAsync(f.Remote, [remote.Id], new SyncPolicy(), f.Ct);
};
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The server's settings the user kept are on both sides: this PC's edit was not pushed over them meanwhile.
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, remote.SettingsJson), remote.SettingsJson);
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson));
}
[Test]
public async Task AServerValueThatIsThisPcsOwnOverALineDecidedAsTheServersStopsTheDetails()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair, plan) = await EditedHereServerChosenAsync(f);
// This PC's planned value reaches the server while the photos travel (a push the job did not make).
handler.BeforeStage = stage =>
{
if (stage == IntegrateStage.Photos) f.Remote.EditSettings(remote.Id, SyncFixture.Window41);
return Task.CompletedTask;
};
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
// Not absorbed as a newer edit, which would reverse the user's choice on both sides: the details stop, the photos stay.
Assert.Equal(Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.Equal(3, RemoteHashes(remote).Count);
}
[Test]
public async Task ACancelledQueuedJobSetsThePairBack()
{
await using var f = await FixtureAsync(context);
var (local, remote, pair) = await ScenarioAsync(f);
var (engine, _) = Engine(f);
await using var _ = engine;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
Assert.Equal(job.Id, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.JobId);
await engine.CancelAsync(job.Id, f.Ct);
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal((CopyState.Candidate, (Guid?)null), (after.State, after.JobId));
Assert.Equal(0, f.Remote.IntoUploads);
// Asked again, it can be queued again.
Assert.True((await Planner(f).PlanAsync(f.Remote, after, plan.Choices, PlanContext(), f.Ct)).CanApply);
}
[Test]
public async Task ARetryAfterThePairWasAnsweredOtherwiseRefusesAndTouchesNothing()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
handler.BeforeStage = stage => stage == IntegrateStage.Photos ? throw new IOException("crash before Photos") : Task.CompletedTask;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
// The failed job still names the pair, so a retry may resume it while nobody answered otherwise.
var held = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal((CopyState.Linked, (Guid?)job.Id), (held.State, held.JobId));
// Kept apart meanwhile: the retry refuses, sends nothing and forgets its journal.
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, null, CopyState.KeptApart, new CopyAnswer(pair.Fingerprint, null, null, false, DateTimeOffset.UtcNow), f.Ct));
handler.BeforeStage = null;
var uploads = f.Remote.IntoUploads;
await engine.ResumeAsync(job.Id, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.Equal(uploads, f.Remote.IntoUploads);
Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
}
[Test]
public async Task ALinkTheUserGaveTheServerSequenceAfterTheJournalIsNeverUndoneOnResume()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
var other = await LocalAsync(f, "Comet copy", [(1, 0, "IMG_0001.JPG")]);
handler.BeforeStage = stage => stage == IntegrateStage.Link ? throw new IOException("crash before Link") : Task.CompletedTask;
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.True(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
// Meanwhile the user links the server sequence to another copy of this PC.
await f.Sync.LinkAsync(f.Server, remote.Id, other.Id, f.Ct);
handler.BeforeStage = null;
var uploads = f.Remote.IntoUploads;
await engine.ResumeAsync(job.Id, f.Ct);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Copies.StalePlan"), failed.Error);
Assert.Equal(other.Id, (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))!.LocalId);
Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct));
Assert.Equal(uploads, f.Remote.IntoUploads);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
Assert.NotEqual(CopyState.Integrating, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
}
[Test]
public async Task ARetryAfterAFailureBeforeTheJournalRunsTheIntegrate()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
var (local, remote, pair) = await ScenarioAsync(f);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
// The plan made again is blocked: the job fails before it writes its journal.
f.Remote.Capabilities.Remove(ServerCapabilities.UploadInto);
await engine.StartAsync(f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
Assert.Equal(CopyState.Candidate, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
await engine.ResumeAsync(job.Id, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(CopyState.Integrated, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.Contains(Sha(3, 20_000), await LocalHashesAsync(f, local.Id));
}
[Test]
public async Task ProjectsAreLeftWhereTheyAreWhenTheChoiceSaysSo()
{
await using var f = await FixtureAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.Projects);
var (local, remote, pair) = await ScenarioAsync(f);
var standIn = await LocalAsync(f, "Stand-in", [(50, 0, "IMG_0050.JPG")]);
var projects = f.Archive.Service.Projects;
var serverHome = new ArchiveProjectHome(f.Db, projects, standIn.Id);
await projects.CreateAsync(local.Id, "Vertical", cancellationToken: f.Ct);
await projects.CreateAsync(standIn.Id, "Wide", cancellationToken: f.Ct);
var planner = new IntegratePlanner(f.Sync, new CopySides(f.Db), projects, (_, _) => serverHome);
var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union) with { Projects = CopyProjects.Leave };
// Even where the user keeps projects, Leave plans no project step.
var plan = await planner.PlanAsync(f.Remote, pair, choices, PlanContext(keepsProjects: true), f.Ct);
Assert.True(plan.CanApply);
Assert.Empty(plan.Projects);
var connector = new SyncFixture.Connector(f.Remote);
var handler = new IntegrateJobHandler(f.Sync, planner, new ProjectTransplant(), connector, f.Sync.Copies, f.Archive.Service);
await using var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync(f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
static List<string> Names(IEnumerable<SequenceProject> list) => [.. list.Where(p => !p.IsMain).Select(p => p.Name).Order(StringComparer.Ordinal)];
Assert.SequenceEqual(["Vertical"], Names(await projects.ListAsync(local.Id, f.Ct)));
Assert.SequenceEqual(["Wide"], Names(await projects.ListAsync(standIn.Id, f.Ct)));
}
[Test]
public async Task ACancelAfterTheLinkLeavesThePairLinkedAndFreeOfTheJob()
{
await using var f = await FixtureAsync(context);
var (engine, handler) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
var running = new TaskCompletionSource<Guid>(TaskCreationOptions.RunContinuationsAsynchronously);
handler.BeforeStage = async stage =>
{
if (stage == IntegrateStage.Photos) await engine.CancelAsync(await running.Task, CancellationToken.None);
};
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
running.SetResult(job.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Cancelled);
var until = DateTime.UtcNow.AddSeconds(10);
var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
while ((after.State != CopyState.Linked || after.JobId is not null) && DateTime.UtcNow < until)
{
await Task.Delay(20, f.Ct);
after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
}
// The link the job made stays (both sides are joined); the pair no longer waits for the cancelled job.
Assert.Equal((CopyState.Linked, (Guid?)null), (after.State, after.JobId));
Assert.Equal(local.Id, (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))?.LocalId);
}
[Test]
public async Task ProjectsAreCopiedAcrossBothWaysUnderFreeNames()
{
await using var f = await FixtureAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.Projects);
var (local, remote, pair) = await ScenarioAsync(f);
// The server's projects: a home of another sequence of this index stands in for it.
var standIn = await LocalAsync(f, "Stand-in", [(50, 0, "IMG_0050.JPG"), (51, 5_000, "IMG_0051.JPG")]);
var projects = f.Archive.Service.Projects;
var serverHome = new ArchiveProjectHome(f.Db, projects, standIn.Id);
var localFrames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
var standInFrames = await f.Db.Frames.ListBySequenceAsync(standIn.Id, cancellationToken: f.Ct);
async Task<SequenceProject> ProjectAsync(Guid sequenceId, string name, ProjectSettings settings, IEnumerable<Frame> frames)
{
var project = await projects.CreateAsync(sequenceId, name, cancellationToken: f.Ct);
var head = await projects.HeadAsync(project.Id, f.Ct);
await projects.PushAsync(project.Id, Projects.ProjectServiceTests.Draft(head, settings, frames), f.Ct);
return project;
}
var mine = await ProjectAsync(local.Id, "Vertical", new ProjectSettings { ExcludedFrames = [1] }, localFrames);
await ProjectAsync(standIn.Id, "Vertical", new ProjectSettings { ExcludedFrames = [0] }, standInFrames);
await ProjectAsync(standIn.Id, "Wide", new ProjectSettings { HeroFrame = 1 }, standInFrames);
var planner = new IntegratePlanner(f.Sync, new CopySides(f.Db), projects, (_, _) => serverHome);
var plan = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(keepsProjects: true), f.Ct);
Assert.SequenceEqual(["ThisPc:Vertical (STUDIO-PC)", "Other:Vertical (NAS)", "Other:Wide"], plan.Projects.Select(p => $"{p.From}:{p.TargetName}"));
// Without the projects choice, nothing.
Assert.Empty((await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct)).Projects);
var connector = new SyncFixture.Connector(f.Remote);
var handler = new IntegrateJobHandler(f.Sync, planner, new ProjectTransplant(), connector, f.Sync.Copies, f.Archive.Service);
await using var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync(f.Ct);
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
static List<string> Names(IEnumerable<SequenceProject> list) => [.. list.Where(p => !p.IsMain).Select(p => p.Name).Order(StringComparer.Ordinal)];
Assert.SequenceEqual(["Vertical", "Vertical (NAS)", "Wide"], Names(await projects.ListAsync(local.Id, f.Ct)));
Assert.SequenceEqual(["Vertical", "Vertical (STUDIO-PC)", "Wide"], Names(await projects.ListAsync(standIn.Id, f.Ct)));
var copied = (await projects.ListAsync(standIn.Id, f.Ct)).Single(p => p.Name == "Vertical (STUDIO-PC)");
Assert.Equal((await projects.HeadAsync(mine.Id, f.Ct)).ContentHash, (await projects.HeadAsync(copied.Id, f.Ct)).ContentHash);
}
[Test]
public async Task TombstonesAreHonouredBothWays()
{
await using var f = await FixtureAsync(context);
var (engine, _) = Engine(f);
await using var _ = engine;
await engine.StartAsync(f.Ct);
var (local, remote, pair) = await ScenarioAsync(f);
// The server deleted photo 2 (only here otherwise); this PC deleted photo 3 (only there otherwise).
remote.Removed.Add(new RemovedPhotoDto(Sha(2, 10_000), null, "deleted", DateTimeOffset.UtcNow));
var three = Jpeg(3, 20_000);
await f.AddLocalAsync(local.Id, three);
var frame = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(three));
var trash = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [frame.Id], null, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(trash, null, null, f.Ct);
var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
Assert.False(plan.Photos.Any(p => p.Action is PhotoAction.Send && p.Sha256 == Sha(2, 10_000)));
Assert.False(plan.Photos.Any(p => p.Action is PhotoAction.Fetch && p.Sha256 == Sha(3, 20_000)));
var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.DoesNotContain(Sha(2, 10_000), RemoteHashes(remote));
Assert.DoesNotContain(Sha(3, 20_000), await LocalHashesAsync(f, local.Id));
Assert.Contains(Sha(5, 30_250), await LocalHashesAsync(f, local.Id));
}
/// <summary>Waits until no job works on <paramref name="sequenceId"/> (the passes an Integrate queues after its photos).</summary>
/// <summary>The fixed text of a localized line between two placeholders' markers.</summary>
private static string Between(string text, char from, char to)
{
var start = text.IndexOf(from, StringComparison.Ordinal) + 1;
return text[start..text.IndexOf(to, start)];
}
private static async Task WaitIdleAsync(SyncFixture f, Guid sequenceId)
{
var until = DateTime.UtcNow.AddSeconds(30);
while (DateTime.UtcNow < until)
{
var held = await f.Db.Jobs.ListPendingTargetsAsync(JobKind.Thumbs, f.Ct);
held.UnionWith(await f.Db.Jobs.ListPendingTargetsAsync(JobKind.LightAnalyse, f.Ct));
if (!held.Contains(sequenceId)) return;
await Task.Delay(20, f.Ct);
}
}
}