using System.Security.Cryptography; using Chronos.Core.Codecs.Jpeg; using Chronos.Core.Imaging; using Chronos.Core.Metadata; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data; using Chronos.Data.Archiving; using Chronos.Data.Copies; using Chronos.Data.Jobs; using Chronos.Data.Projects; using Chronos.Data.Repositories; using Chronos.Data.Sync; using Chronos.Protocol.Client; using Chronos.Server.Storage; using Chronos.Testing; namespace Chronos.Server.Tests.Sync; /// /// The owner's scenario of review 6 (#73) end to end, against a real in-process server: a time-lapse uploaded from this PC, /// an old copy of it found again on a disk adopted as an archive, compared, kept apart, integrated (Union) both ways with /// the details settled line by line and the projects copied across, its new photos moving by themselves unless the /// connection is metered, Upload all local leaving a suspected copy out, and two copies on this PC merged into one. /// [Category("server")] [Timeout(300_000)] public sealed class CopyScenarioTests(TestContext context) { private const string PcName = "STUDIO"; private const string ServerName = "Test server"; private static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0); private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(60); private CancellationToken Ct => context.Cancellation; [Test] public async Task AnOldCopyFoundOnAnAdoptedDiskIsIntegratedWithTheServerAndMergedOnThisPc() { await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register); var (_, _, token) = await server.UserAsync("alberto", "editor"); await using var pc = await Pc.StartAsync(context, server, token); // 1. A time-lapse of eight photos, rated 4, uploaded from this PC. // Photo 3 carries the camera's sub-second time: two versions of it at that instant under one name are one photo that differs. static DateTime Captured(int i) => Start.AddSeconds(10 * i).AddMilliseconds(i == 3 ? 250 : 0); var photos = Enumerable.Range(1, 8).Select(i => Jpeg(Captured(i), i)).ToList(); var original = await pc.LocalAsync(pc.Main, "Comet over the ridge", photos.Select((p, i) => (p, Captured(i + 1))).ToList()); original = (await pc.Db.Sequences.EditAsync(original.Id, new SequenceEdit { Rating = 4 }, null, Ct)).Sequence!; await pc.WaitAsync((await pc.Sync.EnqueueUploadAsync(pc.Jobs, pc.Server, original, Ct)).Id); var remoteId = (await pc.Sync.LinkOfAsync(pc.Server, original.Id, Ct))!.RemoteId; var onServer = Assert.NotNull(await server.Database.Sequences.GetAsync(remoteId)); Assert.True((await server.Database.Frames.ListBySequenceAsync(remoteId)).Select(f => f.Sha256).ToHashSet().SetEquals(photos.Select(Sha))); await ServerIdleAsync(server); // Freed on this PC since: the server holds it. await pc.Archives.DeleteAsync(await pc.Archives.PlanDeleteAsync([original.Id], cancellationToken: Ct), "alberto", cancellationToken: Ct); await pc.Sync.RefreshAsync(pc.Remote, Policy, Ct); Assert.Null(await pc.Sync.LinkOfAsync(pc.Server, original.Id, Ct)); // 2. An old disk adopted as an archive: photos 1-6 (photo 3 in another version), two more of its own, another title and rating. var manifest = await pc.Remote.GetManifestAsync(remoteId, Ct); var names = manifest.Frames.OrderBy(f => f.Index).Select(f => f.FileName).ToList(); var diskRoot = context.CreateTempDirectory("old-disk"); var folder = Path.Combine(diskRoot, "2026-09-28 comet"); Directory.CreateDirectory(folder); var differing = Jpeg(Captured(3), 103); var extras = new[] { Jpeg(Start.AddSeconds(5), 201), Jpeg(Start.AddSeconds(75), 202) }; for (var i = 0; i < 6; i++) await File.WriteAllBytesAsync(Path.Combine(folder, names[i]), i == 2 ? differing : photos[i], Ct); await File.WriteAllBytesAsync(Path.Combine(folder, "IMG_9001.JPG"), extras[0], Ct); await File.WriteAllBytesAsync(Path.Combine(folder, "IMG_9002.JPG"), extras[1], Ct); var disk = await pc.Archives.AdoptAsync(diskRoot, "Old disk", Ct); await pc.Archives.Indexer.ScanAsync(disk, ScanMode.Full, null, _ => ValueTask.CompletedTask, Ct); var old = Assert.NotNull((await pc.Db.Sequences.ListAsync(new SequenceQuery { ArchiveId = disk.Id }, Ct)).Items.SingleOrDefault()); old = (await pc.Db.Sequences.EditAsync(old.Id, new SequenceEdit { Title = "Old disk comet", Rating = 2 }, null, Ct)).Sequence!; Assert.Equal(8, old.FrameCount); // 3. Detection with Ask always (a refresh in the background): the match is listed, told, and not linked. var told = new List(); pc.Copies.Found += told.Add; await pc.Sync.RefreshAsync(pc.Remote, Policy, Ct); var pair = Assert.Contains(await pc.Copies.ListAsync(Ct), v => v.Copy.LocalId == old.Id).Copy; Assert.Equal((CopyKind.LocalServer, CopyState.Candidate, (Guid?)remoteId), (pair.Kind, pair.State, pair.RemoteId)); Assert.Contains(told.SelectMany(r => r.New), c => c.Id == pair.Id); Assert.Empty(told.SelectMany(r => r.AutoLinked)); Assert.Null(await pc.Sync.LinkOfAsync(pc.Server, old.Id, Ct)); // 4. Verified (an adopted folder holds no hashes until then), the comparison shows each class of photo. await pc.Archives.Verifier.VerifyAsync(disk, [old.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: Ct); var review = await pc.Copies.CompareAsync(pair.Id, pc.Remote, Ct); int Count(PhotoClass c) => review.Photos.Photos.Count(p => p.Class == c); Assert.Equal((5, 2, 2, 1), (Count(PhotoClass.Same), Count(PhotoClass.OnlyHere), Count(PhotoClass.OnlyThere), Count(PhotoClass.Differs))); Assert.True(review.Photos.Photos.Where(p => p.Class == PhotoClass.OnlyThere).Select(p => p.There!.Sha256).ToHashSet().SetEquals([Sha(photos[6]), Sha(photos[7])])); Assert.True(review.Photos.Photos.Where(p => p.Class == PhotoClass.OnlyHere).Select(p => p.Here!.Sha256).ToHashSet().SetEquals(extras.Select(Sha))); var differs = review.Photos.Photos.Single(p => p.Class == PhotoClass.Differs); Assert.Equal((Sha(differing), Sha(photos[2])), (differs.Here!.Sha256, differs.There!.Sha256)); Assert.Contains(review.Details, d => d.Path == "sequence.title" && d.Here == "Old disk comet" && d.There == onServer.Title); Assert.Contains(review.Details, d => d.Path == "sequence.rating" && d.Here == "2" && d.There == onServer.Rating.ToString(System.Globalization.CultureInfo.InvariantCulture)); // 5. Keep apart is remembered across refreshes; Ask again brings it back. await pc.Copies.KeepApartAsync(pair.Id, Ct); for (var i = 0; i < 2; i++) { await pc.Sync.Finder.ResetProbesAsync(pc.Server, Ct); await pc.Sync.RefreshAsync(pc.Remote, Policy, Ct); } Assert.Equal(CopyState.KeptApart, (await pc.Sync.Copies.GetAsync(pair.Id, Ct))!.State); Assert.Null(await pc.Sync.LinkOfAsync(pc.Server, old.Id, Ct)); await pc.Copies.AskAgainAsync(pair.Id, Ct); // 6. A project other than Main on each side, under one name, each edited its own way. await ProjectAsync(new ArchiveProjectHome(pc.Db, pc.Archives.Projects, old.Id), "Vertical", 41); await ProjectAsync(pc.ServerHome(remoteId), "Vertical", 33); // 7. Integrate with Union: the server's title, this PC's rating, the rest the server's; the projects copied across. review = await pc.Copies.CompareAsync(pair.Id, pc.Remote, Ct); var details = review.Details.Select(d => d.Path).Distinct() .ToDictionary(p => p, p => p == "sequence.rating" ? CopySide.ThisPc : CopySide.Other); var plan = await pc.Copies.PlanIntegrateAsync(pair.Id, pc.Remote, new IntegrateChoices(CopyPhotoPolicy.Union, new Dictionary(), details, CopyProjects.CopyAcross), Ct); Assert.True(plan.CanApply, string.Join(", ", plan.Blocks)); Assert.Equal((0, true, false), (plan.LeftOut, plan.AutoPhotos, plan.Destructive)); Assert.SequenceEqual(["Other:Vertical (Test server)", "ThisPc:Vertical (STUDIO)"], plan.Projects.Select(p => $"{p.From}:{p.TargetName}").Order(StringComparer.Ordinal)); await pc.WaitAsync((await pc.Copies.EnqueueIntegrateAsync(pc.Jobs, plan, Ct)).Id); var union = photos.Select(Sha).Concat(extras.Select(Sha)).Append(Sha(differing)).ToHashSet(StringComparer.Ordinal); Assert.True((await LocalHashesAsync(pc, old.Id)).SetEquals(union), "this PC holds every photo of both, both versions of photo 3"); Assert.True((await server.Database.Frames.ListBySequenceAsync(remoteId)).Select(f => f.Sha256!).ToHashSet().SetEquals(union), "and so does the server"); var here = (await pc.Db.Sequences.GetAsync(old.Id, Ct))!; var there = (await server.Database.Sequences.GetAsync(remoteId))!; Assert.Equal((onServer.Title, 2), (here.Title, here.Rating)); Assert.Equal((onServer.Title, 2), (there.Title, there.Rating)); static List Names(IEnumerable list) => [.. list.Where(p => !p.IsMain).Select(p => p.Name).Order(StringComparer.Ordinal)]; Assert.SequenceEqual(["Vertical", "Vertical (Test server)"], Names(await pc.Archives.Projects.ListAsync(old.Id, Ct))); Assert.SequenceEqual(["Vertical", "Vertical (STUDIO)"], Names(await pc.ServerHome(remoteId).ListAsync(Ct))); var integrated = (await pc.Sync.Copies.GetAsync(pair.Id, Ct))!; Assert.Equal((CopyState.Integrated, true), (integrated.State, integrated.AutoPhotos)); Assert.Equal(remoteId, (await pc.Sync.LinkOfAsync(pc.Server, old.Id, Ct))?.RemoteId); await ServerIdleAsync(server); // 8. A new photo on this PC moves to the server by itself, but not while the connection is metered. var later = Jpeg(Start.AddSeconds(200), 301); await pc.AddPhotoAsync(disk, old.Id, later, Start.AddSeconds(200)); var held = await pc.StopJobsAsync(); await using (var agent = new SyncAgent(pc.Sync, pc.Remote, () => Policy, held, new SyncAgentOptions { RefreshInterval = TimeSpan.FromHours(1), EventDelay = TimeSpan.FromMilliseconds(20), MinBackoff = TimeSpan.FromMilliseconds(20), MaxBackoff = TimeSpan.FromMilliseconds(200), })) { await agent.RefreshAsync(Ct); var automatic = (await pc.Db.Jobs.ListAsync(JobState.Queued, JobKind.UploadCommit, 10, cancellationToken: Ct)).Items.Single(); Assert.Equal((old.Id, JobPriority.Background), (automatic.TargetIds.Single(), automatic.Priority)); // Metered: the hold pauses the transfers with the server (UploadHold, the desktop's, does this on Windows). await held.PauseAsync(automatic.Id, Ct); await held.StartAsync(Ct); await Task.Delay(TimeSpan.FromSeconds(1), Ct); Assert.Equal(JobState.Paused, (await pc.Db.Jobs.GetAsync(automatic.Id, Ct))!.State); Assert.DoesNotContain(Sha(later), (await server.Database.Frames.ListBySequenceAsync(remoteId)).Select(f => f.Sha256)); // No longer metered: it goes on. await held.ResumeAsync(automatic.Id, Ct); await pc.WaitAsync(automatic.Id); Assert.Contains(Sha(later), (await server.Database.Frames.ListBySequenceAsync(remoteId)).Select(f => f.Sha256)); } // 9. Upload all local: a copy filed again from the card is a suspected duplicate, skipped and flagged; the rest goes. var card = await pc.LocalAsync(pc.Main, "Comet from the card", [(photos[0], Start.AddSeconds(10)), (photos[1], Start.AddSeconds(20)), (Jpeg(Start.AddSeconds(210), 401), Start.AddSeconds(210))]); var other = await pc.LocalAsync(pc.Main, "Harbour", [(Jpeg(Start.AddDays(1), 501), Start.AddDays(1)), (Jpeg(Start.AddDays(1).AddSeconds(10), 502), Start.AddDays(1).AddSeconds(10))]); await pc.Copies.FindAsync(pc.Remote, Ct); var local = Assert.Contains(await pc.Copies.ListAsync(Ct), v => v.Copy.Kind == CopyKind.LocalLocal).Copy; Assert.True(new[] { local.LocalId, local.OtherId!.Value }.ToHashSet().SetEquals([old.Id, card.Id])); Assert.Equal(CopyState.Candidate, local.State); var entries = (await pc.Sync.ListAsync(pc.Server, Ct)).Where(e => e.Status == SyncStatus.LocalOnly); var (upload, skipped) = UploadAllRule.Select(entries, await pc.Copies.ListAllPairsAsync(Ct), new HashSet(), new HashSet()); Assert.Equal((card.Id, UploadSkip.SuspectedCopy), (skipped.Single().Entry.Local!.Id, skipped.Single().Why)); Assert.Equal(other.Id, upload.Single().Local!.Id); await pc.WaitAsync((await pc.Sync.EnqueueUploadAsync(held, pc.Server, upload.Single().Local!, Ct)).Id); Assert.NotNull(await pc.Sync.LinkOfAsync(pc.Server, other.Id, Ct)); Assert.Null(await pc.Sync.LinkOfAsync(pc.Server, card.Id, Ct)); // 10. The two copies on this PC merge into the linked one; the other goes to the archive trash once every photo is verified there. review = await pc.Copies.CompareAsync(local.Id, null, Ct); var merge = await pc.Copies.PlanMergeAsync(local.Id, old.Id, new MergeChoices(CopyPhotoPolicy.Union, new Dictionary(), review.Details.Select(d => d.Path).Distinct().ToDictionary(p => p, _ => CopySide.ThisPc)), Ct); Assert.True(merge.CanApply, string.Join(", ", merge.Blocks)); Assert.Equal((old.Id, card.Id), (merge.KeepId, merge.DropId)); await pc.WaitAsync((await pc.Copies.EnqueueMergeAsync(held, merge, Ct)).Id); Assert.Null(await pc.Db.Sequences.GetAsync(card.Id, Ct)); var kept = await pc.Db.Frames.ListBySequenceAsync(old.Id, cancellationToken: Ct); Assert.Contains(Sha(Jpeg(Start.AddSeconds(210), 401)), kept.Select(f => f.Sha256)); foreach (var frame in kept) Assert.Equal(frame.Sha256, Sha(await File.ReadAllBytesAsync(Path.Combine(disk.RootPath, frame.RelativePath), Ct))); Assert.Contains(await pc.Archives.ListTrashAsync(pc.Main.Id, Ct), t => t.SequenceId == card.Id); Assert.Equal(remoteId, (await pc.Sync.LinkOfAsync(pc.Server, old.Id, Ct))?.RemoteId); } private static readonly SyncPolicy Policy = new(Linking: CopyLinking.AskAlways); private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes)); /// A project named at with one version of its own (a deflicker window). private async Task ProjectAsync(IProjectHome home, string name, int window) { var project = await home.CreateAsync(name, null, "alberto", Ct); var head = await home.HeadAsync(project.Id, Ct); var portable = ProjectContent.ToPortable(ProjectSettings.FromJson("""{"exposure":{"deflickerWindow":{"isAuto":false,"manual":""" + window + "}}}"), new FramePhotos([])); var pushed = await home.PushAsync(project.Id, new ProjectVersion { Id = Guid.CreateVersion7(), ProjectId = project.Id, ParentId = head.Id, Number = head.Number + 1, Kind = ProjectVersionKind.Commit, Author = "alberto", ClientName = PcName, SettingsJson = portable, ContentHash = ProjectContent.Hash(portable), FramesVersion = head.FramesVersion, CreatedAt = DateTimeOffset.UtcNow, }, Ct); Assert.NotNull(pushed.Version); } /// A small real JPEG with Exif (camera and capture time); each seed gives distinct bytes. private static byte[] Jpeg(DateTime captured, int seed) { using var image = new ImageBuffer(48, 32); var random = new Random(seed); for (var c = 0; c < 3; c++) { var plane = image.Plane(c); var level = (float)random.NextDouble(); for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % 48) / 48f); } var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { Make = "Canon", Model = "Canon EOS R5", BodySerial = "0431", CapturedAt = captured, Iso = 100 }); return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); } private static async Task> LocalHashesAsync(Pc pc, Guid id) => (await pc.Db.Frames.ListBySequenceAsync(id, cancellationToken: pc.Ct)).Select(f => f.Sha256!).ToHashSet(StringComparer.Ordinal); private Task ServerIdleAsync(TestServer server) => UntilAsync(async () => await server.Database.Jobs.CountAsync(DateTimeOffset.UtcNow) is { Queued: 0, Running: 0 }, "the server's jobs"); private async Task UntilAsync(Func> condition, string what) { var until = DateTime.UtcNow + Deadline; while (!await condition()) { if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what); await Task.Delay(50, Ct); } } /// /// This PC as the desktop composes it (AppServices): its index and archive, the sync engine, the matches engine with its /// planners and the server's project homes, and a job engine with the archive, sync, Integrate and Merge handlers. /// private sealed class Pc : IAsyncDisposable { private readonly TestContext _context; private readonly ChronosClient _client; private readonly Connector _connector; private Pc(TestContext context, ChronosDatabase db, ArchiveService archives, Archive main, SyncService sync, ChronosClient client) { _context = context; Db = db; Archives = archives; Main = main; Sync = sync; _client = client; Remote = new ChronosSyncRemote(client, ownsClient: false); _connector = new Connector(client); var sides = new CopySides(db); var homes = CopyJobs.ServerHomes(_connector); Copies = new CopyEngine(sync, archives, new IntegratePlanner(sync, sides, archives.Projects, homes), new SequenceMerger(db, archives, sides), new CopyReviewBuilder(db, archives.Projects, sides, homes), remote => new CopyPlanContext(PcName, remote is null ? null : ServerName, true, true, true, false, false, remote is not null, null, new HashSet(), new HashSet()), serverNames: _ => ServerName); Jobs = NewJobs(); } public ChronosDatabase Db { get; } public ArchiveService Archives { get; } public Archive Main { get; } public SyncService Sync { get; } public ChronosSyncRemote Remote { get; } public CopyEngine Copies { get; } public JobEngine Jobs { get; private set; } public string Server => Remote.Server; public CancellationToken Ct => _context.Cancellation; public static async Task StartAsync(TestContext context, TestServer server, string token) { var ct = context.Cancellation; var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("studio-db"), "chronos.db")), ct); var options = new ArchiveServiceOptions { Owner = PcName, LeaseWait = TimeSpan.FromSeconds(20) }; var archives = new ArchiveService(db, options); var main = await archives.CreateAsync(context.CreateTempDirectory("studio-archive"), "Main", cancellationToken: ct); var sync = new SyncService(db, archives, new SyncOptions { Archive = options, ProxyCacheDirectory = context.CreateTempDirectory("studio-proxies") }); var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), PcName) { Token = token }; var pc = new Pc(context, db, archives, main, sync, client); await pc.Jobs.StartAsync(ct); return pc; } /// A job engine over this index with every handler the desktop registers for these jobs (not started). public JobEngine NewJobs() => new(Db, [ .. Archives.CreateJobHandlers(), .. Sync.CreateJobHandlers(_connector), CopyJobs.CreateIntegrateHandler(Sync, Archives, _connector, Sync.Copies), SequenceMerger.CreateHandler(Db, new SequenceMerger(Db, Archives, new CopySides(Db)), Archives, new ProjectTransplant(), new NoWorkspace()), ], new JobEngineOptions { Workers = 2, WorkerId = PcName, PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); /// Stops the engine that runs jobs as they come and puts one not started in its place: the next jobs wait for it. public async Task StopJobsAsync() { await Jobs.DisposeAsync(); return Jobs = NewJobs(); } public IProjectHome ServerHome(Guid remoteId) => new ServerProjectHome(_client, remoteId, ServerName, null, Server); /// A local sequence filed photo by photo through the archive ingest. public async Task LocalAsync(Archive archive, string title, IReadOnlyList<(byte[] Bytes, DateTime Captured)> photos) { Guid? id = null; for (var i = 0; i < photos.Count; i++) { var result = await IngestAsync(archive, id, id is null ? title : null, photos[i].Bytes, $"IMG_{photos[i].Captured:HHmmss}.JPG", photos[i].Captured); id = result.SequenceId; } return (await Db.Sequences.GetAsync(id!.Value, Ct))!; } public Task AddPhotoAsync(Archive archive, Guid sequenceId, byte[] photo, DateTime captured) => IngestAsync(archive, sequenceId, null, photo, $"IMG_{captured:HHmmss}.JPG", captured); private async Task IngestAsync(Archive archive, Guid? sequenceId, string? title, byte[] photo, string name, DateTime captured) { var staging = Path.Combine(archive.RootPath, ".chronos", "staging", "test"); Directory.CreateDirectory(staging); var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part"); await File.WriteAllBytesAsync(path, photo, Ct); return await Archives.Ingest.CommitAsync(new IngestRequest(archive.Id, sequenceId, title, null, path, name, photo.Length, Sha(photo), captured, null), Ct); } public async Task WaitAsync(Guid jobId) { var until = DateTime.UtcNow.AddSeconds(TestTiming.Budget(60)); while (true) { var job = await Db.Jobs.GetAsync(jobId, Ct); if (job?.State == JobState.Done) return job; if (job?.State is JobState.Failed or JobState.Cancelled) Assert.Fail($"Job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}"); if (DateTime.UtcNow > until) Assert.Fail($"Job {jobId} did not finish (now {job?.State})"); await Task.Delay(25, Ct); } } public async ValueTask DisposeAsync() { await Jobs.DisposeAsync(); Copies.Dispose(); Remote.Dispose(); _client.Dispose(); await Db.DisposeAsync(); } /// No workspace here: no sequence has workspace work not sent home. private sealed class NoWorkspace : IWorkspaceGuard { public Task HasUnsentAsync(Guid sequenceId, CancellationToken ct) => Task.FromResult(false); } /// The signed-in session: a remote per job and the server's project homes (Settings › Server's server keeps projects). private sealed class Connector(ChronosClient client) : ISyncConnector { public ValueTask ConnectAsync(string server, CancellationToken cancellationToken) { ISyncRemote remote = new ChronosSyncRemote(client, ownsClient: false); return ValueTask.FromResult(remote.Server == server ? remote : null); } public IProjectHome? ProjectHome(BundleHome home) => home.IsArchive ? null : new ServerProjectHome(client, home.SequenceId, ServerName, null, home.Server); } } }