using Chronos.Core.Codecs.Jpeg; using Chronos.Core.Imaging; using Chronos.Core.Imaging.Decoders; using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Core.Render; using Chronos.Data.Archiving; using Chronos.Data.Sync; using Chronos.Data.Tests.Archiving; using Chronos.Protocol; using Chronos.Testing; namespace Chronos.Data.Tests.Sync; /// Upload, download and proxy jobs on the job engine against an in-memory server. [Category("sync")] public sealed class SyncTransferTests(TestContext context) { [Test] public async Task UploadSendsTheOriginalsAndSettingsOfALocalSequence() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Comet", photos, SyncFixture.Window41); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); Assert.Equal(job.Id, (await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id); var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal)); var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet"); Assert.True(remote.Frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived"); Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson), "the settings travel with the originals"); var entry = await f.EntryAsync("Comet"); Assert.Equal(SyncStatus.InSync, entry.Status); Assert.Equal(remote.Id, entry.Link!.RemoteId); } [Test] public async Task AnUploadSendsTheProxiesTheServerCannotMakeItself() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Holy grail", photos); // The server has the originals already (the Uploader sent them) but cannot decode them (HEIF, BRIEF §2.3). var remote = f.Remote.Add("Holy grail", photos); remote.Undecodable = true; await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var ahead = await f.EntryAsync("Holy grail"); Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); foreach (var proxy in f.Remote.SentProxies.Values) { var info = JpegDecoder.ReadInfo(proxy); Assert.True(info.Width <= 1280 && info.Height <= 1280, "made as the server's policy says"); } Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status); } [Test] public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); // An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds // under the names it gave them, and cannot decode (HEIF, BRIEF §2.3). var folder = Path.Combine(f.Archive.Paths.Root, "Adopted"); Directory.CreateDirectory(folder); for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct); await f.Archive.ScanAsync(); var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted"); Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed"); var remote = f.Remote.Add("Adopted", photos); remote.Undecodable = true; await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var ahead = await f.EntryAsync(local.DisplayTitle); Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status); } [Test] public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Holy grail", photos); var remote = f.Remote.Add("Holy grail", photos); remote.Undecodable = true; await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote); var calls = f.Remote.ManifestCalls; await f.RefreshAsync(); Assert.Equal(calls, f.Remote.ManifestCalls); // A render node sends them after a job: the server's frame counters stand still, its sequence changes. foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy; remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1); await f.RefreshAsync(); var entry = await f.EntryAsync("Holy grail"); Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote)); } [Test] public async Task PhotosOneSideDeletedAreNeitherSentNorFetchedAgain() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(4); var local = await f.LocalAsync("Comet", photos); // The server holds the first three and deleted the fourth, which this PC still holds. var remote = f.Remote.Add("Comet", photos[..3]); remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow)); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); Assert.Equal(0, (await f.EntryAsync("Comet")).Link!.MissingRemote); var engine = await f.StartEngineAsync(); var upload = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); await f.Archive.WaitForAsync(upload.Id, JobState.Done); Assert.Equal(3, remote.Frames.Count); Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)); // This PC deletes the first photo, which the server still holds: it is not fetched back, and nothing local is deleted. var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [frames[0].Id], sequence.FramesVersion, f.Ct); await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct); await f.RefreshAsync(); var entry = await f.EntryAsync("Comet"); Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal)); var download = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(download.Id, JobState.Done); var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.SequenceEqual(photos[1..].Select(Sha), kept.Select(k => k.Sha256!)); } [Test] public async Task APhotoDeletedHereIsNotFetchedAgainFromAServerThatHasNotHashedIt() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); // A folder the server adopted and never verified (its manifest lists no SHA-256), downloaded here. var remote = f.Remote.Add("Adopted on the server", photos); remote.Unhashed = true; await f.RefreshAsync(); var engine = await f.StartEngineAsync(); var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id); await f.Archive.WaitForAsync(first.Id, JobState.Done); var local = Assert.NotNull((await f.EntryAsync("Adopted on the server")).Local); // This PC deletes a photo the server still lists, unhashed: only its content tells it is the one deleted here. await UntilIdleAsync(f); var gone = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(photos[0])); var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [gone.Id], sequence.FramesVersion, f.Ct); await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct); await f.RefreshAsync(); var again = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null); var done = await f.Archive.WaitForAsync(again.Id, JobState.Done); var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted"); Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone"); Assert.Contains(Loc.L("Archive.Log.SyncDeletedHere", remote.Frames.Single(fr => fr.Sha256 == Sha(photos[0])).FileName), done.LogTail); Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up"); } [Test] public async Task AnUnverifiedCopyDoesNotSendBackAPhotoTheServerDeleted() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(4); // An adopted folder (never verified, so no hashes here) that still holds a photo the server has deleted since. var folder = Path.Combine(f.Archive.Paths.Root, "Adopted"); Directory.CreateDirectory(folder); for (var i = 0; i < 4; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct); await f.Archive.ScanAsync(); var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted"); var remote = f.Remote.Add("Adopted", photos[..3]); remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow)); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); // Without its hash the photo cannot be told from one the server lacks: the upload finds out. Assert.Equal(1, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingRemote); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal(3, remote.Frames.Count); Assert.Contains(Loc.L("Archive.Log.SyncDeletedThere", "DSC_0004.JPG"), done.LogTail); // Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good. Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify"); var entry = await f.EntryAsync(local.DisplayTitle); Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal)); var calls = remote.Frames.Count; var next = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var settled = await f.Archive.WaitForAsync(next.Id, JobState.Done); Assert.Equal((4L, 4L, calls), (settled.ProgressDone, settled.ProgressTotal, remote.Frames.Count)); } [Test] public async Task ATombstoneBesideAPhotoTheServerStillHoldsLeavesTheCopiesInSync() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Comet", photos); // The server holds both photos and a tombstone of the second: it deleted one of two copies of it, or the file came // back by hand. Neither side lacks anything. var remote = f.Remote.Add("Comet", photos); remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), null, "deleted", DateTimeOffset.UtcNow)); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var entry = await f.EntryAsync("Comet"); Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal)); Assert.Equal(SyncStatus.InSync, entry.Status); } [Test] public async Task AResumedUploadSendsOnlyWhatTheServerLacks() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(4); var local = await f.LocalAsync("Comet", photos); // A crashed attempt had created the server sequence and sent two originals. var remote = f.Remote.Add("Comet", photos[..2]); var engine = await f.StartEngineAsync(); var job = await f.Archive.EnqueueWithCheckpointAsync(engine, new(JobKind.UploadCommit, [local.Id], "Comet", """{"server":"https://nas.test:8443"}""", JobPriority.User), remote.Id.ToString()); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Count(1, f.Remote.Sequences.Values); Assert.Equal(4, remote.Frames.Count); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Comet")).Status); } [Test] public async Task OriginalsChangedSinceTheyWereArchivedAreNeverUploaded() { await using var f = await SyncFixture.CreateAsync(context); var local = await f.LocalAsync("Comet", f.NewPhotos(3)); var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; var path = f.Archive.Paths.FromRelative(damaged.RelativePath); var bytes = await File.ReadAllBytesAsync(path, f.Ct); bytes[^3] ^= 0xFF; await File.WriteAllBytesAsync(path, bytes, f.Ct); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Contains("1", failed.Error); var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet"); Assert.Equal(2, remote.Frames.Count); Assert.DoesNotContain(damaged.Sha256!, remote.Frames.Select(r => r.Sha256)); Assert.Equal(SyncStatus.LocalAhead, (await f.EntryAsync("Comet")).Status); } [Test] public async Task DownloadedOriginalsAreVerifiedAndCommittedThroughTheArchive() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var remote = f.Remote.Add("Construction cam", photos, SyncFixture.Window33); remote.Tags = ["site"]; f.Remote.CorruptOnce = remote.Frames[1].Id; await f.RefreshAsync(); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Originals, f.Archive.Archive.Id); await f.Archive.WaitForAsync(job.Id, JobState.Done); var entry = await f.EntryAsync("Construction cam"); Assert.Equal(SyncStatus.InSync, entry.Status); var local = Assert.NotNull(entry.Local); Assert.Equal(SequenceOrigin.Server, local.Origin); Assert.True(SyncRules.SameSettings(SyncFixture.Window33, local.SettingsJson), "the server's settings came along"); Assert.SequenceEqual(["site"], local.Tags); var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.Equal(3, frames.Count); foreach (var frame in frames) { var bytes = await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frame.RelativePath), f.Ct); Assert.Equal(frame.Sha256, Sha(bytes)); } Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "the damaged transfer was fetched again"); Assert.FileExists(f.Archive.Paths.Sequence(local.Folder).ManifestPath); Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up"); } [Test] public async Task DownloadFillsInWhatALocalCopyLacks() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Harbour", photos[..1], SyncFixture.Window41); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); await f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.KeepMine, f.Ct); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal(3, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Count); var entry = await f.EntryAsync("Harbour"); Assert.Equal(SyncStatus.InSync, entry.Status); Assert.Equal(SequenceOrigin.Local, entry.Local!.Origin); Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local.SettingsJson), "an existing copy keeps its settings"); } [Test] public async Task DownloadingIntoAnUnverifiedCopyHashesItFirstAndFetchesOnlyWhatItLacks() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(5); // An adopted folder: indexed by a scan, never verified, so its frames have no hash yet. var folder = Path.Combine(f.Archive.Paths.Root, "Adopted"); Directory.CreateDirectory(folder); for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"IMG_{i + 1:0000}.JPG"), photos[i], f.Ct); await f.Archive.ScanAsync(); var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted"); Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed"); var remote = f.Remote.Add("Adopted", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); Assert.Equal(2, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingLocal); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.Equal(5, frames.Count); Assert.True(frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "no original arrived twice"); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status); } [Test] public async Task DownloadingOriginalsRestoresTheOnesMissingHere() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Harbour", photos); var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; var place = f.Archive.Paths.FromRelative(lost.RelativePath); File.Delete(place); await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var behind = await f.EntryAsync("Harbour"); Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(place, f.Ct))); var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.Equal(3, frames.Count); Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the restored original is no longer missing"); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } [Test] public async Task AnOriginalIsRestoredOnlyWhereTheSequenceIsNow() { // The folder the sequence left holds another sequence now, or nothing. foreach (var taken in new[] { true, false }) { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Harbour", photos); var stranger = taken ? await f.LocalAsync("Stranger", f.NewPhotos(2)) : null; var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath)); await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var engine = await f.StartEngineAsync(); // Another index moves the folder away, then the other sequence (first parked elsewhere: its folder is the same // name with a number) into the folder it left; this index has scanned neither. var folder = f.Archive.Paths.Sequence(local.Folder).Folder; var (other, archive) = await f.Archive.OtherWriterAsync(); var moves = stranger is null ? [(local.Id, "moved/{title}")] : new[] { (local.Id, "moved/{title}"), (stranger.Id, "parked/{title}"), (stranger.Id, local.Folder) }; foreach (var (id, template) in moves) await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], template, null, f.Ct), null, f.Ct); var before = taken ? ArchiveFixture.FilesOf(folder) : null; var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error, taken.ToString()); if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched"); else Assert.False(Directory.Exists(folder), "the folder it left is not created again"); // Once this index's scan found it there, the original goes back into the folder the sequence is in now. await f.Archive.ScanAsync(); job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var moved = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Folder; Assert.Equal("moved/Harbour", moved); var restored = (await f.Db.Frames.GetAsync(lost.Id, f.Ct))!; Assert.Equal(moved + lost.RelativePath[local.Folder.Length..], restored.RelativePath); Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(restored.RelativePath), f.Ct))); Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => (fr.Flags & FrameFlags.Missing) == 0), "nothing missing"); if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched"); else Assert.False(Directory.Exists(folder)); } } [Test] public async Task OriginalsTheServerHasNotHashedAreDownloadedAndHashedHere() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); // A folder the server adopted and never verified: its manifest lists no SHA-256. var remote = f.Remote.Add("Adopted on the server", photos); remote.Unhashed = true; await f.RefreshAsync(); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id); await f.Archive.WaitForAsync(job.Id, JobState.Done); var entry = await f.EntryAsync("Adopted on the server"); var frames = await f.Db.Frames.ListBySequenceAsync(entry.Local!.Id, cancellationToken: f.Ct); Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived, hashed here"); // Matched by size and capture time, the copies are the server's originals. Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal)); // One more photo adopted there: only that one travels. remote.AddFrame(f.NewPhotos(1)[0]); await f.RefreshAsync(); Assert.Equal(1, (await f.EntryAsync("Adopted on the server")).Link!.MissingLocal); job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal(4, (await f.Db.Frames.ListBySequenceAsync(entry.Local.Id, cancellationToken: f.Ct)).Count); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Adopted on the server")).Status); } [Test] public async Task AnUploadDoesNotSendAgainWhatTheServerHoldsWithoutAHash() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(4); var local = await f.LocalAsync("Harbour", photos); // The server adopted the first three photos and never verified them: it cannot recognise them by content. var remote = f.Remote.Add("Harbour", photos[..3]); remote.Unhashed = true; await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var ahead = (await f.EntryAsync("Harbour")).Link!; Assert.Equal((1, 0), (ahead.MissingRemote, ahead.MissingLocal)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal((4L, 4L), (done.ProgressDone, done.ProgressTotal)); Assert.SequenceEqual(photos.Select(Sha), remote.Frames.Select(r => r.Sha256)); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } [Test] public async Task ProxiesOfFramesTheServerHasNotHashedAreSentToo() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Holy grail", photos); // The server adopted the same originals, cannot decode them and has not hashed them. var remote = f.Remote.Add("Holy grail", photos); remote.Undecodable = true; remote.Unhashed = true; await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); Assert.Equal(3, (await f.EntryAsync("Holy grail")).Link!.MissingRemote); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); Assert.Equal((SyncStatus.InSync, 0), ((await f.EntryAsync("Holy grail")).Status, (await f.EntryAsync("Holy grail")).Link!.MissingRemote)); } [Test] public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(6); var local = await f.LocalAsync("Harbour", photos[1..5]); var remote = f.Remote.Add("Harbour", photos[1..5]); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); // Another PC sends the server an earlier and a later photo; then an editor there excludes two photos, picks the // hero and keys the camera, by the server's frames. remote.InsertFrame(0, photos[0]); remote.AddFrame(photos[5]); f.Remote.EditSettings(remote.Id, new ProjectSettings { ExcludedFrames = [0, 3], HeroFrame = 5, Camera = new CameraSettings { Keyframes = [new CameraKey(2, 0.4, 0.5, 1.5, 0)] }, }.ToJson()); await f.RefreshAsync(); var taken = await NamedAsync(f, local.Id); Assert.SequenceEqual([Sha(photos[3])], taken.Excluded); Assert.Null(taken.Hero); Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes); Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them"); // Each waits with its capture time too, whichever side hashed what: the same project reads the same either way. var waiting = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).PendingFrames!; Assert.True(waiting.All(p => p.CapturedAt is not null), "a photo waits with its capture time"); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); // The same photos as on the server, and nothing waits any more. var arrived = await NamedAsync(f, local.Id); Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], arrived.Excluded); Assert.Equal(Sha(photos[5]), arrived.Hero); Assert.SequenceEqual([Sha(photos[2])], arrived.Keyframes); Assert.Empty(arrived.Pending); var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.Equal(Sha(photos[5]), frames.Single(fr => fr.Id == row.HeroFrameId).Sha256); Assert.True(SyncRules.SameSettings(row.SettingsJson, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(row.Folder).ProjectPath, f.Ct)), "project.json mirrors the index"); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); // An edit here later sends the server the same photos, by its frames. var edited = ProjectSettings.FromJson(row.SettingsJson!) with { Exposure = new ExposureSettings { DeflickerWindow = new Auto().WithManual(41) } }; await f.EditLocalSettingsAsync(local.Id, edited.ToJson()); await f.RefreshAsync(); var server = ProjectSettings.FromJson(remote.SettingsJson!); Assert.Equal(41, server.Exposure.DeflickerWindow.Manual); Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], server.ExcludedFrames.Select(i => remote.Frames[i].Sha256)); Assert.Equal(Sha(photos[5]), remote.Frames[server.HeroFrame!.Value].Sha256); Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256)); } [Test] public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy() { await using var f = await SyncFixture.CreateAsync(context); var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt); await f.RefreshAsync(); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in"); Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies"); // Proxies arrive on the server (a client or render node sent them); the frames did not change. remote.Status |= SequenceStatus.ProxiesBuilt; job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length)); Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale); } [Test] public async Task ACacheMadeBeforeTheServerDrewItsNewFramesStaysStaleUntilItHasTheirProxies() { await using var f = await SyncFixture.CreateAsync(context); var remote = f.Remote.Add("Harbour", f.NewPhotos(2)); await f.RefreshAsync(); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale); // Two photos reach the server. Its proxy flag stays set until its next proxy pass, which has not drawn them yet: // the first has its thumbnail only, the second nothing at all. var added = f.NewPhotos(2).Select(photo => remote.AddFrame(photo)).ToList(); f.Remote.Undrawn.UnionWith([(added[0].Id, "proxy"), (added[1].Id, "proxy"), (added[1].Id, "thumb")]); await f.RefreshAsync(); job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Take(3).Select(file => (int)new FileInfo(file!).Length)); Assert.Null(opened.Files[3]); Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale, "a thumbnail or a missing file stands in for a proxy the server offers"); // The pass draws them: the frames stay as they were, the server's sequence records the pass. f.Remote.Undrawn.Clear(); remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1); await f.RefreshAsync(); Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale); job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length)); Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale); } [Test] public async Task TheServersRendersAreFetchedIntoTheLocalExports() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Star trails", photos); var remote = f.Remote.Add("Star trails", photos); var render = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 5000).Select(i => (byte)i)]); remote.Renders.Add(render); var still = new FakeRender(Guid.CreateVersion7(), "Star trails - 0002.png", "PNG", [.. Enumerable.Range(0, 700).Select(i => (byte)(i * 5))], RenderKind.Still, Recorded: string.Empty); remote.Renders.Add(still); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var entry = await f.EntryAsync("Star trails"); Assert.Equal((SyncStatus.InSync, 2, 5700L), (entry.Status, entry.Link!.ExportsToGet, entry.Link.ExportsToGetBytes)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var fetched = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == render.Id); Assert.Equal("4K HEVC", fetched.PresetName); var file = f.Archive.Paths.FromRelative(fetched.OutputPath); Assert.Equal(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 4K.mp4"), file); Assert.SequenceEqual(render.Bytes, await File.ReadAllBytesAsync(file, f.Ct)); Assert.Equal((render.Sha256, (string?)null), (fetched.Sha256, fetched.Kind), "the server's hash, checked on arrival"); // A still the server has not hashed: its kind, and the hash of what arrived. var saved = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == still.Id); Assert.Equal((Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(still.Bytes)), RenderKind.Still), (saved.Sha256, saved.Kind)); Assert.Equal(0, (await f.EntryAsync("Star trails")).Link!.ExportsToGet); // A file unlike the hash the server records is fetched again, then refused: nothing is filed. var damaged = new FakeRender(Guid.CreateVersion7(), "Star trails 1080p.mp4", "1080p", [1, 2, 3, 4], Recorded: new string('a', 64)); remote.Renders.Add(damaged); remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1); job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null); var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Equal(Loc.L("Errors.Sync.Hash", "Star trails 1080p.mp4"), failed.Error); Assert.Null(await f.Db.Renders.GetAsync(damaged.Id, f.Ct)); Assert.False(File.Exists(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 1080p.mp4"))); } [Test] public async Task ASentRenderKeepsItsKind() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Harbour", photos); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); var exports = f.Archive.Paths.Sequence(local.Folder).Exports; Directory.CreateDirectory(exports); byte[] bytes = [.. Enumerable.Range(0, 900).Select(i => (byte)(i * 11))]; await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour - 0001.png"), bytes, f.Ct); var still = await AddRenderAsync(f, local, "Harbour - 0001.png", bytes.Length, RenderKind.Still); await f.RefreshAsync(); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local); await f.Archive.WaitForAsync(job.Id, JobState.Done); var sent = Assert.Contains(remote.Renders, r => r.Id == still.Id); Assert.Equal(RenderKind.Still, sent.Kind); } [Test] public async Task TheServersRendersGoWhereTheSequenceIsNow() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Star trails", photos); var remote = f.Remote.Add("Star trails", photos); var first = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 3000).Select(i => (byte)i)]); remote.Renders.Add(first); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var engine = await f.StartEngineAsync(); var left = f.Archive.Paths.Sequence(local.Folder).Folder; // A Reorganize here moves the folder while the server's renders are listed: the render goes where the sequence is. string? moved = null; f.Remote.OnceBeforeManifest = async () => { var plan = await f.Archive.Service.PlanRenameAsync(f.Archive.Archive.Id, [local.Id], "moved/{title}", null, f.Ct); await f.Archive.Service.ApplyRenameAsync(plan, null, f.Ct); moved = plan.Steps.Single().ToFolder; }; var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal($"{moved}/exports/Star trails 4K.mp4", (await f.Db.Renders.GetAsync(first.Id, f.Ct))!.OutputPath); Assert.FileExists(f.Archive.Paths.FromRelative($"{moved}/exports/Star trails 4K.mp4")); Assert.False(Directory.Exists(left), "the folder it left is not created again"); // Another index moves it: nothing is filed until this index's scan finds it there. var second = new FakeRender(Guid.CreateVersion7(), "Star trails 1080p.mp4", "1080p", [.. Enumerable.Range(0, 2000).Select(i => (byte)(i * 3))]); remote.Renders.Add(second); remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1); var (other, archive) = await f.Archive.OtherWriterAsync(); var away = await other.PlanRenameAsync(archive.Id, [local.Id], "elsewhere/{title}", null, f.Ct); await other.ApplyRenameAsync(away, null, f.Ct); job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null); var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error); Assert.False(Directory.Exists(f.Archive.Paths.Sequence(moved!).Folder), "the folder it left is not created again"); Assert.Null(await f.Db.Renders.GetAsync(second.Id, f.Ct)); await f.Archive.ScanAsync(); job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var fetched = (await f.Db.Renders.GetAsync(second.Id, f.Ct))!; Assert.Equal("elsewhere/Star trails/exports/Star trails 1080p.mp4", fetched.OutputPath); Assert.SequenceEqual(second.Bytes, await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(fetched.OutputPath), f.Ct)); } [Test] public async Task RendersMadeHereAreSentOnRequestAndAGoneFileIsReported() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Harbour", photos); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); var exports = f.Archive.Paths.Sequence(local.Folder).Exports; Directory.CreateDirectory(exports); var bytes = Enumerable.Range(0, 3000).Select(i => (byte)(i * 7)).ToArray(); await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour 1080p.mp4"), bytes, f.Ct); var made = await AddRenderAsync(f, local, "Harbour 1080p.mp4", bytes.Length); var gone = await AddRenderAsync(f, local, "Harbour draft.mp4", 10); await f.RefreshAsync(); Assert.Equal((1, 3000L), ((await f.EntryAsync("Harbour")).Link!.ExportsToSend, (await f.EntryAsync("Harbour")).Link!.ExportsToSendBytes)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local); Assert.Equal(SyncTransferKind.SendExports, SyncService.TransferKind(job)); var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Contains("1", failed.Error); var sent = Assert.Contains(remote.Renders, r => r.Id == made.Id); Assert.SequenceEqual(bytes, sent.Bytes); Assert.DoesNotContain(gone.Id, remote.Renders.Select(r => r.Id)); Assert.Equal(0, (await f.EntryAsync("Harbour")).Link!.ExportsToSend); } [Test] public async Task ProxiesOpenARemoteSequenceForTheEditor() { await using var f = await SyncFixture.CreateAsync(context); var remote = f.Remote.Add("Milan skyline", f.NewPhotos(3), SyncFixture.Window33); await f.RefreshAsync(); Assert.Null(f.Sync.OpenRemote(f.Server, remote.Id)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Milan skyline", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var entry = await f.EntryAsync("Milan skyline"); Assert.Equal(SyncStatus.ProxiesOnly, entry.Status); Assert.Equal(3, entry.Link!.ProxiesCached); Assert.False(entry.Link.ProxiesStale); var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.Equal(remote.Id, opened.Sequence.Id); Assert.Equal(SequenceOrigin.Server, opened.Sequence.Origin); Assert.True(SyncRules.SameSettings(SyncFixture.Window33, opened.Sequence.SettingsJson)); Assert.Equal(3, opened.CachedCount); var source = opened.OpenFrames(DecoderFactory.CreateServer()); await using (source as IAsyncDisposable) { Assert.Equal(3, source.FrameCount); using var frame = await source.GetFrameAsync(2, 0, DecodePurpose.Render, f.Ct); Assert.Equal((48, 32), (frame.Width, frame.Height)); } // Nothing local was created, and a frame added on the server makes the cache stale. Assert.Null(entry.Local); remote.AddFrame(f.NewPhotos(1)[0]); await f.RefreshAsync(); Assert.True((await f.EntryAsync("Milan skyline")).Link!.ProxiesStale); } [Test] public async Task ThumbnailsStandInWhenTheServerHasNoProxies() { await using var f = await SyncFixture.CreateAsync(context); var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.Equal(2, opened.CachedCount); Assert.True(opened.Files.All(file => new FileInfo(file!).Length == 64), "thumbnails"); } [Test] public async Task TransfersFailWithAReasonWhenThisPcIsNotConnected() { await using var f = await SyncFixture.CreateAsync(context); var local = await f.LocalAsync("Comet", f.NewPhotos(1)); var engine = await f.StartEngineAsync(new SyncFixture.Connector(null)); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Sync.NotConnected"), failed.Error); Assert.Empty(f.Remote.Sequences); } [Test] public async Task APhotoEachSideConvertedIsNeitherSentNorFetchedAgain() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Comet", photos); // Each side converted its own copy of the first photo, to other bytes (another decoder, another quality). var (mine, theirs) = (f.NewPhotos(1)[0], f.NewPhotos(1)[0]); await f.ReplaceLocalAsync(local.Id, 0, mine); var remote = f.Remote.Add("Comet", [theirs, photos[1]]); remote.Removed.Add(new RemovedPhotoDto(Sha(photos[0]), Sha(theirs), "replaced", DateTimeOffset.UtcNow)); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var entry = await f.EntryAsync("Comet"); Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal)); var engine = await f.StartEngineAsync(); await f.Archive.WaitForAsync((await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id, JobState.Done); await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null)).Id, JobState.Done); Assert.SequenceEqual([Sha(theirs), Sha(photos[1])], remote.Frames.Select(r => r.Sha256)); Assert.SequenceEqual([Sha(mine), Sha(photos[1])], (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Select(r => r.Sha256!)); } [Test] public async Task CopiesConvertedOnOneSideBeforeTheyWereLinkedAreLinkedByContent() { await using var f = await SyncFixture.CreateAsync(context); // The Uploader sent the card to the server, which converted it before this PC compared the two. var photos = f.NewPhotos(3); var local = await f.LocalAsync("Comet", photos); var converted = f.NewPhotos(3); var remote = f.Remote.Add("2026-09-28 Canon", converted); foreach (var (original, replacement) in photos.Zip(converted)) remote.Removed.Add(new RemovedPhotoDto(Sha(original), Sha(replacement), "replaced", DateTimeOffset.UtcNow)); remote.FirstCapture = local.FirstCapture; remote.CameraModel = local.CameraModel; // And the other way round: this PC converted its copy of another card the server holds as it came. var cards = f.NewPhotos(2); var mine = await f.LocalAsync("Harbour", cards); foreach (var (index, replacement) in f.NewPhotos(2).Index()) await f.ReplaceLocalAsync(mine.Id, index, replacement); mine = (await f.Db.Sequences.GetAsync(mine.Id, f.Ct))!; var theirs = f.Remote.Add("2026-09-28 Canon (2)", cards); theirs.FirstCapture = mine.FirstCapture; theirs.CameraModel = mine.CameraModel; await f.RefreshAsync(); var comet = await f.EntryAsync("Comet"); Assert.Equal(remote.Id, comet.Link?.RemoteId); Assert.Equal((0, 0), (comet.Link!.MissingRemote, comet.Link.MissingLocal)); var harbour = await f.EntryAsync("Harbour"); Assert.Equal(theirs.Id, harbour.Link?.RemoteId); Assert.Equal((0, 0), (harbour.Link!.MissingRemote, harbour.Link.MissingLocal)); } [Test] public async Task AFrameMissingHereTakesBackItsPhotoAsTheServerConvertedIt() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Harbour", photos); var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath)); await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); // The server converted that photo. var converted = f.NewPhotos(1)[0]; var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]); remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow)); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var behind = await f.EntryAsync("Harbour"); Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); // The same frame, holding the server's conversion, and no frame beside it. var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.Equal(3, frames.Count); Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256)); Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the frame is no longer missing"); Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct))); Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct), "the original is an alias of its conversion here too"); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } [Test] public async Task AFrameThatTookBackItsPhotoInAnotherEncodingIsDrawnAgain() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Harbour", photos); var engine = await f.StartEngineAsync(); var drawn = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([local.Id], "Harbour"), cancellationToken: f.Ct); await f.Archive.WaitForAsync(drawn.Id, JobState.Done); var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath)); await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn before"); // The server converted that photo. var converted = f.NewPhotos(1)[0]; var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]); remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow)); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); // Nothing was filed, only a frame swapped, which cleared what covered the sequence: the thumbnails it had are drawn again. Assert.Equal(lost.Id, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1].Id); var again = Assert.Contains((await f.Db.Jobs.ListAsync(null, JobKind.Thumbs, 50, null, f.Ct)).Items, j => j.Id != drawn.Id && j.TargetIds.Contains(local.Id)); await f.Archive.WaitForAsync(again.Id, JobState.Done); Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn again"); } [Test] public async Task AFrameTakesBackItsPhotoInAnotherEncodingOnlyWhereTheSequenceIsNow() { // The folder the sequence left holds another sequence now, or nothing. foreach (var taken in new[] { true, false }) { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Harbour", photos); var stranger = taken ? await f.LocalAsync("Stranger", f.NewPhotos(2)) : null; var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath)); await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); // The server converted that photo. var converted = f.NewPhotos(1)[0]; var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]); remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow)); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var engine = await f.StartEngineAsync(); // Another index moves the folder away, then the other sequence into the folder it left; this index has scanned neither. var folder = f.Archive.Paths.Sequence(local.Folder).Folder; var (other, archive) = await f.Archive.OtherWriterAsync(); var moves = stranger is null ? [(local.Id, "moved/{title}")] : new[] { (local.Id, "moved/{title}"), (stranger.Id, "parked/{title}"), (stranger.Id, local.Folder) }; foreach (var (id, template) in moves) await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], template, null, f.Ct), null, f.Ct); var before = taken ? ArchiveFixture.FilesOf(folder) : null; var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error, taken.ToString()); if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched"); else Assert.False(Directory.Exists(folder), "the folder it left is not created again"); Assert.Equal(lost.Sha256, (await f.Db.Frames.GetAsync(lost.Id, f.Ct))!.Sha256); // Once this index's scan found it there, the converted photo takes the frame's place in the folder the sequence is in now. await f.Archive.ScanAsync(); job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.Equal(3, frames.Count); Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256)); Assert.True(frames[1].RelativePath.StartsWith("moved/Harbour/", StringComparison.Ordinal), frames[1].RelativePath); Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct))); Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "nothing missing"); if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched"); else Assert.False(Directory.Exists(folder)); } } [Test] public async Task AConversionMissingHereTakesBackTheOriginalTheServerHolds() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Harbour", photos); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); // This PC converted the second photo, and that file is gone since. var converted = f.NewPhotos(1)[0]; await f.ReplaceLocalAsync(local.Id, 1, converted); var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath)); await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); await f.RefreshAsync(); Assert.Equal(1, (await f.EntryAsync("Harbour")).Link!.MissingLocal); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.Equal(3, frames.Count); Assert.Equal((lost.Id, Sha(photos[1])), (frames[1].Id, frames[1].Sha256)); Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(converted), f.Ct), "the conversion is an alias of the original"); Assert.False(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct)); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } [Test] public async Task AnUploadSendsTheProxyOfAPhotoThisPcConverted() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Holy grail", photos); // The server holds the originals and cannot decode them (HEIF, BRIEF §2.3); this PC converted its copy of the first. var remote = f.Remote.Add("Holy grail", photos); remote.Undecodable = true; await f.ReplaceLocalAsync(local.Id, 0, f.NewPhotos(1)[0]); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var ahead = await f.EntryAsync("Holy grail"); Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); Assert.Equal(3, remote.Frames.Count); Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); } private static async Task AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes, string? kind = null) { var render = new Render { Id = Guid.CreateVersion7(), SequenceId = local.Id, PresetName = "1080p H.264", OutputPath = local.Folder + "/exports/" + name, Codec = "h264", Width = 1920, Height = 1080, Fps = 30, DurationSeconds = 10, Bytes = bytes, Created = DateTimeOffset.UtcNow, ProducedBy = "studio", Kind = kind, }; await f.Db.Renders.InsertAsync(render, f.Ct); return render; } private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes)); /// Waits until no job is waiting or running (the thumbnails and analyses a download queues after it). private static async Task UntilIdleAsync(SyncFixture f) { var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30)); foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running }) { while ((await f.Db.Jobs.ListAsync(state, cancellationToken: f.Ct)).Items.Count > 0) { Assert.True(DateTime.UtcNow < until, $"jobs still {state}"); await Task.Delay(50, f.Ct); } } } /// The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for. private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId) { var project = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(localId, f.Ct))!.SettingsJson!); var frames = await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct); return ([.. project.ExcludedFrames.Select(i => frames[i].Sha256!)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null, [.. project.Camera.Keyframes.Select(k => frames[k.Frame].Sha256!)], [.. (project.PendingFrames ?? []).Select(p => p.Sha256!)]); } }