using Chronos.Core.Codecs.Jpeg; using Chronos.Core.Imaging; using Chronos.Core.Imaging.Decoders; using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Core.Render; using Chronos.Data.Sync; using Chronos.Protocol; using Chronos.Testing; namespace Chronos.Data.Tests.Sync; /// Upload, download and proxy jobs on the job engine against an in-memory server. [Category("sync")] public sealed class SyncTransferTests(TestContext context) { [Test] public async Task UploadSendsTheOriginalsAndSettingsOfALocalSequence() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Comet", photos, SyncFixture.Window41); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); Assert.Equal(job.Id, (await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id); var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal)); var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet"); Assert.True(remote.Frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived"); Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson), "the settings travel with the originals"); var entry = await f.EntryAsync("Comet"); Assert.Equal(SyncStatus.InSync, entry.Status); Assert.Equal(remote.Id, entry.Link!.RemoteId); } [Test] public async Task AnUploadSendsTheProxiesTheServerCannotMakeItself() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Holy grail", photos); // The server has the originals already (the Uploader sent them) but cannot decode them (HEIF, BRIEF §2.3). var remote = f.Remote.Add("Holy grail", photos); remote.Undecodable = true; await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var ahead = await f.EntryAsync("Holy grail"); Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); foreach (var proxy in f.Remote.SentProxies.Values) { var info = JpegDecoder.ReadInfo(proxy); Assert.True(info.Width <= 1280 && info.Height <= 1280, "made as the server's policy says"); } Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status); } [Test] public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); // An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds // under the names it gave them, and cannot decode (HEIF, BRIEF §2.3). var folder = Path.Combine(f.Archive.Paths.Root, "Adopted"); Directory.CreateDirectory(folder); for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct); await f.Archive.ScanAsync(); var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted"); Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed"); var remote = f.Remote.Add("Adopted", photos); remote.Undecodable = true; await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var ahead = await f.EntryAsync(local.DisplayTitle); Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status); } [Test] public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Holy grail", photos); var remote = f.Remote.Add("Holy grail", photos); remote.Undecodable = true; await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote); var calls = f.Remote.ManifestCalls; await f.RefreshAsync(); Assert.Equal(calls, f.Remote.ManifestCalls); // A render node sends them after a job: the server's frame counters stand still, its sequence changes. foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy; remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1); await f.RefreshAsync(); var entry = await f.EntryAsync("Holy grail"); Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote)); } [Test] public async Task PhotosOneSideDeletedAreNeitherSentNorFetchedAgain() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(4); var local = await f.LocalAsync("Comet", photos); // The server holds the first three and deleted the fourth, which this PC still holds. var remote = f.Remote.Add("Comet", photos[..3]); remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow)); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); Assert.Equal(0, (await f.EntryAsync("Comet")).Link!.MissingRemote); var engine = await f.StartEngineAsync(); var upload = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); await f.Archive.WaitForAsync(upload.Id, JobState.Done); Assert.Equal(3, remote.Frames.Count); Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)); // This PC deletes the first photo, which the server still holds: it is not fetched back, and nothing local is deleted. var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [frames[0].Id], sequence.FramesVersion, f.Ct); await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct); await f.RefreshAsync(); var entry = await f.EntryAsync("Comet"); Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal)); var download = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(download.Id, JobState.Done); var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.SequenceEqual(photos[1..].Select(Sha), kept.Select(k => k.Sha256!)); } [Test] public async Task AResumedUploadSendsOnlyWhatTheServerLacks() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(4); var local = await f.LocalAsync("Comet", photos); // A crashed attempt had created the server sequence and sent two originals. var remote = f.Remote.Add("Comet", photos[..2]); var engine = await f.StartEngineAsync(); var job = await f.Archive.EnqueueWithCheckpointAsync(engine, new(JobKind.UploadCommit, [local.Id], "Comet", """{"server":"https://nas.test:8443"}""", JobPriority.User), remote.Id.ToString()); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Count(1, f.Remote.Sequences.Values); Assert.Equal(4, remote.Frames.Count); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Comet")).Status); } [Test] public async Task OriginalsChangedSinceTheyWereArchivedAreNeverUploaded() { await using var f = await SyncFixture.CreateAsync(context); var local = await f.LocalAsync("Comet", f.NewPhotos(3)); var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; var path = f.Archive.Paths.FromRelative(damaged.RelativePath); var bytes = await File.ReadAllBytesAsync(path, f.Ct); bytes[^3] ^= 0xFF; await File.WriteAllBytesAsync(path, bytes, f.Ct); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Contains("1", failed.Error); var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet"); Assert.Equal(2, remote.Frames.Count); Assert.DoesNotContain(damaged.Sha256!, remote.Frames.Select(r => r.Sha256)); Assert.Equal(SyncStatus.LocalAhead, (await f.EntryAsync("Comet")).Status); } [Test] public async Task DownloadedOriginalsAreVerifiedAndCommittedThroughTheArchive() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var remote = f.Remote.Add("Construction cam", photos, SyncFixture.Window33); remote.Tags = ["site"]; f.Remote.CorruptOnce = remote.Frames[1].Id; await f.RefreshAsync(); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Originals, f.Archive.Archive.Id); await f.Archive.WaitForAsync(job.Id, JobState.Done); var entry = await f.EntryAsync("Construction cam"); Assert.Equal(SyncStatus.InSync, entry.Status); var local = Assert.NotNull(entry.Local); Assert.Equal(SequenceOrigin.Server, local.Origin); Assert.True(SyncRules.SameSettings(SyncFixture.Window33, local.SettingsJson), "the server's settings came along"); Assert.SequenceEqual(["site"], local.Tags); var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.Equal(3, frames.Count); foreach (var frame in frames) { var bytes = await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frame.RelativePath), f.Ct); Assert.Equal(frame.Sha256, Sha(bytes)); } Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "the damaged transfer was fetched again"); Assert.FileExists(f.Archive.Paths.Sequence(local.Folder).ManifestPath); Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up"); } [Test] public async Task DownloadFillsInWhatALocalCopyLacks() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Harbour", photos[..1], SyncFixture.Window41); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); await f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.KeepMine, f.Ct); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal(3, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Count); var entry = await f.EntryAsync("Harbour"); Assert.Equal(SyncStatus.InSync, entry.Status); Assert.Equal(SequenceOrigin.Local, entry.Local!.Origin); Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local.SettingsJson), "an existing copy keeps its settings"); } [Test] public async Task DownloadingIntoAnUnverifiedCopyHashesItFirstAndFetchesOnlyWhatItLacks() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(5); // An adopted folder: indexed by a scan, never verified, so its frames have no hash yet. var folder = Path.Combine(f.Archive.Paths.Root, "Adopted"); Directory.CreateDirectory(folder); for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"IMG_{i + 1:0000}.JPG"), photos[i], f.Ct); await f.Archive.ScanAsync(); var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted"); Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed"); var remote = f.Remote.Add("Adopted", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); Assert.Equal(2, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingLocal); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.Equal(5, frames.Count); Assert.True(frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "no original arrived twice"); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status); } [Test] public async Task DownloadingOriginalsRestoresTheOnesMissingHere() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Harbour", photos); var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; var place = f.Archive.Paths.FromRelative(lost.RelativePath); File.Delete(place); await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var behind = await f.EntryAsync("Harbour"); Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(place, f.Ct))); var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.Equal(3, frames.Count); Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the restored original is no longer missing"); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } [Test] public async Task OriginalsTheServerHasNotHashedAreDownloadedAndHashedHere() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); // A folder the server adopted and never verified: its manifest lists no SHA-256. var remote = f.Remote.Add("Adopted on the server", photos); remote.Unhashed = true; await f.RefreshAsync(); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id); await f.Archive.WaitForAsync(job.Id, JobState.Done); var entry = await f.EntryAsync("Adopted on the server"); var frames = await f.Db.Frames.ListBySequenceAsync(entry.Local!.Id, cancellationToken: f.Ct); Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived, hashed here"); // Matched by size and capture time, the copies are the server's originals. Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal)); // One more photo adopted there: only that one travels. remote.AddFrame(f.NewPhotos(1)[0]); await f.RefreshAsync(); Assert.Equal(1, (await f.EntryAsync("Adopted on the server")).Link!.MissingLocal); job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal(4, (await f.Db.Frames.ListBySequenceAsync(entry.Local.Id, cancellationToken: f.Ct)).Count); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Adopted on the server")).Status); } [Test] public async Task AnUploadDoesNotSendAgainWhatTheServerHoldsWithoutAHash() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(4); var local = await f.LocalAsync("Harbour", photos); // The server adopted the first three photos and never verified them: it cannot recognise them by content. var remote = f.Remote.Add("Harbour", photos[..3]); remote.Unhashed = true; await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var ahead = (await f.EntryAsync("Harbour")).Link!; Assert.Equal((1, 0), (ahead.MissingRemote, ahead.MissingLocal)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal((4L, 4L), (done.ProgressDone, done.ProgressTotal)); Assert.SequenceEqual(photos.Select(Sha), remote.Frames.Select(r => r.Sha256)); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } [Test] public async Task ProxiesOfFramesTheServerHasNotHashedAreSentToo() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Holy grail", photos); // The server adopted the same originals, cannot decode them and has not hashed them. var remote = f.Remote.Add("Holy grail", photos); remote.Undecodable = true; remote.Unhashed = true; await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); Assert.Equal(3, (await f.EntryAsync("Holy grail")).Link!.MissingRemote); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); Assert.Equal((SyncStatus.InSync, 0), ((await f.EntryAsync("Holy grail")).Status, (await f.EntryAsync("Holy grail")).Link!.MissingRemote)); } [Test] public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(6); var local = await f.LocalAsync("Harbour", photos[1..5]); var remote = f.Remote.Add("Harbour", photos[1..5]); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); // Another PC sends the server an earlier and a later photo; then an editor there excludes two photos, picks the // hero and keys the camera, by the server's frames. remote.InsertFrame(0, photos[0]); remote.AddFrame(photos[5]); f.Remote.EditSettings(remote.Id, new ProjectSettings { ExcludedFrames = [0, 3], HeroFrame = 5, Camera = new CameraSettings { Keyframes = [new CameraKey(2, 0.4, 0.5, 1.5, 0)] }, }.ToJson()); await f.RefreshAsync(); var taken = await NamedAsync(f, local.Id); Assert.SequenceEqual([Sha(photos[3])], taken.Excluded); Assert.Null(taken.Hero); Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes); Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them"); // Each waits with its capture time too, whichever side hashed what: the same project reads the same either way. var waiting = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).PendingFrames!; Assert.True(waiting.All(p => p.CapturedAt is not null), "a photo waits with its capture time"); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); // The same photos as on the server, and nothing waits any more. var arrived = await NamedAsync(f, local.Id); Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], arrived.Excluded); Assert.Equal(Sha(photos[5]), arrived.Hero); Assert.SequenceEqual([Sha(photos[2])], arrived.Keyframes); Assert.Empty(arrived.Pending); var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.Equal(Sha(photos[5]), frames.Single(fr => fr.Id == row.HeroFrameId).Sha256); Assert.True(SyncRules.SameSettings(row.SettingsJson, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(row.Folder).ProjectPath, f.Ct)), "project.json mirrors the index"); Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); // An edit here later sends the server the same photos, by its frames. var edited = ProjectSettings.FromJson(row.SettingsJson!) with { Exposure = new ExposureSettings { DeflickerWindow = new Auto().WithManual(41) } }; await f.EditLocalSettingsAsync(local.Id, edited.ToJson()); await f.RefreshAsync(); var server = ProjectSettings.FromJson(remote.SettingsJson!); Assert.Equal(41, server.Exposure.DeflickerWindow.Manual); Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], server.ExcludedFrames.Select(i => remote.Frames[i].Sha256)); Assert.Equal(Sha(photos[5]), remote.Frames[server.HeroFrame!.Value].Sha256); Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256)); } [Test] public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy() { await using var f = await SyncFixture.CreateAsync(context); var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt); await f.RefreshAsync(); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in"); Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies"); // Proxies arrive on the server (a client or render node sent them); the frames did not change. remote.Status |= SequenceStatus.ProxiesBuilt; job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length)); Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale); } [Test] public async Task ACacheMadeBeforeTheServerDrewItsNewFramesStaysStaleUntilItHasTheirProxies() { await using var f = await SyncFixture.CreateAsync(context); var remote = f.Remote.Add("Harbour", f.NewPhotos(2)); await f.RefreshAsync(); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale); // Two photos reach the server. Its proxy flag stays set until its next proxy pass, which has not drawn them yet: // the first has its thumbnail only, the second nothing at all. var added = f.NewPhotos(2).Select(photo => remote.AddFrame(photo)).ToList(); f.Remote.Undrawn.UnionWith([(added[0].Id, "proxy"), (added[1].Id, "proxy"), (added[1].Id, "thumb")]); await f.RefreshAsync(); job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Take(3).Select(file => (int)new FileInfo(file!).Length)); Assert.Null(opened.Files[3]); Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale, "a thumbnail or a missing file stands in for a proxy the server offers"); // The pass draws them: the frames stay as they were, the server's sequence records the pass. f.Remote.Undrawn.Clear(); remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1); await f.RefreshAsync(); Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale); job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length)); Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale); } [Test] public async Task TheServersRendersAreFetchedIntoTheLocalExports() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Star trails", photos); var remote = f.Remote.Add("Star trails", photos); var render = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 5000).Select(i => (byte)i)]); remote.Renders.Add(render); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await f.RefreshAsync(); var entry = await f.EntryAsync("Star trails"); Assert.Equal((SyncStatus.InSync, 1, 5000L), (entry.Status, entry.Link!.ExportsToGet, entry.Link.ExportsToGetBytes)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var fetched = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == render.Id); Assert.Equal("4K HEVC", fetched.PresetName); var file = f.Archive.Paths.FromRelative(fetched.OutputPath); Assert.Equal(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 4K.mp4"), file); Assert.SequenceEqual(render.Bytes, await File.ReadAllBytesAsync(file, f.Ct)); Assert.Equal(0, (await f.EntryAsync("Star trails")).Link!.ExportsToGet); } [Test] public async Task RendersMadeHereAreSentOnRequestAndAGoneFileIsReported() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Harbour", photos); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); var exports = f.Archive.Paths.Sequence(local.Folder).Exports; Directory.CreateDirectory(exports); var bytes = Enumerable.Range(0, 3000).Select(i => (byte)(i * 7)).ToArray(); await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour 1080p.mp4"), bytes, f.Ct); var made = await AddRenderAsync(f, local, "Harbour 1080p.mp4", bytes.Length); var gone = await AddRenderAsync(f, local, "Harbour draft.mp4", 10); await f.RefreshAsync(); Assert.Equal((1, 3000L), ((await f.EntryAsync("Harbour")).Link!.ExportsToSend, (await f.EntryAsync("Harbour")).Link!.ExportsToSendBytes)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local); Assert.Equal(SyncTransferKind.SendExports, SyncService.TransferKind(job)); var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Contains("1", failed.Error); var sent = Assert.Contains(remote.Renders, r => r.Id == made.Id); Assert.SequenceEqual(bytes, sent.Bytes); Assert.DoesNotContain(gone.Id, remote.Renders.Select(r => r.Id)); Assert.Equal(0, (await f.EntryAsync("Harbour")).Link!.ExportsToSend); } [Test] public async Task ProxiesOpenARemoteSequenceForTheEditor() { await using var f = await SyncFixture.CreateAsync(context); var remote = f.Remote.Add("Milan skyline", f.NewPhotos(3), SyncFixture.Window33); await f.RefreshAsync(); Assert.Null(f.Sync.OpenRemote(f.Server, remote.Id)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Milan skyline", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var entry = await f.EntryAsync("Milan skyline"); Assert.Equal(SyncStatus.ProxiesOnly, entry.Status); Assert.Equal(3, entry.Link!.ProxiesCached); Assert.False(entry.Link.ProxiesStale); var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.Equal(remote.Id, opened.Sequence.Id); Assert.Equal(SequenceOrigin.Server, opened.Sequence.Origin); Assert.True(SyncRules.SameSettings(SyncFixture.Window33, opened.Sequence.SettingsJson)); Assert.Equal(3, opened.CachedCount); var source = opened.OpenFrames(DecoderFactory.CreateServer()); await using (source as IAsyncDisposable) { Assert.Equal(3, source.FrameCount); using var frame = await source.GetFrameAsync(2, 0, DecodePurpose.Render, f.Ct); Assert.Equal((48, 32), (frame.Width, frame.Height)); } // Nothing local was created, and a frame added on the server makes the cache stale. Assert.Null(entry.Local); remote.AddFrame(f.NewPhotos(1)[0]); await f.RefreshAsync(); Assert.True((await f.EntryAsync("Milan skyline")).Link!.ProxiesStale); } [Test] public async Task ThumbnailsStandInWhenTheServerHasNoProxies() { await using var f = await SyncFixture.CreateAsync(context); var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(job.Id, JobState.Done); var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.Equal(2, opened.CachedCount); Assert.True(opened.Files.All(file => new FileInfo(file!).Length == 64), "thumbnails"); } [Test] public async Task TransfersFailWithAReasonWhenThisPcIsNotConnected() { await using var f = await SyncFixture.CreateAsync(context); var local = await f.LocalAsync("Comet", f.NewPhotos(1)); var engine = await f.StartEngineAsync(new SyncFixture.Connector(null)); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Sync.NotConnected"), failed.Error); Assert.Empty(f.Remote.Sequences); } private static async Task AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes) { var render = new Render { Id = Guid.CreateVersion7(), SequenceId = local.Id, PresetName = "1080p H.264", OutputPath = local.Folder + "/exports/" + name, Codec = "h264", Width = 1920, Height = 1080, Fps = 30, DurationSeconds = 10, Bytes = bytes, Created = DateTimeOffset.UtcNow, ProducedBy = "studio", }; await f.Db.Renders.InsertAsync(render, f.Ct); return render; } private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes)); /// The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for. private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId) { var project = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(localId, f.Ct))!.SettingsJson!); var frames = await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct); return ([.. project.ExcludedFrames.Select(i => frames[i].Sha256!)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null, [.. project.Camera.Keyframes.Select(k => frames[k.Frame].Sha256!)], [.. (project.PendingFrames ?? []).Select(p => p.Sha256!)]); } }