Files
Chronos/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs
T
Alby96 c3756d03c9 fix: keep render filing retryable and writers out of a folder another index moved
Third fix round of owner review 2, phase 4a (Reorganize), cross-cutting items.

- Server render filing now runs under the sequence lease, so it can meet a busy
  sequence or an offline archive. Both completions (client render uploads and
  render-node results) used to answer 500 and leave the upload stuck in
  Committing until a restart. They now answer 503 sequence_busy or
  archive_offline, which clients retry. The upload goes back to receiving with
  its part file and the node keeps its job (UploadService.ReceiveAgainAsync).
- Save both: when this PC's copy cannot be written (another index moved the
  folder), the server's version is no longer applied, even if this index's scan
  relocates the row before the apply would run. A copy that has to be dropped is
  deleted from where the folder is now, as a best effort.
- Export rebase on a folder move (scan and Reorganize): the sequence's own
  export follows the folder unless its file is still at the old path and not at
  the new one. A file written into a re-created old folder therefore keeps a row
  that finds it, so the file is never stranded.
- A render job refuses to start when its folder is not where the index has it,
  which also stops it from re-creating the old folder. Renders fetched by sync
  are now filed under the sequence lease into the current folder, or not at all
  (new string Errors.Archive.FolderMoved, en and it).
- The proxy pass holds the thumbnail lease one batch at a time (a quarter of
  the lease wait), then lets it go for longer than a waiter's poll and reads
  the folder and sidecar again. Before, another process's Reorganize, sequence
  deletion or pack compaction could wait out its lease wait and fail. Updated
  the stale comments on that lease and ARCHITECTURE.md to match.

Tests cover each fix, and each new test fails with its fix reverted. Also
covered: proxy store and pass in a moved folder, RenderStore with the folder
gone, Take server with the folder moved, and an editor save with the folder
moved.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 811509d32a118ed50ad10d47d4b002e534da9d77)
2026-10-27 19:16:28 +01:00

793 lines
44 KiB
C#

using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Core.Render;
using Chronos.Data.Sync;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Sync;
/// <summary>Upload, download and proxy jobs on the job engine against an in-memory server.</summary>
[Category("sync")]
public sealed class SyncTransferTests(TestContext context)
{
[Test]
public async Task UploadSendsTheOriginalsAndSettingsOfALocalSequence()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos, SyncFixture.Window41);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
Assert.Equal(job.Id, (await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal));
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
Assert.True(remote.Frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived");
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson), "the settings travel with the originals");
var entry = await f.EntryAsync("Comet");
Assert.Equal(SyncStatus.InSync, entry.Status);
Assert.Equal(remote.Id, entry.Link!.RemoteId);
}
[Test]
public async Task AnUploadSendsTheProxiesTheServerCannotMakeItself()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server has the originals already (the Uploader sent them) but cannot decode them (HEIF, BRIEF §2.3).
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
foreach (var proxy in f.Remote.SentProxies.Values)
{
var info = JpegDecoder.ReadInfo(proxy);
Assert.True(info.Width <= 1280 && info.Height <= 1280, "made as the server's policy says");
}
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status);
}
[Test]
public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds
// under the names it gave them, and cannot decode (HEIF, BRIEF §2.3).
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
var remote = f.Remote.Add("Adopted", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync(local.DisplayTitle);
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
}
[Test]
public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Holy grail", photos);
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
var calls = f.Remote.ManifestCalls;
await f.RefreshAsync();
Assert.Equal(calls, f.Remote.ManifestCalls);
// A render node sends them after a job: the server's frame counters stand still, its sequence changes.
foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy;
remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1);
await f.RefreshAsync();
var entry = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote));
}
[Test]
public async Task PhotosOneSideDeletedAreNeitherSentNorFetchedAgain()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
// The server holds the first three and deleted the fourth, which this PC still holds.
var remote = f.Remote.Add("Comet", photos[..3]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(0, (await f.EntryAsync("Comet")).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var upload = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(upload.Id, JobState.Done);
Assert.Equal(3, remote.Frames.Count);
Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
// This PC deletes the first photo, which the server still holds: it is not fetched back, and nothing local is deleted.
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [frames[0].Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal));
var download = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(download.Id, JobState.Done);
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.SequenceEqual(photos[1..].Select(Sha), kept.Select(k => k.Sha256!));
}
[Test]
public async Task APhotoDeletedHereIsNotFetchedAgainFromAServerThatHasNotHashedIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// A folder the server adopted and never verified (its manifest lists no SHA-256), downloaded here.
var remote = f.Remote.Add("Adopted on the server", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(first.Id, JobState.Done);
var local = Assert.NotNull((await f.EntryAsync("Adopted on the server")).Local);
// This PC deletes a photo the server still lists, unhashed: only its content tells it is the one deleted here.
await UntilIdleAsync(f);
var gone = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(photos[0]));
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [gone.Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct);
await f.RefreshAsync();
var again = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
var done = await f.Archive.WaitForAsync(again.Id, JobState.Done);
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted");
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone");
Assert.Contains("deleted on this PC, not downloaded", done.LogTail);
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
}
[Test]
public async Task AnUnverifiedCopyDoesNotSendBackAPhotoTheServerDeleted()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
// An adopted folder (never verified, so no hashes here) that still holds a photo the server has deleted since.
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 4; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
var remote = f.Remote.Add("Adopted", photos[..3]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
// Without its hash the photo cannot be told from one the server lacks: the upload finds out.
Assert.Equal(1, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(3, remote.Frames.Count);
Assert.Contains("deleted on the server, not uploaded", done.LogTail);
// Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good.
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify");
var entry = await f.EntryAsync(local.DisplayTitle);
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
var calls = remote.Frames.Count;
var next = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var settled = await f.Archive.WaitForAsync(next.Id, JobState.Done);
Assert.Equal((4L, 4L, calls), (settled.ProgressDone, settled.ProgressTotal, remote.Frames.Count));
}
[Test]
public async Task ATombstoneBesideAPhotoTheServerStillHoldsLeavesTheCopiesInSync()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
// The server holds both photos and a tombstone of the second: it deleted one of two copies of it, or the file came
// back by hand. Neither side lacks anything.
var remote = f.Remote.Add("Comet", photos);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal));
Assert.Equal(SyncStatus.InSync, entry.Status);
}
[Test]
public async Task AResumedUploadSendsOnlyWhatTheServerLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
// A crashed attempt had created the server sequence and sent two originals.
var remote = f.Remote.Add("Comet", photos[..2]);
var engine = await f.StartEngineAsync();
var job = await f.Archive.EnqueueWithCheckpointAsync(engine,
new(JobKind.UploadCommit, [local.Id], "Comet", """{"server":"https://nas.test:8443"}""", JobPriority.User), remote.Id.ToString());
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Count(1, f.Remote.Sequences.Values);
Assert.Equal(4, remote.Frames.Count);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Comet")).Status);
}
[Test]
public async Task OriginalsChangedSinceTheyWereArchivedAreNeverUploaded()
{
await using var f = await SyncFixture.CreateAsync(context);
var local = await f.LocalAsync("Comet", f.NewPhotos(3));
var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
var path = f.Archive.Paths.FromRelative(damaged.RelativePath);
var bytes = await File.ReadAllBytesAsync(path, f.Ct);
bytes[^3] ^= 0xFF;
await File.WriteAllBytesAsync(path, bytes, f.Ct);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains("1", failed.Error);
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
Assert.Equal(2, remote.Frames.Count);
Assert.DoesNotContain(damaged.Sha256!, remote.Frames.Select(r => r.Sha256));
Assert.Equal(SyncStatus.LocalAhead, (await f.EntryAsync("Comet")).Status);
}
[Test]
public async Task DownloadedOriginalsAreVerifiedAndCommittedThroughTheArchive()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var remote = f.Remote.Add("Construction cam", photos, SyncFixture.Window33);
remote.Tags = ["site"];
f.Remote.CorruptOnce = remote.Frames[1].Id;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Construction cam");
Assert.Equal(SyncStatus.InSync, entry.Status);
var local = Assert.NotNull(entry.Local);
Assert.Equal(SequenceOrigin.Server, local.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, local.SettingsJson), "the server's settings came along");
Assert.SequenceEqual(["site"], local.Tags);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
foreach (var frame in frames)
{
var bytes = await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frame.RelativePath), f.Ct);
Assert.Equal(frame.Sha256, Sha(bytes));
}
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "the damaged transfer was fetched again");
Assert.FileExists(f.Archive.Paths.Sequence(local.Folder).ManifestPath);
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
}
[Test]
public async Task DownloadFillsInWhatALocalCopyLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos[..1], SyncFixture.Window41);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
await f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.KeepMine, f.Ct);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(3, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Count);
var entry = await f.EntryAsync("Harbour");
Assert.Equal(SyncStatus.InSync, entry.Status);
Assert.Equal(SequenceOrigin.Local, entry.Local!.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local.SettingsJson), "an existing copy keeps its settings");
}
[Test]
public async Task DownloadingIntoAnUnverifiedCopyHashesItFirstAndFetchesOnlyWhatItLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(5);
// An adopted folder: indexed by a scan, never verified, so its frames have no hash yet.
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"IMG_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
var remote = f.Remote.Add("Adopted", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(2, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingLocal);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(5, frames.Count);
Assert.True(frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "no original arrived twice");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
}
[Test]
public async Task DownloadingOriginalsRestoresTheOnesMissingHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
var place = f.Archive.Paths.FromRelative(lost.RelativePath);
File.Delete(place);
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var behind = await f.EntryAsync("Harbour");
Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(place, f.Ct)));
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the restored original is no longer missing");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task OriginalsTheServerHasNotHashedAreDownloadedAndHashedHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// A folder the server adopted and never verified: its manifest lists no SHA-256.
var remote = f.Remote.Add("Adopted on the server", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Adopted on the server");
var frames = await f.Db.Frames.ListBySequenceAsync(entry.Local!.Id, cancellationToken: f.Ct);
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived, hashed here");
// Matched by size and capture time, the copies are the server's originals.
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
// One more photo adopted there: only that one travels.
remote.AddFrame(f.NewPhotos(1)[0]);
await f.RefreshAsync();
Assert.Equal(1, (await f.EntryAsync("Adopted on the server")).Link!.MissingLocal);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(4, (await f.Db.Frames.ListBySequenceAsync(entry.Local.Id, cancellationToken: f.Ct)).Count);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Adopted on the server")).Status);
}
[Test]
public async Task AnUploadDoesNotSendAgainWhatTheServerHoldsWithoutAHash()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Harbour", photos);
// The server adopted the first three photos and never verified them: it cannot recognise them by content.
var remote = f.Remote.Add("Harbour", photos[..3]);
remote.Unhashed = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = (await f.EntryAsync("Harbour")).Link!;
Assert.Equal((1, 0), (ahead.MissingRemote, ahead.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal((4L, 4L), (done.ProgressDone, done.ProgressTotal));
Assert.SequenceEqual(photos.Select(Sha), remote.Frames.Select(r => r.Sha256));
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task ProxiesOfFramesTheServerHasNotHashedAreSentToo()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server adopted the same originals, cannot decode them and has not hashed them.
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
remote.Unhashed = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(3, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Equal((SyncStatus.InSync, 0), ((await f.EntryAsync("Holy grail")).Status, (await f.EntryAsync("Holy grail")).Link!.MissingRemote));
}
[Test]
public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(6);
var local = await f.LocalAsync("Harbour", photos[1..5]);
var remote = f.Remote.Add("Harbour", photos[1..5]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
// Another PC sends the server an earlier and a later photo; then an editor there excludes two photos, picks the
// hero and keys the camera, by the server's frames.
remote.InsertFrame(0, photos[0]);
remote.AddFrame(photos[5]);
f.Remote.EditSettings(remote.Id, new ProjectSettings
{
ExcludedFrames = [0, 3],
HeroFrame = 5,
Camera = new CameraSettings { Keyframes = [new CameraKey(2, 0.4, 0.5, 1.5, 0)] },
}.ToJson());
await f.RefreshAsync();
var taken = await NamedAsync(f, local.Id);
Assert.SequenceEqual([Sha(photos[3])], taken.Excluded);
Assert.Null(taken.Hero);
Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes);
Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them");
// Each waits with its capture time too, whichever side hashed what: the same project reads the same either way.
var waiting = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).PendingFrames!;
Assert.True(waiting.All(p => p.CapturedAt is not null), "a photo waits with its capture time");
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The same photos as on the server, and nothing waits any more.
var arrived = await NamedAsync(f, local.Id);
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], arrived.Excluded);
Assert.Equal(Sha(photos[5]), arrived.Hero);
Assert.SequenceEqual([Sha(photos[2])], arrived.Keyframes);
Assert.Empty(arrived.Pending);
var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(Sha(photos[5]), frames.Single(fr => fr.Id == row.HeroFrameId).Sha256);
Assert.True(SyncRules.SameSettings(row.SettingsJson, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(row.Folder).ProjectPath, f.Ct)),
"project.json mirrors the index");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
// An edit here later sends the server the same photos, by its frames.
var edited = ProjectSettings.FromJson(row.SettingsJson!) with { Exposure = new ExposureSettings { DeflickerWindow = new Auto<int>().WithManual(41) } };
await f.EditLocalSettingsAsync(local.Id, edited.ToJson());
await f.RefreshAsync();
var server = ProjectSettings.FromJson(remote.SettingsJson!);
Assert.Equal(41, server.Exposure.DeflickerWindow.Manual);
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], server.ExcludedFrames.Select(i => remote.Frames[i].Sha256));
Assert.Equal(Sha(photos[5]), remote.Frames[server.HeroFrame!.Value].Sha256);
Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256));
}
[Test]
public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies");
// Proxies arrive on the server (a client or render node sent them); the frames did not change.
remote.Status |= SequenceStatus.ProxiesBuilt;
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
}
[Test]
public async Task ACacheMadeBeforeTheServerDrewItsNewFramesStaysStaleUntilItHasTheirProxies()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Harbour", f.NewPhotos(2));
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
// Two photos reach the server. Its proxy flag stays set until its next proxy pass, which has not drawn them yet:
// the first has its thumbnail only, the second nothing at all.
var added = f.NewPhotos(2).Select(photo => remote.AddFrame(photo)).ToList();
f.Remote.Undrawn.UnionWith([(added[0].Id, "proxy"), (added[1].Id, "proxy"), (added[1].Id, "thumb")]);
await f.RefreshAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Take(3).Select(file => (int)new FileInfo(file!).Length));
Assert.Null(opened.Files[3]);
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale, "a thumbnail or a missing file stands in for a proxy the server offers");
// The pass draws them: the frames stay as they were, the server's sequence records the pass.
f.Remote.Undrawn.Clear();
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
await f.RefreshAsync();
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
}
[Test]
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Star trails", photos);
var remote = f.Remote.Add("Star trails", photos);
var render = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 5000).Select(i => (byte)i)]);
remote.Renders.Add(render);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Star trails");
Assert.Equal((SyncStatus.InSync, 1, 5000L), (entry.Status, entry.Link!.ExportsToGet, entry.Link.ExportsToGetBytes));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var fetched = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == render.Id);
Assert.Equal("4K HEVC", fetched.PresetName);
var file = f.Archive.Paths.FromRelative(fetched.OutputPath);
Assert.Equal(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 4K.mp4"), file);
Assert.SequenceEqual(render.Bytes, await File.ReadAllBytesAsync(file, f.Ct));
Assert.Equal(0, (await f.EntryAsync("Star trails")).Link!.ExportsToGet);
}
[Test]
public async Task TheServersRendersGoWhereTheSequenceIsNow()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Star trails", photos);
var remote = f.Remote.Add("Star trails", photos);
var first = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 3000).Select(i => (byte)i)]);
remote.Renders.Add(first);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var left = f.Archive.Paths.Sequence(local.Folder).Folder;
// A Reorganize here moves the folder while the server's renders are listed: the render goes where the sequence is.
string? moved = null;
f.Remote.OnceBeforeManifest = async () =>
{
var plan = await f.Archive.Service.PlanRenameAsync(f.Archive.Archive.Id, [local.Id], "moved/{title}", null, f.Ct);
await f.Archive.Service.ApplyRenameAsync(plan, null, f.Ct);
moved = plan.Steps.Single().ToFolder;
};
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal($"{moved}/exports/Star trails 4K.mp4", (await f.Db.Renders.GetAsync(first.Id, f.Ct))!.OutputPath);
Assert.FileExists(f.Archive.Paths.FromRelative($"{moved}/exports/Star trails 4K.mp4"));
Assert.False(Directory.Exists(left), "the folder it left is not created again");
// Another index moves it: nothing is filed until this index's scan finds it there.
var second = new FakeRender(Guid.CreateVersion7(), "Star trails 1080p.mp4", "1080p", [.. Enumerable.Range(0, 2000).Select(i => (byte)(i * 3))]);
remote.Renders.Add(second);
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
var (other, archive) = await f.Archive.OtherWriterAsync();
var away = await other.PlanRenameAsync(archive.Id, [local.Id], "elsewhere/{title}", null, f.Ct);
await other.ApplyRenameAsync(away, null, f.Ct);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error);
Assert.False(Directory.Exists(f.Archive.Paths.Sequence(moved!).Folder), "the folder it left is not created again");
Assert.Null(await f.Db.Renders.GetAsync(second.Id, f.Ct));
await f.Archive.ScanAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var fetched = (await f.Db.Renders.GetAsync(second.Id, f.Ct))!;
Assert.Equal("elsewhere/Star trails/exports/Star trails 1080p.mp4", fetched.OutputPath);
Assert.SequenceEqual(second.Bytes, await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(fetched.OutputPath), f.Ct));
}
[Test]
public async Task RendersMadeHereAreSentOnRequestAndAGoneFileIsReported()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var exports = f.Archive.Paths.Sequence(local.Folder).Exports;
Directory.CreateDirectory(exports);
var bytes = Enumerable.Range(0, 3000).Select(i => (byte)(i * 7)).ToArray();
await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour 1080p.mp4"), bytes, f.Ct);
var made = await AddRenderAsync(f, local, "Harbour 1080p.mp4", bytes.Length);
var gone = await AddRenderAsync(f, local, "Harbour draft.mp4", 10);
await f.RefreshAsync();
Assert.Equal((1, 3000L), ((await f.EntryAsync("Harbour")).Link!.ExportsToSend, (await f.EntryAsync("Harbour")).Link!.ExportsToSendBytes));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local);
Assert.Equal(SyncTransferKind.SendExports, SyncService.TransferKind(job));
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains("1", failed.Error);
var sent = Assert.Contains(remote.Renders, r => r.Id == made.Id);
Assert.SequenceEqual(bytes, sent.Bytes);
Assert.DoesNotContain(gone.Id, remote.Renders.Select(r => r.Id));
Assert.Equal(0, (await f.EntryAsync("Harbour")).Link!.ExportsToSend);
}
[Test]
public async Task ProxiesOpenARemoteSequenceForTheEditor()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Milan skyline", f.NewPhotos(3), SyncFixture.Window33);
await f.RefreshAsync();
Assert.Null(f.Sync.OpenRemote(f.Server, remote.Id));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Milan skyline", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Milan skyline");
Assert.Equal(SyncStatus.ProxiesOnly, entry.Status);
Assert.Equal(3, entry.Link!.ProxiesCached);
Assert.False(entry.Link.ProxiesStale);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.Equal(remote.Id, opened.Sequence.Id);
Assert.Equal(SequenceOrigin.Server, opened.Sequence.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, opened.Sequence.SettingsJson));
Assert.Equal(3, opened.CachedCount);
var source = opened.OpenFrames(DecoderFactory.CreateServer());
await using (source as IAsyncDisposable)
{
Assert.Equal(3, source.FrameCount);
using var frame = await source.GetFrameAsync(2, 0, DecodePurpose.Render, f.Ct);
Assert.Equal((48, 32), (frame.Width, frame.Height));
}
// Nothing local was created, and a frame added on the server makes the cache stale.
Assert.Null(entry.Local);
remote.AddFrame(f.NewPhotos(1)[0]);
await f.RefreshAsync();
Assert.True((await f.EntryAsync("Milan skyline")).Link!.ProxiesStale);
}
[Test]
public async Task ThumbnailsStandInWhenTheServerHasNoProxies()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.Equal(2, opened.CachedCount);
Assert.True(opened.Files.All(file => new FileInfo(file!).Length == 64), "thumbnails");
}
[Test]
public async Task TransfersFailWithAReasonWhenThisPcIsNotConnected()
{
await using var f = await SyncFixture.CreateAsync(context);
var local = await f.LocalAsync("Comet", f.NewPhotos(1));
var engine = await f.StartEngineAsync(new SyncFixture.Connector(null));
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Sync.NotConnected"), failed.Error);
Assert.Empty(f.Remote.Sequences);
}
private static async Task<Render> AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes)
{
var render = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = local.Id,
PresetName = "1080p H.264",
OutputPath = local.Folder + "/exports/" + name,
Codec = "h264",
Width = 1920,
Height = 1080,
Fps = 30,
DurationSeconds = 10,
Bytes = bytes,
Created = DateTimeOffset.UtcNow,
ProducedBy = "studio",
};
await f.Db.Renders.InsertAsync(render, f.Ct);
return render;
}
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes));
/// <summary>Waits until no job is waiting or running (the thumbnails and analyses a download queues after it).</summary>
private static async Task UntilIdleAsync(SyncFixture f)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30));
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await f.Db.Jobs.ListAsync(state, cancellationToken: f.Ct)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50, f.Ct);
}
}
}
/// <summary>The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for.</summary>
private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId)
{
var project = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(localId, f.Ct))!.SettingsJson!);
var frames = await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct);
return ([.. project.ExcludedFrames.Select(i => frames[i].Sha256!)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null,
[.. project.Camera.Keyframes.Select(k => frames[k.Frame].Sha256!)], [.. (project.PendingFrames ?? []).Select(p => p.Sha256!)]);
}
}