Files
Chronos/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs
T
Alby96andClaude Opus 5.5 6d87a1bc6d feat(P8): integrate phase 5 (workspace stills, strings, prune)
Wire the phase 5 parcels into one working whole.

- Save frame works in a workspace project. WorkspaceSequenceSource now
  makes still jobs (IProjectStills) routed like its renders: with the
  originals on this PC, a WorkspaceRender still into exports/; with
  proxies only, the home's still job with the portable settings and the
  project id. The tooltip for a source that saves no frames is now a real
  string.
- The Export footer has Save project file again for library sequences;
  workspace projects keep it in their project menu only.
- Sync render downloads are hashed and compared with the server's
  sha256 (one retry, then the job fails without filing the file), and
  uploads and downloads carry the render kind, so a saved frame no longer
  counts as a recent preset.
- Add the missing en/it strings: still frame out of the sequence
  (Core and both job handlers), keyframes not applicable, workspace
  stills, Videos refresh / clear sequence / play-from-server tips, the
  videos-folder creation error, and the admin panel kind labels, play
  and trash tips.
- Delete the export history, batch and card strings no longer used from
  English and prune them from every culture.
- Videos: the sequence chip clears only the sequence filter (desktop and
  admin panel), the desktop remembers cards or list, and a Refresh button.

Refs #21

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 03:16:52 +01:00

1105 lines
64 KiB
C#

using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Core.Render;
using Chronos.Data.Archiving;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Archiving;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Sync;
/// <summary>Upload, download and proxy jobs on the job engine against an in-memory server.</summary>
[Category("sync")]
public sealed class SyncTransferTests(TestContext context)
{
[Test]
public async Task UploadSendsTheOriginalsAndSettingsOfALocalSequence()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos, SyncFixture.Window41);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
Assert.Equal(job.Id, (await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal));
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
Assert.True(remote.Frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived");
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson), "the settings travel with the originals");
var entry = await f.EntryAsync("Comet");
Assert.Equal(SyncStatus.InSync, entry.Status);
Assert.Equal(remote.Id, entry.Link!.RemoteId);
}
[Test]
public async Task AnUploadSendsTheProxiesTheServerCannotMakeItself()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server has the originals already (the Uploader sent them) but cannot decode them (HEIF, BRIEF §2.3).
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
foreach (var proxy in f.Remote.SentProxies.Values)
{
var info = JpegDecoder.ReadInfo(proxy);
Assert.True(info.Width <= 1280 && info.Height <= 1280, "made as the server's policy says");
}
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status);
}
[Test]
public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds
// under the names it gave them, and cannot decode (HEIF, BRIEF §2.3).
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
var remote = f.Remote.Add("Adopted", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync(local.DisplayTitle);
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
}
[Test]
public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Holy grail", photos);
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
var calls = f.Remote.ManifestCalls;
await f.RefreshAsync();
Assert.Equal(calls, f.Remote.ManifestCalls);
// A render node sends them after a job: the server's frame counters stand still, its sequence changes.
foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy;
remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1);
await f.RefreshAsync();
var entry = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote));
}
[Test]
public async Task PhotosOneSideDeletedAreNeitherSentNorFetchedAgain()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
// The server holds the first three and deleted the fourth, which this PC still holds.
var remote = f.Remote.Add("Comet", photos[..3]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(0, (await f.EntryAsync("Comet")).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var upload = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(upload.Id, JobState.Done);
Assert.Equal(3, remote.Frames.Count);
Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
// This PC deletes the first photo, which the server still holds: it is not fetched back, and nothing local is deleted.
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [frames[0].Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal));
var download = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(download.Id, JobState.Done);
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.SequenceEqual(photos[1..].Select(Sha), kept.Select(k => k.Sha256!));
}
[Test]
public async Task APhotoDeletedHereIsNotFetchedAgainFromAServerThatHasNotHashedIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// A folder the server adopted and never verified (its manifest lists no SHA-256), downloaded here.
var remote = f.Remote.Add("Adopted on the server", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(first.Id, JobState.Done);
var local = Assert.NotNull((await f.EntryAsync("Adopted on the server")).Local);
// This PC deletes a photo the server still lists, unhashed: only its content tells it is the one deleted here.
await UntilIdleAsync(f);
var gone = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(photos[0]));
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [gone.Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct);
await f.RefreshAsync();
var again = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
var done = await f.Archive.WaitForAsync(again.Id, JobState.Done);
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted");
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone");
Assert.Contains(Loc.L("Archive.Log.SyncDeletedHere", remote.Frames.Single(fr => fr.Sha256 == Sha(photos[0])).FileName), done.LogTail);
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
}
[Test]
public async Task AnUnverifiedCopyDoesNotSendBackAPhotoTheServerDeleted()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
// An adopted folder (never verified, so no hashes here) that still holds a photo the server has deleted since.
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 4; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
var remote = f.Remote.Add("Adopted", photos[..3]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
// Without its hash the photo cannot be told from one the server lacks: the upload finds out.
Assert.Equal(1, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(3, remote.Frames.Count);
Assert.Contains(Loc.L("Archive.Log.SyncDeletedThere", "DSC_0004.JPG"), done.LogTail);
// Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good.
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify");
var entry = await f.EntryAsync(local.DisplayTitle);
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
var calls = remote.Frames.Count;
var next = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var settled = await f.Archive.WaitForAsync(next.Id, JobState.Done);
Assert.Equal((4L, 4L, calls), (settled.ProgressDone, settled.ProgressTotal, remote.Frames.Count));
}
[Test]
public async Task ATombstoneBesideAPhotoTheServerStillHoldsLeavesTheCopiesInSync()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
// The server holds both photos and a tombstone of the second: it deleted one of two copies of it, or the file came
// back by hand. Neither side lacks anything.
var remote = f.Remote.Add("Comet", photos);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal));
Assert.Equal(SyncStatus.InSync, entry.Status);
}
[Test]
public async Task AResumedUploadSendsOnlyWhatTheServerLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
// A crashed attempt had created the server sequence and sent two originals.
var remote = f.Remote.Add("Comet", photos[..2]);
var engine = await f.StartEngineAsync();
var job = await f.Archive.EnqueueWithCheckpointAsync(engine,
new(JobKind.UploadCommit, [local.Id], "Comet", """{"server":"https://nas.test:8443"}""", JobPriority.User), remote.Id.ToString());
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Count(1, f.Remote.Sequences.Values);
Assert.Equal(4, remote.Frames.Count);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Comet")).Status);
}
[Test]
public async Task OriginalsChangedSinceTheyWereArchivedAreNeverUploaded()
{
await using var f = await SyncFixture.CreateAsync(context);
var local = await f.LocalAsync("Comet", f.NewPhotos(3));
var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
var path = f.Archive.Paths.FromRelative(damaged.RelativePath);
var bytes = await File.ReadAllBytesAsync(path, f.Ct);
bytes[^3] ^= 0xFF;
await File.WriteAllBytesAsync(path, bytes, f.Ct);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains("1", failed.Error);
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
Assert.Equal(2, remote.Frames.Count);
Assert.DoesNotContain(damaged.Sha256!, remote.Frames.Select(r => r.Sha256));
Assert.Equal(SyncStatus.LocalAhead, (await f.EntryAsync("Comet")).Status);
}
[Test]
public async Task DownloadedOriginalsAreVerifiedAndCommittedThroughTheArchive()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var remote = f.Remote.Add("Construction cam", photos, SyncFixture.Window33);
remote.Tags = ["site"];
f.Remote.CorruptOnce = remote.Frames[1].Id;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Construction cam");
Assert.Equal(SyncStatus.InSync, entry.Status);
var local = Assert.NotNull(entry.Local);
Assert.Equal(SequenceOrigin.Server, local.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, local.SettingsJson), "the server's settings came along");
Assert.SequenceEqual(["site"], local.Tags);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
foreach (var frame in frames)
{
var bytes = await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frame.RelativePath), f.Ct);
Assert.Equal(frame.Sha256, Sha(bytes));
}
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "the damaged transfer was fetched again");
Assert.FileExists(f.Archive.Paths.Sequence(local.Folder).ManifestPath);
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
}
[Test]
public async Task DownloadFillsInWhatALocalCopyLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos[..1], SyncFixture.Window41);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
await f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.KeepMine, f.Ct);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(3, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Count);
var entry = await f.EntryAsync("Harbour");
Assert.Equal(SyncStatus.InSync, entry.Status);
Assert.Equal(SequenceOrigin.Local, entry.Local!.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local.SettingsJson), "an existing copy keeps its settings");
}
[Test]
public async Task DownloadingIntoAnUnverifiedCopyHashesItFirstAndFetchesOnlyWhatItLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(5);
// An adopted folder: indexed by a scan, never verified, so its frames have no hash yet.
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"IMG_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
var remote = f.Remote.Add("Adopted", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(2, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingLocal);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(5, frames.Count);
Assert.True(frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "no original arrived twice");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
}
[Test]
public async Task DownloadingOriginalsRestoresTheOnesMissingHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
var place = f.Archive.Paths.FromRelative(lost.RelativePath);
File.Delete(place);
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var behind = await f.EntryAsync("Harbour");
Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(place, f.Ct)));
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the restored original is no longer missing");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task AnOriginalIsRestoredOnlyWhereTheSequenceIsNow()
{
// The folder the sequence left holds another sequence now, or nothing.
foreach (var taken in new[] { true, false })
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var stranger = taken ? await f.LocalAsync("Stranger", f.NewPhotos(2)) : null;
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
// Another index moves the folder away, then the other sequence (first parked elsewhere: its folder is the same
// name with a number) into the folder it left; this index has scanned neither.
var folder = f.Archive.Paths.Sequence(local.Folder).Folder;
var (other, archive) = await f.Archive.OtherWriterAsync();
var moves = stranger is null ? [(local.Id, "moved/{title}")] : new[] { (local.Id, "moved/{title}"), (stranger.Id, "parked/{title}"), (stranger.Id, local.Folder) };
foreach (var (id, template) in moves)
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], template, null, f.Ct), null, f.Ct);
var before = taken ? ArchiveFixture.FilesOf(folder) : null;
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error, taken.ToString());
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched");
else Assert.False(Directory.Exists(folder), "the folder it left is not created again");
// Once this index's scan found it there, the original goes back into the folder the sequence is in now.
await f.Archive.ScanAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var moved = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Folder;
Assert.Equal("moved/Harbour", moved);
var restored = (await f.Db.Frames.GetAsync(lost.Id, f.Ct))!;
Assert.Equal(moved + lost.RelativePath[local.Folder.Length..], restored.RelativePath);
Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(restored.RelativePath), f.Ct)));
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => (fr.Flags & FrameFlags.Missing) == 0), "nothing missing");
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched");
else Assert.False(Directory.Exists(folder));
}
}
[Test]
public async Task OriginalsTheServerHasNotHashedAreDownloadedAndHashedHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// A folder the server adopted and never verified: its manifest lists no SHA-256.
var remote = f.Remote.Add("Adopted on the server", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Adopted on the server");
var frames = await f.Db.Frames.ListBySequenceAsync(entry.Local!.Id, cancellationToken: f.Ct);
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived, hashed here");
// Matched by size and capture time, the copies are the server's originals.
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
// One more photo adopted there: only that one travels.
remote.AddFrame(f.NewPhotos(1)[0]);
await f.RefreshAsync();
Assert.Equal(1, (await f.EntryAsync("Adopted on the server")).Link!.MissingLocal);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(4, (await f.Db.Frames.ListBySequenceAsync(entry.Local.Id, cancellationToken: f.Ct)).Count);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Adopted on the server")).Status);
}
[Test]
public async Task AnUploadDoesNotSendAgainWhatTheServerHoldsWithoutAHash()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Harbour", photos);
// The server adopted the first three photos and never verified them: it cannot recognise them by content.
var remote = f.Remote.Add("Harbour", photos[..3]);
remote.Unhashed = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = (await f.EntryAsync("Harbour")).Link!;
Assert.Equal((1, 0), (ahead.MissingRemote, ahead.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal((4L, 4L), (done.ProgressDone, done.ProgressTotal));
Assert.SequenceEqual(photos.Select(Sha), remote.Frames.Select(r => r.Sha256));
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task ProxiesOfFramesTheServerHasNotHashedAreSentToo()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server adopted the same originals, cannot decode them and has not hashed them.
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
remote.Unhashed = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(3, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Equal((SyncStatus.InSync, 0), ((await f.EntryAsync("Holy grail")).Status, (await f.EntryAsync("Holy grail")).Link!.MissingRemote));
}
[Test]
public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(6);
var local = await f.LocalAsync("Harbour", photos[1..5]);
var remote = f.Remote.Add("Harbour", photos[1..5]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
// Another PC sends the server an earlier and a later photo; then an editor there excludes two photos, picks the
// hero and keys the camera, by the server's frames.
remote.InsertFrame(0, photos[0]);
remote.AddFrame(photos[5]);
f.Remote.EditSettings(remote.Id, new ProjectSettings
{
ExcludedFrames = [0, 3],
HeroFrame = 5,
Camera = new CameraSettings { Keyframes = [new CameraKey(2, 0.4, 0.5, 1.5, 0)] },
}.ToJson());
await f.RefreshAsync();
var taken = await NamedAsync(f, local.Id);
Assert.SequenceEqual([Sha(photos[3])], taken.Excluded);
Assert.Null(taken.Hero);
Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes);
Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them");
// Each waits with its capture time too, whichever side hashed what: the same project reads the same either way.
var waiting = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).PendingFrames!;
Assert.True(waiting.All(p => p.CapturedAt is not null), "a photo waits with its capture time");
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The same photos as on the server, and nothing waits any more.
var arrived = await NamedAsync(f, local.Id);
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], arrived.Excluded);
Assert.Equal(Sha(photos[5]), arrived.Hero);
Assert.SequenceEqual([Sha(photos[2])], arrived.Keyframes);
Assert.Empty(arrived.Pending);
var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(Sha(photos[5]), frames.Single(fr => fr.Id == row.HeroFrameId).Sha256);
Assert.True(SyncRules.SameSettings(row.SettingsJson, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(row.Folder).ProjectPath, f.Ct)),
"project.json mirrors the index");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
// An edit here later sends the server the same photos, by its frames.
var edited = ProjectSettings.FromJson(row.SettingsJson!) with { Exposure = new ExposureSettings { DeflickerWindow = new Auto<int>().WithManual(41) } };
await f.EditLocalSettingsAsync(local.Id, edited.ToJson());
await f.RefreshAsync();
var server = ProjectSettings.FromJson(remote.SettingsJson!);
Assert.Equal(41, server.Exposure.DeflickerWindow.Manual);
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], server.ExcludedFrames.Select(i => remote.Frames[i].Sha256));
Assert.Equal(Sha(photos[5]), remote.Frames[server.HeroFrame!.Value].Sha256);
Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256));
}
[Test]
public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies");
// Proxies arrive on the server (a client or render node sent them); the frames did not change.
remote.Status |= SequenceStatus.ProxiesBuilt;
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
}
[Test]
public async Task ACacheMadeBeforeTheServerDrewItsNewFramesStaysStaleUntilItHasTheirProxies()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Harbour", f.NewPhotos(2));
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
// Two photos reach the server. Its proxy flag stays set until its next proxy pass, which has not drawn them yet:
// the first has its thumbnail only, the second nothing at all.
var added = f.NewPhotos(2).Select(photo => remote.AddFrame(photo)).ToList();
f.Remote.Undrawn.UnionWith([(added[0].Id, "proxy"), (added[1].Id, "proxy"), (added[1].Id, "thumb")]);
await f.RefreshAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Take(3).Select(file => (int)new FileInfo(file!).Length));
Assert.Null(opened.Files[3]);
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale, "a thumbnail or a missing file stands in for a proxy the server offers");
// The pass draws them: the frames stay as they were, the server's sequence records the pass.
f.Remote.Undrawn.Clear();
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
await f.RefreshAsync();
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
}
[Test]
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Star trails", photos);
var remote = f.Remote.Add("Star trails", photos);
var render = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 5000).Select(i => (byte)i)]);
remote.Renders.Add(render);
var still = new FakeRender(Guid.CreateVersion7(), "Star trails - 0002.png", "PNG", [.. Enumerable.Range(0, 700).Select(i => (byte)(i * 5))], RenderKind.Still,
Recorded: string.Empty);
remote.Renders.Add(still);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Star trails");
Assert.Equal((SyncStatus.InSync, 2, 5700L), (entry.Status, entry.Link!.ExportsToGet, entry.Link.ExportsToGetBytes));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var fetched = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == render.Id);
Assert.Equal("4K HEVC", fetched.PresetName);
var file = f.Archive.Paths.FromRelative(fetched.OutputPath);
Assert.Equal(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 4K.mp4"), file);
Assert.SequenceEqual(render.Bytes, await File.ReadAllBytesAsync(file, f.Ct));
Assert.Equal((render.Sha256, (string?)null), (fetched.Sha256, fetched.Kind), "the server's hash, checked on arrival");
// A still the server has not hashed: its kind, and the hash of what arrived.
var saved = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == still.Id);
Assert.Equal((Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(still.Bytes)), RenderKind.Still), (saved.Sha256, saved.Kind));
Assert.Equal(0, (await f.EntryAsync("Star trails")).Link!.ExportsToGet);
// A file unlike the hash the server records is fetched again, then refused: nothing is filed.
var damaged = new FakeRender(Guid.CreateVersion7(), "Star trails 1080p.mp4", "1080p", [1, 2, 3, 4], Recorded: new string('a', 64));
remote.Renders.Add(damaged);
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Sync.Hash", "Star trails 1080p.mp4"), failed.Error);
Assert.Null(await f.Db.Renders.GetAsync(damaged.Id, f.Ct));
Assert.False(File.Exists(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 1080p.mp4")));
}
[Test]
public async Task ASentRenderKeepsItsKind()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var exports = f.Archive.Paths.Sequence(local.Folder).Exports;
Directory.CreateDirectory(exports);
byte[] bytes = [.. Enumerable.Range(0, 900).Select(i => (byte)(i * 11))];
await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour - 0001.png"), bytes, f.Ct);
var still = await AddRenderAsync(f, local, "Harbour - 0001.png", bytes.Length, RenderKind.Still);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var sent = Assert.Contains(remote.Renders, r => r.Id == still.Id);
Assert.Equal(RenderKind.Still, sent.Kind);
}
[Test]
public async Task TheServersRendersGoWhereTheSequenceIsNow()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Star trails", photos);
var remote = f.Remote.Add("Star trails", photos);
var first = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 3000).Select(i => (byte)i)]);
remote.Renders.Add(first);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var left = f.Archive.Paths.Sequence(local.Folder).Folder;
// A Reorganize here moves the folder while the server's renders are listed: the render goes where the sequence is.
string? moved = null;
f.Remote.OnceBeforeManifest = async () =>
{
var plan = await f.Archive.Service.PlanRenameAsync(f.Archive.Archive.Id, [local.Id], "moved/{title}", null, f.Ct);
await f.Archive.Service.ApplyRenameAsync(plan, null, f.Ct);
moved = plan.Steps.Single().ToFolder;
};
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal($"{moved}/exports/Star trails 4K.mp4", (await f.Db.Renders.GetAsync(first.Id, f.Ct))!.OutputPath);
Assert.FileExists(f.Archive.Paths.FromRelative($"{moved}/exports/Star trails 4K.mp4"));
Assert.False(Directory.Exists(left), "the folder it left is not created again");
// Another index moves it: nothing is filed until this index's scan finds it there.
var second = new FakeRender(Guid.CreateVersion7(), "Star trails 1080p.mp4", "1080p", [.. Enumerable.Range(0, 2000).Select(i => (byte)(i * 3))]);
remote.Renders.Add(second);
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
var (other, archive) = await f.Archive.OtherWriterAsync();
var away = await other.PlanRenameAsync(archive.Id, [local.Id], "elsewhere/{title}", null, f.Ct);
await other.ApplyRenameAsync(away, null, f.Ct);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error);
Assert.False(Directory.Exists(f.Archive.Paths.Sequence(moved!).Folder), "the folder it left is not created again");
Assert.Null(await f.Db.Renders.GetAsync(second.Id, f.Ct));
await f.Archive.ScanAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var fetched = (await f.Db.Renders.GetAsync(second.Id, f.Ct))!;
Assert.Equal("elsewhere/Star trails/exports/Star trails 1080p.mp4", fetched.OutputPath);
Assert.SequenceEqual(second.Bytes, await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(fetched.OutputPath), f.Ct));
}
[Test]
public async Task RendersMadeHereAreSentOnRequestAndAGoneFileIsReported()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var exports = f.Archive.Paths.Sequence(local.Folder).Exports;
Directory.CreateDirectory(exports);
var bytes = Enumerable.Range(0, 3000).Select(i => (byte)(i * 7)).ToArray();
await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour 1080p.mp4"), bytes, f.Ct);
var made = await AddRenderAsync(f, local, "Harbour 1080p.mp4", bytes.Length);
var gone = await AddRenderAsync(f, local, "Harbour draft.mp4", 10);
await f.RefreshAsync();
Assert.Equal((1, 3000L), ((await f.EntryAsync("Harbour")).Link!.ExportsToSend, (await f.EntryAsync("Harbour")).Link!.ExportsToSendBytes));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local);
Assert.Equal(SyncTransferKind.SendExports, SyncService.TransferKind(job));
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains("1", failed.Error);
var sent = Assert.Contains(remote.Renders, r => r.Id == made.Id);
Assert.SequenceEqual(bytes, sent.Bytes);
Assert.DoesNotContain(gone.Id, remote.Renders.Select(r => r.Id));
Assert.Equal(0, (await f.EntryAsync("Harbour")).Link!.ExportsToSend);
}
[Test]
public async Task ProxiesOpenARemoteSequenceForTheEditor()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Milan skyline", f.NewPhotos(3), SyncFixture.Window33);
await f.RefreshAsync();
Assert.Null(f.Sync.OpenRemote(f.Server, remote.Id));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Milan skyline", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Milan skyline");
Assert.Equal(SyncStatus.ProxiesOnly, entry.Status);
Assert.Equal(3, entry.Link!.ProxiesCached);
Assert.False(entry.Link.ProxiesStale);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.Equal(remote.Id, opened.Sequence.Id);
Assert.Equal(SequenceOrigin.Server, opened.Sequence.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, opened.Sequence.SettingsJson));
Assert.Equal(3, opened.CachedCount);
var source = opened.OpenFrames(DecoderFactory.CreateServer());
await using (source as IAsyncDisposable)
{
Assert.Equal(3, source.FrameCount);
using var frame = await source.GetFrameAsync(2, 0, DecodePurpose.Render, f.Ct);
Assert.Equal((48, 32), (frame.Width, frame.Height));
}
// Nothing local was created, and a frame added on the server makes the cache stale.
Assert.Null(entry.Local);
remote.AddFrame(f.NewPhotos(1)[0]);
await f.RefreshAsync();
Assert.True((await f.EntryAsync("Milan skyline")).Link!.ProxiesStale);
}
[Test]
public async Task ThumbnailsStandInWhenTheServerHasNoProxies()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.Equal(2, opened.CachedCount);
Assert.True(opened.Files.All(file => new FileInfo(file!).Length == 64), "thumbnails");
}
[Test]
public async Task TransfersFailWithAReasonWhenThisPcIsNotConnected()
{
await using var f = await SyncFixture.CreateAsync(context);
var local = await f.LocalAsync("Comet", f.NewPhotos(1));
var engine = await f.StartEngineAsync(new SyncFixture.Connector(null));
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Sync.NotConnected"), failed.Error);
Assert.Empty(f.Remote.Sequences);
}
[Test]
public async Task APhotoEachSideConvertedIsNeitherSentNorFetchedAgain()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
// Each side converted its own copy of the first photo, to other bytes (another decoder, another quality).
var (mine, theirs) = (f.NewPhotos(1)[0], f.NewPhotos(1)[0]);
await f.ReplaceLocalAsync(local.Id, 0, mine);
var remote = f.Remote.Add("Comet", [theirs, photos[1]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[0]), Sha(theirs), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
var engine = await f.StartEngineAsync();
await f.Archive.WaitForAsync((await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id, JobState.Done);
await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null)).Id,
JobState.Done);
Assert.SequenceEqual([Sha(theirs), Sha(photos[1])], remote.Frames.Select(r => r.Sha256));
Assert.SequenceEqual([Sha(mine), Sha(photos[1])], (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Select(r => r.Sha256!));
}
[Test]
public async Task CopiesConvertedOnOneSideBeforeTheyWereLinkedAreLinkedByContent()
{
await using var f = await SyncFixture.CreateAsync(context);
// The Uploader sent the card to the server, which converted it before this PC compared the two.
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
var converted = f.NewPhotos(3);
var remote = f.Remote.Add("2026-09-28 Canon", converted);
foreach (var (original, replacement) in photos.Zip(converted))
remote.Removed.Add(new RemovedPhotoDto(Sha(original), Sha(replacement), "replaced", DateTimeOffset.UtcNow));
remote.FirstCapture = local.FirstCapture;
remote.CameraModel = local.CameraModel;
// And the other way round: this PC converted its copy of another card the server holds as it came.
var cards = f.NewPhotos(2);
var mine = await f.LocalAsync("Harbour", cards);
foreach (var (index, replacement) in f.NewPhotos(2).Index()) await f.ReplaceLocalAsync(mine.Id, index, replacement);
mine = (await f.Db.Sequences.GetAsync(mine.Id, f.Ct))!;
var theirs = f.Remote.Add("2026-09-28 Canon (2)", cards);
theirs.FirstCapture = mine.FirstCapture;
theirs.CameraModel = mine.CameraModel;
await f.RefreshAsync();
var comet = await f.EntryAsync("Comet");
Assert.Equal(remote.Id, comet.Link?.RemoteId);
Assert.Equal((0, 0), (comet.Link!.MissingRemote, comet.Link.MissingLocal));
var harbour = await f.EntryAsync("Harbour");
Assert.Equal(theirs.Id, harbour.Link?.RemoteId);
Assert.Equal((0, 0), (harbour.Link!.MissingRemote, harbour.Link.MissingLocal));
}
[Test]
public async Task AFrameMissingHereTakesBackItsPhotoAsTheServerConvertedIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
// The server converted that photo.
var converted = f.NewPhotos(1)[0];
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var behind = await f.EntryAsync("Harbour");
Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The same frame, holding the server's conversion, and no frame beside it.
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256));
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the frame is no longer missing");
Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct)));
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct), "the original is an alias of its conversion here too");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task AFrameThatTookBackItsPhotoInAnotherEncodingIsDrawnAgain()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var engine = await f.StartEngineAsync();
var drawn = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([local.Id], "Harbour"), cancellationToken: f.Ct);
await f.Archive.WaitForAsync(drawn.Id, JobState.Done);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn before");
// The server converted that photo.
var converted = f.NewPhotos(1)[0];
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// Nothing was filed, only a frame swapped, which cleared what covered the sequence: the thumbnails it had are drawn again.
Assert.Equal(lost.Id, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1].Id);
var again = Assert.Contains((await f.Db.Jobs.ListAsync(null, JobKind.Thumbs, 50, null, f.Ct)).Items, j => j.Id != drawn.Id && j.TargetIds.Contains(local.Id));
await f.Archive.WaitForAsync(again.Id, JobState.Done);
Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn again");
}
[Test]
public async Task AFrameTakesBackItsPhotoInAnotherEncodingOnlyWhereTheSequenceIsNow()
{
// The folder the sequence left holds another sequence now, or nothing.
foreach (var taken in new[] { true, false })
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var stranger = taken ? await f.LocalAsync("Stranger", f.NewPhotos(2)) : null;
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
// The server converted that photo.
var converted = f.NewPhotos(1)[0];
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
// Another index moves the folder away, then the other sequence into the folder it left; this index has scanned neither.
var folder = f.Archive.Paths.Sequence(local.Folder).Folder;
var (other, archive) = await f.Archive.OtherWriterAsync();
var moves = stranger is null ? [(local.Id, "moved/{title}")] : new[] { (local.Id, "moved/{title}"), (stranger.Id, "parked/{title}"), (stranger.Id, local.Folder) };
foreach (var (id, template) in moves)
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], template, null, f.Ct), null, f.Ct);
var before = taken ? ArchiveFixture.FilesOf(folder) : null;
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error, taken.ToString());
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched");
else Assert.False(Directory.Exists(folder), "the folder it left is not created again");
Assert.Equal(lost.Sha256, (await f.Db.Frames.GetAsync(lost.Id, f.Ct))!.Sha256);
// Once this index's scan found it there, the converted photo takes the frame's place in the folder the sequence is in now.
await f.Archive.ScanAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256));
Assert.True(frames[1].RelativePath.StartsWith("moved/Harbour/", StringComparison.Ordinal), frames[1].RelativePath);
Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct)));
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "nothing missing");
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched");
else Assert.False(Directory.Exists(folder));
}
}
[Test]
public async Task AConversionMissingHereTakesBackTheOriginalTheServerHolds()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
// This PC converted the second photo, and that file is gone since.
var converted = f.NewPhotos(1)[0];
await f.ReplaceLocalAsync(local.Id, 1, converted);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
await f.RefreshAsync();
Assert.Equal(1, (await f.EntryAsync("Harbour")).Link!.MissingLocal);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(photos[1])), (frames[1].Id, frames[1].Sha256));
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(converted), f.Ct), "the conversion is an alias of the original");
Assert.False(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct));
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task AnUploadSendsTheProxyOfAPhotoThisPcConverted()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server holds the originals and cannot decode them (HEIF, BRIEF §2.3); this PC converted its copy of the first.
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.ReplaceLocalAsync(local.Id, 0, f.NewPhotos(1)[0]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Equal(3, remote.Frames.Count);
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
}
private static async Task<Render> AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes, string? kind = null)
{
var render = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = local.Id,
PresetName = "1080p H.264",
OutputPath = local.Folder + "/exports/" + name,
Codec = "h264",
Width = 1920,
Height = 1080,
Fps = 30,
DurationSeconds = 10,
Bytes = bytes,
Created = DateTimeOffset.UtcNow,
ProducedBy = "studio",
Kind = kind,
};
await f.Db.Renders.InsertAsync(render, f.Ct);
return render;
}
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes));
/// <summary>Waits until no job is waiting or running (the thumbnails and analyses a download queues after it).</summary>
private static async Task UntilIdleAsync(SyncFixture f)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30));
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await f.Db.Jobs.ListAsync(state, cancellationToken: f.Ct)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50, f.Ct);
}
}
}
/// <summary>The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for.</summary>
private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId)
{
var project = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(localId, f.Ct))!.SettingsJson!);
var frames = await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct);
return ([.. project.ExcludedFrames.Select(i => frames[i].Sha256!)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null,
[.. project.Camera.Keyframes.Select(k => frames[k.Frame].Sha256!)], [.. (project.PendingFrames ?? []).Select(p => p.Sha256!)]);
}
}