Files
Chronos/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs
T
Alby96andClaude Opus 5.5 e3cbe30b29 fix: conversions on the released phase 4a keep out of folders a sequence left
Phase 4b and its first fix round were written on phase 4a before its rounds 4-6. Moved onto the
released 4a, the conversion now follows 4a's rule for writers into a sequence's folder
(SequenceFolders.Holds), and the parallel fixes meet:

- The job leaves a sequence another index moved away (its folder missing, or holding another
  sequence's sidecar) as Blocked with Errors.Archive.FolderMoved, nothing settled, staged or
  replaced there; the retry after this index's scan converts it where it is. It read the
  sidecar there and reported the reader's technical text.
- New sequence reads the source's project under the source's lease, where its sidecar was just
  read as its own, and writes the new sequence's project under that sequence's lease only into
  a folder that holds it. It read the project from the folder the row named afterwards, without
  a lease, and wrote through the store without the check.
- Download originals (resolved in the cherry-pick of the trash and sync fix, tested here) swaps
  a copy fetched in another encoding into its missing frame after the transfers, as round 6
  restores originals, and only once the sequence is found in the folder the index has;
  otherwise the copies stay staged and the job fails with FolderMoved after its other work.
  The swap ran inside the transfer loop and failed the job at the first refusal.
- The New sequence wording of the three skip reasons the codecs fix added (Archive.Convert.
  NotCopied.EightBit, ColorProfile, TooLarge, en and it): the server and the desktop look them
  up by reason.
- The server's free space test sets the staging folder's figures too: the job fix measures the
  tighter of the archive root's and the staging folder's volumes.

Tests: a conversion of a sequence another index moved (its folder taken or empty), and a
download whose missing frame takes the server's conversion where the sequence is now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 22:33:11 +01:00

1034 lines
59 KiB
C#

using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Core.Render;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Archiving;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Sync;
/// <summary>Upload, download and proxy jobs on the job engine against an in-memory server.</summary>
[Category("sync")]
public sealed class SyncTransferTests(TestContext context)
{
[Test]
public async Task UploadSendsTheOriginalsAndSettingsOfALocalSequence()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos, SyncFixture.Window41);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
Assert.Equal(job.Id, (await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal));
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
Assert.True(remote.Frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived");
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson), "the settings travel with the originals");
var entry = await f.EntryAsync("Comet");
Assert.Equal(SyncStatus.InSync, entry.Status);
Assert.Equal(remote.Id, entry.Link!.RemoteId);
}
[Test]
public async Task AnUploadSendsTheProxiesTheServerCannotMakeItself()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server has the originals already (the Uploader sent them) but cannot decode them (HEIF, BRIEF §2.3).
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
foreach (var proxy in f.Remote.SentProxies.Values)
{
var info = JpegDecoder.ReadInfo(proxy);
Assert.True(info.Width <= 1280 && info.Height <= 1280, "made as the server's policy says");
}
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status);
}
[Test]
public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds
// under the names it gave them, and cannot decode (HEIF, BRIEF §2.3).
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
var remote = f.Remote.Add("Adopted", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync(local.DisplayTitle);
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
}
[Test]
public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Holy grail", photos);
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
var calls = f.Remote.ManifestCalls;
await f.RefreshAsync();
Assert.Equal(calls, f.Remote.ManifestCalls);
// A render node sends them after a job: the server's frame counters stand still, its sequence changes.
foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy;
remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1);
await f.RefreshAsync();
var entry = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote));
}
[Test]
public async Task PhotosOneSideDeletedAreNeitherSentNorFetchedAgain()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
// The server holds the first three and deleted the fourth, which this PC still holds.
var remote = f.Remote.Add("Comet", photos[..3]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(0, (await f.EntryAsync("Comet")).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var upload = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(upload.Id, JobState.Done);
Assert.Equal(3, remote.Frames.Count);
Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
// This PC deletes the first photo, which the server still holds: it is not fetched back, and nothing local is deleted.
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [frames[0].Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal));
var download = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(download.Id, JobState.Done);
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.SequenceEqual(photos[1..].Select(Sha), kept.Select(k => k.Sha256!));
}
[Test]
public async Task APhotoDeletedHereIsNotFetchedAgainFromAServerThatHasNotHashedIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// A folder the server adopted and never verified (its manifest lists no SHA-256), downloaded here.
var remote = f.Remote.Add("Adopted on the server", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(first.Id, JobState.Done);
var local = Assert.NotNull((await f.EntryAsync("Adopted on the server")).Local);
// This PC deletes a photo the server still lists, unhashed: only its content tells it is the one deleted here.
await UntilIdleAsync(f);
var gone = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(photos[0]));
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [gone.Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct);
await f.RefreshAsync();
var again = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
var done = await f.Archive.WaitForAsync(again.Id, JobState.Done);
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted");
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone");
Assert.Contains("deleted on this PC, not downloaded", done.LogTail);
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
}
[Test]
public async Task AnUnverifiedCopyDoesNotSendBackAPhotoTheServerDeleted()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
// An adopted folder (never verified, so no hashes here) that still holds a photo the server has deleted since.
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 4; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
var remote = f.Remote.Add("Adopted", photos[..3]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
// Without its hash the photo cannot be told from one the server lacks: the upload finds out.
Assert.Equal(1, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(3, remote.Frames.Count);
Assert.Contains("deleted on the server, not uploaded", done.LogTail);
// Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good.
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify");
var entry = await f.EntryAsync(local.DisplayTitle);
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
var calls = remote.Frames.Count;
var next = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var settled = await f.Archive.WaitForAsync(next.Id, JobState.Done);
Assert.Equal((4L, 4L, calls), (settled.ProgressDone, settled.ProgressTotal, remote.Frames.Count));
}
[Test]
public async Task ATombstoneBesideAPhotoTheServerStillHoldsLeavesTheCopiesInSync()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
// The server holds both photos and a tombstone of the second: it deleted one of two copies of it, or the file came
// back by hand. Neither side lacks anything.
var remote = f.Remote.Add("Comet", photos);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal));
Assert.Equal(SyncStatus.InSync, entry.Status);
}
[Test]
public async Task AResumedUploadSendsOnlyWhatTheServerLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
// A crashed attempt had created the server sequence and sent two originals.
var remote = f.Remote.Add("Comet", photos[..2]);
var engine = await f.StartEngineAsync();
var job = await f.Archive.EnqueueWithCheckpointAsync(engine,
new(JobKind.UploadCommit, [local.Id], "Comet", """{"server":"https://nas.test:8443"}""", JobPriority.User), remote.Id.ToString());
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Count(1, f.Remote.Sequences.Values);
Assert.Equal(4, remote.Frames.Count);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Comet")).Status);
}
[Test]
public async Task OriginalsChangedSinceTheyWereArchivedAreNeverUploaded()
{
await using var f = await SyncFixture.CreateAsync(context);
var local = await f.LocalAsync("Comet", f.NewPhotos(3));
var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
var path = f.Archive.Paths.FromRelative(damaged.RelativePath);
var bytes = await File.ReadAllBytesAsync(path, f.Ct);
bytes[^3] ^= 0xFF;
await File.WriteAllBytesAsync(path, bytes, f.Ct);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains("1", failed.Error);
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
Assert.Equal(2, remote.Frames.Count);
Assert.DoesNotContain(damaged.Sha256!, remote.Frames.Select(r => r.Sha256));
Assert.Equal(SyncStatus.LocalAhead, (await f.EntryAsync("Comet")).Status);
}
[Test]
public async Task DownloadedOriginalsAreVerifiedAndCommittedThroughTheArchive()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var remote = f.Remote.Add("Construction cam", photos, SyncFixture.Window33);
remote.Tags = ["site"];
f.Remote.CorruptOnce = remote.Frames[1].Id;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Construction cam");
Assert.Equal(SyncStatus.InSync, entry.Status);
var local = Assert.NotNull(entry.Local);
Assert.Equal(SequenceOrigin.Server, local.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, local.SettingsJson), "the server's settings came along");
Assert.SequenceEqual(["site"], local.Tags);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
foreach (var frame in frames)
{
var bytes = await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frame.RelativePath), f.Ct);
Assert.Equal(frame.Sha256, Sha(bytes));
}
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "the damaged transfer was fetched again");
Assert.FileExists(f.Archive.Paths.Sequence(local.Folder).ManifestPath);
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
}
[Test]
public async Task DownloadFillsInWhatALocalCopyLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos[..1], SyncFixture.Window41);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
await f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.KeepMine, f.Ct);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(3, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Count);
var entry = await f.EntryAsync("Harbour");
Assert.Equal(SyncStatus.InSync, entry.Status);
Assert.Equal(SequenceOrigin.Local, entry.Local!.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local.SettingsJson), "an existing copy keeps its settings");
}
[Test]
public async Task DownloadingIntoAnUnverifiedCopyHashesItFirstAndFetchesOnlyWhatItLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(5);
// An adopted folder: indexed by a scan, never verified, so its frames have no hash yet.
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"IMG_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
var remote = f.Remote.Add("Adopted", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(2, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingLocal);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(5, frames.Count);
Assert.True(frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "no original arrived twice");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
}
[Test]
public async Task DownloadingOriginalsRestoresTheOnesMissingHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
var place = f.Archive.Paths.FromRelative(lost.RelativePath);
File.Delete(place);
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var behind = await f.EntryAsync("Harbour");
Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(place, f.Ct)));
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the restored original is no longer missing");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task AnOriginalIsRestoredOnlyWhereTheSequenceIsNow()
{
// The folder the sequence left holds another sequence now, or nothing.
foreach (var taken in new[] { true, false })
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var stranger = taken ? await f.LocalAsync("Stranger", f.NewPhotos(2)) : null;
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
// Another index moves the folder away, then the other sequence (first parked elsewhere: its folder is the same
// name with a number) into the folder it left; this index has scanned neither.
var folder = f.Archive.Paths.Sequence(local.Folder).Folder;
var (other, archive) = await f.Archive.OtherWriterAsync();
var moves = stranger is null ? [(local.Id, "moved/{title}")] : new[] { (local.Id, "moved/{title}"), (stranger.Id, "parked/{title}"), (stranger.Id, local.Folder) };
foreach (var (id, template) in moves)
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], template, null, f.Ct), null, f.Ct);
var before = taken ? ArchiveFixture.FilesOf(folder) : null;
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error, taken.ToString());
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched");
else Assert.False(Directory.Exists(folder), "the folder it left is not created again");
// Once this index's scan found it there, the original goes back into the folder the sequence is in now.
await f.Archive.ScanAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var moved = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Folder;
Assert.Equal("moved/Harbour", moved);
var restored = (await f.Db.Frames.GetAsync(lost.Id, f.Ct))!;
Assert.Equal(moved + lost.RelativePath[local.Folder.Length..], restored.RelativePath);
Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(restored.RelativePath), f.Ct)));
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => (fr.Flags & FrameFlags.Missing) == 0), "nothing missing");
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched");
else Assert.False(Directory.Exists(folder));
}
}
[Test]
public async Task OriginalsTheServerHasNotHashedAreDownloadedAndHashedHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// A folder the server adopted and never verified: its manifest lists no SHA-256.
var remote = f.Remote.Add("Adopted on the server", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Adopted on the server");
var frames = await f.Db.Frames.ListBySequenceAsync(entry.Local!.Id, cancellationToken: f.Ct);
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived, hashed here");
// Matched by size and capture time, the copies are the server's originals.
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
// One more photo adopted there: only that one travels.
remote.AddFrame(f.NewPhotos(1)[0]);
await f.RefreshAsync();
Assert.Equal(1, (await f.EntryAsync("Adopted on the server")).Link!.MissingLocal);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(4, (await f.Db.Frames.ListBySequenceAsync(entry.Local.Id, cancellationToken: f.Ct)).Count);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Adopted on the server")).Status);
}
[Test]
public async Task AnUploadDoesNotSendAgainWhatTheServerHoldsWithoutAHash()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Harbour", photos);
// The server adopted the first three photos and never verified them: it cannot recognise them by content.
var remote = f.Remote.Add("Harbour", photos[..3]);
remote.Unhashed = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = (await f.EntryAsync("Harbour")).Link!;
Assert.Equal((1, 0), (ahead.MissingRemote, ahead.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal((4L, 4L), (done.ProgressDone, done.ProgressTotal));
Assert.SequenceEqual(photos.Select(Sha), remote.Frames.Select(r => r.Sha256));
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task ProxiesOfFramesTheServerHasNotHashedAreSentToo()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server adopted the same originals, cannot decode them and has not hashed them.
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
remote.Unhashed = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(3, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Equal((SyncStatus.InSync, 0), ((await f.EntryAsync("Holy grail")).Status, (await f.EntryAsync("Holy grail")).Link!.MissingRemote));
}
[Test]
public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(6);
var local = await f.LocalAsync("Harbour", photos[1..5]);
var remote = f.Remote.Add("Harbour", photos[1..5]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
// Another PC sends the server an earlier and a later photo; then an editor there excludes two photos, picks the
// hero and keys the camera, by the server's frames.
remote.InsertFrame(0, photos[0]);
remote.AddFrame(photos[5]);
f.Remote.EditSettings(remote.Id, new ProjectSettings
{
ExcludedFrames = [0, 3],
HeroFrame = 5,
Camera = new CameraSettings { Keyframes = [new CameraKey(2, 0.4, 0.5, 1.5, 0)] },
}.ToJson());
await f.RefreshAsync();
var taken = await NamedAsync(f, local.Id);
Assert.SequenceEqual([Sha(photos[3])], taken.Excluded);
Assert.Null(taken.Hero);
Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes);
Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them");
// Each waits with its capture time too, whichever side hashed what: the same project reads the same either way.
var waiting = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).PendingFrames!;
Assert.True(waiting.All(p => p.CapturedAt is not null), "a photo waits with its capture time");
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The same photos as on the server, and nothing waits any more.
var arrived = await NamedAsync(f, local.Id);
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], arrived.Excluded);
Assert.Equal(Sha(photos[5]), arrived.Hero);
Assert.SequenceEqual([Sha(photos[2])], arrived.Keyframes);
Assert.Empty(arrived.Pending);
var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(Sha(photos[5]), frames.Single(fr => fr.Id == row.HeroFrameId).Sha256);
Assert.True(SyncRules.SameSettings(row.SettingsJson, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(row.Folder).ProjectPath, f.Ct)),
"project.json mirrors the index");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
// An edit here later sends the server the same photos, by its frames.
var edited = ProjectSettings.FromJson(row.SettingsJson!) with { Exposure = new ExposureSettings { DeflickerWindow = new Auto<int>().WithManual(41) } };
await f.EditLocalSettingsAsync(local.Id, edited.ToJson());
await f.RefreshAsync();
var server = ProjectSettings.FromJson(remote.SettingsJson!);
Assert.Equal(41, server.Exposure.DeflickerWindow.Manual);
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], server.ExcludedFrames.Select(i => remote.Frames[i].Sha256));
Assert.Equal(Sha(photos[5]), remote.Frames[server.HeroFrame!.Value].Sha256);
Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256));
}
[Test]
public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies");
// Proxies arrive on the server (a client or render node sent them); the frames did not change.
remote.Status |= SequenceStatus.ProxiesBuilt;
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
}
[Test]
public async Task ACacheMadeBeforeTheServerDrewItsNewFramesStaysStaleUntilItHasTheirProxies()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Harbour", f.NewPhotos(2));
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
// Two photos reach the server. Its proxy flag stays set until its next proxy pass, which has not drawn them yet:
// the first has its thumbnail only, the second nothing at all.
var added = f.NewPhotos(2).Select(photo => remote.AddFrame(photo)).ToList();
f.Remote.Undrawn.UnionWith([(added[0].Id, "proxy"), (added[1].Id, "proxy"), (added[1].Id, "thumb")]);
await f.RefreshAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Take(3).Select(file => (int)new FileInfo(file!).Length));
Assert.Null(opened.Files[3]);
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale, "a thumbnail or a missing file stands in for a proxy the server offers");
// The pass draws them: the frames stay as they were, the server's sequence records the pass.
f.Remote.Undrawn.Clear();
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
await f.RefreshAsync();
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
}
[Test]
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Star trails", photos);
var remote = f.Remote.Add("Star trails", photos);
var render = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 5000).Select(i => (byte)i)]);
remote.Renders.Add(render);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Star trails");
Assert.Equal((SyncStatus.InSync, 1, 5000L), (entry.Status, entry.Link!.ExportsToGet, entry.Link.ExportsToGetBytes));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var fetched = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == render.Id);
Assert.Equal("4K HEVC", fetched.PresetName);
var file = f.Archive.Paths.FromRelative(fetched.OutputPath);
Assert.Equal(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 4K.mp4"), file);
Assert.SequenceEqual(render.Bytes, await File.ReadAllBytesAsync(file, f.Ct));
Assert.Equal(0, (await f.EntryAsync("Star trails")).Link!.ExportsToGet);
}
[Test]
public async Task TheServersRendersGoWhereTheSequenceIsNow()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Star trails", photos);
var remote = f.Remote.Add("Star trails", photos);
var first = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 3000).Select(i => (byte)i)]);
remote.Renders.Add(first);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var left = f.Archive.Paths.Sequence(local.Folder).Folder;
// A Reorganize here moves the folder while the server's renders are listed: the render goes where the sequence is.
string? moved = null;
f.Remote.OnceBeforeManifest = async () =>
{
var plan = await f.Archive.Service.PlanRenameAsync(f.Archive.Archive.Id, [local.Id], "moved/{title}", null, f.Ct);
await f.Archive.Service.ApplyRenameAsync(plan, null, f.Ct);
moved = plan.Steps.Single().ToFolder;
};
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal($"{moved}/exports/Star trails 4K.mp4", (await f.Db.Renders.GetAsync(first.Id, f.Ct))!.OutputPath);
Assert.FileExists(f.Archive.Paths.FromRelative($"{moved}/exports/Star trails 4K.mp4"));
Assert.False(Directory.Exists(left), "the folder it left is not created again");
// Another index moves it: nothing is filed until this index's scan finds it there.
var second = new FakeRender(Guid.CreateVersion7(), "Star trails 1080p.mp4", "1080p", [.. Enumerable.Range(0, 2000).Select(i => (byte)(i * 3))]);
remote.Renders.Add(second);
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
var (other, archive) = await f.Archive.OtherWriterAsync();
var away = await other.PlanRenameAsync(archive.Id, [local.Id], "elsewhere/{title}", null, f.Ct);
await other.ApplyRenameAsync(away, null, f.Ct);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error);
Assert.False(Directory.Exists(f.Archive.Paths.Sequence(moved!).Folder), "the folder it left is not created again");
Assert.Null(await f.Db.Renders.GetAsync(second.Id, f.Ct));
await f.Archive.ScanAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var fetched = (await f.Db.Renders.GetAsync(second.Id, f.Ct))!;
Assert.Equal("elsewhere/Star trails/exports/Star trails 1080p.mp4", fetched.OutputPath);
Assert.SequenceEqual(second.Bytes, await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(fetched.OutputPath), f.Ct));
}
[Test]
public async Task RendersMadeHereAreSentOnRequestAndAGoneFileIsReported()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var exports = f.Archive.Paths.Sequence(local.Folder).Exports;
Directory.CreateDirectory(exports);
var bytes = Enumerable.Range(0, 3000).Select(i => (byte)(i * 7)).ToArray();
await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour 1080p.mp4"), bytes, f.Ct);
var made = await AddRenderAsync(f, local, "Harbour 1080p.mp4", bytes.Length);
var gone = await AddRenderAsync(f, local, "Harbour draft.mp4", 10);
await f.RefreshAsync();
Assert.Equal((1, 3000L), ((await f.EntryAsync("Harbour")).Link!.ExportsToSend, (await f.EntryAsync("Harbour")).Link!.ExportsToSendBytes));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local);
Assert.Equal(SyncTransferKind.SendExports, SyncService.TransferKind(job));
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains("1", failed.Error);
var sent = Assert.Contains(remote.Renders, r => r.Id == made.Id);
Assert.SequenceEqual(bytes, sent.Bytes);
Assert.DoesNotContain(gone.Id, remote.Renders.Select(r => r.Id));
Assert.Equal(0, (await f.EntryAsync("Harbour")).Link!.ExportsToSend);
}
[Test]
public async Task ProxiesOpenARemoteSequenceForTheEditor()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Milan skyline", f.NewPhotos(3), SyncFixture.Window33);
await f.RefreshAsync();
Assert.Null(f.Sync.OpenRemote(f.Server, remote.Id));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Milan skyline", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Milan skyline");
Assert.Equal(SyncStatus.ProxiesOnly, entry.Status);
Assert.Equal(3, entry.Link!.ProxiesCached);
Assert.False(entry.Link.ProxiesStale);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.Equal(remote.Id, opened.Sequence.Id);
Assert.Equal(SequenceOrigin.Server, opened.Sequence.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, opened.Sequence.SettingsJson));
Assert.Equal(3, opened.CachedCount);
var source = opened.OpenFrames(DecoderFactory.CreateServer());
await using (source as IAsyncDisposable)
{
Assert.Equal(3, source.FrameCount);
using var frame = await source.GetFrameAsync(2, 0, DecodePurpose.Render, f.Ct);
Assert.Equal((48, 32), (frame.Width, frame.Height));
}
// Nothing local was created, and a frame added on the server makes the cache stale.
Assert.Null(entry.Local);
remote.AddFrame(f.NewPhotos(1)[0]);
await f.RefreshAsync();
Assert.True((await f.EntryAsync("Milan skyline")).Link!.ProxiesStale);
}
[Test]
public async Task ThumbnailsStandInWhenTheServerHasNoProxies()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.Equal(2, opened.CachedCount);
Assert.True(opened.Files.All(file => new FileInfo(file!).Length == 64), "thumbnails");
}
[Test]
public async Task TransfersFailWithAReasonWhenThisPcIsNotConnected()
{
await using var f = await SyncFixture.CreateAsync(context);
var local = await f.LocalAsync("Comet", f.NewPhotos(1));
var engine = await f.StartEngineAsync(new SyncFixture.Connector(null));
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Sync.NotConnected"), failed.Error);
Assert.Empty(f.Remote.Sequences);
}
[Test]
public async Task APhotoEachSideConvertedIsNeitherSentNorFetchedAgain()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
// Each side converted its own copy of the first photo, to other bytes (another decoder, another quality).
var (mine, theirs) = (f.NewPhotos(1)[0], f.NewPhotos(1)[0]);
await f.ReplaceLocalAsync(local.Id, 0, mine);
var remote = f.Remote.Add("Comet", [theirs, photos[1]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[0]), Sha(theirs), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
var engine = await f.StartEngineAsync();
await f.Archive.WaitForAsync((await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id, JobState.Done);
await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null)).Id,
JobState.Done);
Assert.SequenceEqual([Sha(theirs), Sha(photos[1])], remote.Frames.Select(r => r.Sha256));
Assert.SequenceEqual([Sha(mine), Sha(photos[1])], (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Select(r => r.Sha256!));
}
[Test]
public async Task CopiesConvertedOnOneSideBeforeTheyWereLinkedAreLinkedByContent()
{
await using var f = await SyncFixture.CreateAsync(context);
// The Uploader sent the card to the server, which converted it before this PC compared the two.
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
var converted = f.NewPhotos(3);
var remote = f.Remote.Add("2026-09-28 Canon", converted);
foreach (var (original, replacement) in photos.Zip(converted))
remote.Removed.Add(new RemovedPhotoDto(Sha(original), Sha(replacement), "replaced", DateTimeOffset.UtcNow));
remote.FirstCapture = local.FirstCapture;
remote.CameraModel = local.CameraModel;
// And the other way round: this PC converted its copy of another card the server holds as it came.
var cards = f.NewPhotos(2);
var mine = await f.LocalAsync("Harbour", cards);
foreach (var (index, replacement) in f.NewPhotos(2).Index()) await f.ReplaceLocalAsync(mine.Id, index, replacement);
mine = (await f.Db.Sequences.GetAsync(mine.Id, f.Ct))!;
var theirs = f.Remote.Add("2026-09-28 Canon (2)", cards);
theirs.FirstCapture = mine.FirstCapture;
theirs.CameraModel = mine.CameraModel;
await f.RefreshAsync();
var comet = await f.EntryAsync("Comet");
Assert.Equal(remote.Id, comet.Link?.RemoteId);
Assert.Equal((0, 0), (comet.Link!.MissingRemote, comet.Link.MissingLocal));
var harbour = await f.EntryAsync("Harbour");
Assert.Equal(theirs.Id, harbour.Link?.RemoteId);
Assert.Equal((0, 0), (harbour.Link!.MissingRemote, harbour.Link.MissingLocal));
}
[Test]
public async Task AFrameMissingHereTakesBackItsPhotoAsTheServerConvertedIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
// The server converted that photo.
var converted = f.NewPhotos(1)[0];
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var behind = await f.EntryAsync("Harbour");
Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The same frame, holding the server's conversion, and no frame beside it.
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256));
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the frame is no longer missing");
Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct)));
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct), "the original is an alias of its conversion here too");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task AFrameTakesBackItsPhotoInAnotherEncodingOnlyWhereTheSequenceIsNow()
{
// The folder the sequence left holds another sequence now, or nothing.
foreach (var taken in new[] { true, false })
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var stranger = taken ? await f.LocalAsync("Stranger", f.NewPhotos(2)) : null;
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
// The server converted that photo.
var converted = f.NewPhotos(1)[0];
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
// Another index moves the folder away, then the other sequence into the folder it left; this index has scanned neither.
var folder = f.Archive.Paths.Sequence(local.Folder).Folder;
var (other, archive) = await f.Archive.OtherWriterAsync();
var moves = stranger is null ? [(local.Id, "moved/{title}")] : new[] { (local.Id, "moved/{title}"), (stranger.Id, "parked/{title}"), (stranger.Id, local.Folder) };
foreach (var (id, template) in moves)
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], template, null, f.Ct), null, f.Ct);
var before = taken ? ArchiveFixture.FilesOf(folder) : null;
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error, taken.ToString());
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched");
else Assert.False(Directory.Exists(folder), "the folder it left is not created again");
Assert.Equal(lost.Sha256, (await f.Db.Frames.GetAsync(lost.Id, f.Ct))!.Sha256);
// Once this index's scan found it there, the converted photo takes the frame's place in the folder the sequence is in now.
await f.Archive.ScanAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256));
Assert.True(frames[1].RelativePath.StartsWith("moved/Harbour/", StringComparison.Ordinal), frames[1].RelativePath);
Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct)));
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "nothing missing");
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched");
else Assert.False(Directory.Exists(folder));
}
}
[Test]
public async Task AConversionMissingHereTakesBackTheOriginalTheServerHolds()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
// This PC converted the second photo, and that file is gone since.
var converted = f.NewPhotos(1)[0];
await f.ReplaceLocalAsync(local.Id, 1, converted);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
await f.RefreshAsync();
Assert.Equal(1, (await f.EntryAsync("Harbour")).Link!.MissingLocal);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(photos[1])), (frames[1].Id, frames[1].Sha256));
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(converted), f.Ct), "the conversion is an alias of the original");
Assert.False(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct));
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task AnUploadSendsTheProxyOfAPhotoThisPcConverted()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server holds the originals and cannot decode them (HEIF, BRIEF §2.3); this PC converted its copy of the first.
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.ReplaceLocalAsync(local.Id, 0, f.NewPhotos(1)[0]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Equal(3, remote.Frames.Count);
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
}
private static async Task<Render> AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes)
{
var render = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = local.Id,
PresetName = "1080p H.264",
OutputPath = local.Folder + "/exports/" + name,
Codec = "h264",
Width = 1920,
Height = 1080,
Fps = 30,
DurationSeconds = 10,
Bytes = bytes,
Created = DateTimeOffset.UtcNow,
ProducedBy = "studio",
};
await f.Db.Renders.InsertAsync(render, f.Ct);
return render;
}
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes));
/// <summary>Waits until no job is waiting or running (the thumbnails and analyses a download queues after it).</summary>
private static async Task UntilIdleAsync(SyncFixture f)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30));
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await f.Db.Jobs.ListAsync(state, cancellationToken: f.Ct)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50, f.Ct);
}
}
}
/// <summary>The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for.</summary>
private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId)
{
var project = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(localId, f.Ct))!.SettingsJson!);
var frames = await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct);
return ([.. project.ExcludedFrames.Select(i => frames[i].Sha256!)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null,
[.. project.Camera.Keyframes.Select(k => frames[k.Frame].Sha256!)], [.. (project.PendingFrames ?? []).Select(p => p.Sha256!)]);
}
}