Files
Alby96andClaude Opus 5.5 f33bcac32e
CI / Build and test (push) Successful in 13m44s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 16s
Release / Build, test and publish (push) Successful in 26m34s
CI / PostgreSQL 16 integration (push) Failing after 17m36s
CI / PostgreSQL 17 integration (push) Failing after 15m36s
CI / Docker image (push) Successful in 1m12s
Release / Server image (push) Failing after 15s
feat(copies): matches between this PC and the server, Integrate and merge (owner review 6, v1.2.0) (#74)
The same time-lapse on a server and on this PC under different ids, or twice on this PC, is recognised,
compared and settled as the owner asked in owner review 6.

Matches are found after each sync refresh, through the server's new photo lookup, and after the jobs that
change photos. The default is to ask every time. Server › Matches offers:
- Keep apart, which is remembered;
- Link only and Unlink;
- the Compare dialog: photos, details line by line, projects and the plan.

Integrate completes both sides. Photos default to Union, projects are copied across, and a losing version
goes to the trash only after a hash check. Afterwards new photos move by themselves, and deletions wait for
the user.

A merge of two local sequences verifies every photo before the other sequence goes to its archive's trash.
Upload all local skips suspected matches and flags them.

Migration 0012 adds the found_copies and copy_probes tables. Jobs 25 (Integrate) and 26 (MergeSequences) are
new, and so are the server capabilities photo-lookup and upload-into.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-11-18 15:20:29 +01:00

1106 lines
64 KiB
C#

using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Core.Render;
using Chronos.Data.Archiving;
using Chronos.Data.Copies;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Archiving;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Sync;
/// <summary>Upload, download and proxy jobs on the job engine against an in-memory server.</summary>
[Category("sync")]
public sealed class SyncTransferTests(TestContext context)
{
[Test]
public async Task UploadSendsTheOriginalsAndSettingsOfALocalSequence()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos, SyncFixture.Window41);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
Assert.Equal(job.Id, (await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal));
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
Assert.True(remote.Frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived");
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson), "the settings travel with the originals");
var entry = await f.EntryAsync("Comet");
Assert.Equal(SyncStatus.InSync, entry.Status);
Assert.Equal(remote.Id, entry.Link!.RemoteId);
}
[Test]
public async Task AnUploadSendsTheProxiesTheServerCannotMakeItself()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server has the originals already (the Uploader sent them) but cannot decode them (HEIF, BRIEF §2.3).
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
foreach (var proxy in f.Remote.SentProxies.Values)
{
var info = JpegDecoder.ReadInfo(proxy);
Assert.True(info.Width <= 1280 && info.Height <= 1280, "made as the server's policy says");
}
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status);
}
[Test]
public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds
// under the names it gave them, and cannot decode (HEIF, BRIEF §2.3).
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
var remote = f.Remote.Add("Adopted", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync(local.DisplayTitle);
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
}
[Test]
public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Holy grail", photos);
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
var calls = f.Remote.ManifestCalls;
await f.RefreshAsync();
Assert.Equal(calls, f.Remote.ManifestCalls);
// A render node sends them after a job: the server's frame counters stand still, its sequence changes.
foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy;
remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1);
await f.RefreshAsync();
var entry = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote));
}
[Test]
public async Task PhotosOneSideDeletedAreNeitherSentNorFetchedAgain()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
// The server holds the first three and deleted the fourth, which this PC still holds.
var remote = f.Remote.Add("Comet", photos[..3]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(0, (await f.EntryAsync("Comet")).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var upload = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(upload.Id, JobState.Done);
Assert.Equal(3, remote.Frames.Count);
Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
// This PC deletes the first photo, which the server still holds: it is not fetched back, and nothing local is deleted.
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [frames[0].Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal));
var download = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(download.Id, JobState.Done);
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.SequenceEqual(photos[1..].Select(Sha), kept.Select(k => k.Sha256!));
}
[Test]
public async Task APhotoDeletedHereIsNotFetchedAgainFromAServerThatHasNotHashedIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// A folder the server adopted and never verified (its manifest lists no SHA-256), downloaded here.
var remote = f.Remote.Add("Adopted on the server", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(first.Id, JobState.Done);
var local = Assert.NotNull((await f.EntryAsync("Adopted on the server")).Local);
// This PC deletes a photo the server still lists, unhashed: only its content tells it is the one deleted here.
await UntilIdleAsync(f);
var gone = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(photos[0]));
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [gone.Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct);
await f.RefreshAsync();
var again = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
var done = await f.Archive.WaitForAsync(again.Id, JobState.Done);
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted");
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone");
Assert.Contains(Loc.L("Archive.Log.SyncDeletedHere", remote.Frames.Single(fr => fr.Sha256 == Sha(photos[0])).FileName), done.LogTail);
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
}
[Test]
public async Task AnUnverifiedCopyDoesNotSendBackAPhotoTheServerDeleted()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
// An adopted folder (never verified, so no hashes here) that still holds a photo the server has deleted since.
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 4; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
var remote = f.Remote.Add("Adopted", photos[..3]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
// Without its hash the photo cannot be told from one the server lacks: the upload finds out.
Assert.Equal(1, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(3, remote.Frames.Count);
Assert.Contains(Loc.L("Archive.Log.SyncDeletedThere", "DSC_0004.JPG"), done.LogTail);
// Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good.
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify");
var entry = await f.EntryAsync(local.DisplayTitle);
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
var calls = remote.Frames.Count;
var next = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var settled = await f.Archive.WaitForAsync(next.Id, JobState.Done);
Assert.Equal((4L, 4L, calls), (settled.ProgressDone, settled.ProgressTotal, remote.Frames.Count));
}
[Test]
public async Task ATombstoneBesideAPhotoTheServerStillHoldsLeavesTheCopiesInSync()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
// The server holds both photos and a tombstone of the second: it deleted one of two copies of it, or the file came
// back by hand. Neither side lacks anything.
var remote = f.Remote.Add("Comet", photos);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal));
Assert.Equal(SyncStatus.InSync, entry.Status);
}
[Test]
public async Task AResumedUploadSendsOnlyWhatTheServerLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
// A crashed attempt had created the server sequence and sent two originals.
var remote = f.Remote.Add("Comet", photos[..2]);
var engine = await f.StartEngineAsync();
var job = await f.Archive.EnqueueWithCheckpointAsync(engine,
new(JobKind.UploadCommit, [local.Id], "Comet", """{"server":"https://nas.test:8443"}""", JobPriority.User), remote.Id.ToString());
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Count(1, f.Remote.Sequences.Values);
Assert.Equal(4, remote.Frames.Count);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Comet")).Status);
}
[Test]
public async Task OriginalsChangedSinceTheyWereArchivedAreNeverUploaded()
{
await using var f = await SyncFixture.CreateAsync(context);
var local = await f.LocalAsync("Comet", f.NewPhotos(3));
var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
var path = f.Archive.Paths.FromRelative(damaged.RelativePath);
var bytes = await File.ReadAllBytesAsync(path, f.Ct);
bytes[^3] ^= 0xFF;
await File.WriteAllBytesAsync(path, bytes, f.Ct);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains("1", failed.Error);
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
Assert.Equal(2, remote.Frames.Count);
Assert.DoesNotContain(damaged.Sha256!, remote.Frames.Select(r => r.Sha256));
Assert.Equal(SyncStatus.LocalAhead, (await f.EntryAsync("Comet")).Status);
}
[Test]
public async Task DownloadedOriginalsAreVerifiedAndCommittedThroughTheArchive()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var remote = f.Remote.Add("Construction cam", photos, SyncFixture.Window33);
remote.Tags = ["site"];
f.Remote.CorruptOnce = remote.Frames[1].Id;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Construction cam");
Assert.Equal(SyncStatus.InSync, entry.Status);
var local = Assert.NotNull(entry.Local);
Assert.Equal(SequenceOrigin.Server, local.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, local.SettingsJson), "the server's settings came along");
Assert.SequenceEqual(["site"], local.Tags);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
foreach (var frame in frames)
{
var bytes = await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frame.RelativePath), f.Ct);
Assert.Equal(frame.Sha256, Sha(bytes));
}
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "the damaged transfer was fetched again");
Assert.FileExists(f.Archive.Paths.Sequence(local.Folder).ManifestPath);
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
}
[Test]
public async Task DownloadFillsInWhatALocalCopyLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos[..1], SyncFixture.Window41);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
await f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.KeepMine, f.Ct);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(3, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Count);
var entry = await f.EntryAsync("Harbour");
Assert.Equal(SyncStatus.InSync, entry.Status);
Assert.Equal(SequenceOrigin.Local, entry.Local!.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local.SettingsJson), "an existing copy keeps its settings");
}
[Test]
public async Task DownloadingIntoAnUnverifiedCopyHashesItFirstAndFetchesOnlyWhatItLacks()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(5);
// An adopted folder: indexed by a scan, never verified, so its frames have no hash yet.
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"IMG_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
var remote = f.Remote.Add("Adopted", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(2, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingLocal);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(5, frames.Count);
Assert.True(frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "no original arrived twice");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
}
[Test]
public async Task DownloadingOriginalsRestoresTheOnesMissingHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
var place = f.Archive.Paths.FromRelative(lost.RelativePath);
File.Delete(place);
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var behind = await f.EntryAsync("Harbour");
Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(place, f.Ct)));
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the restored original is no longer missing");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task AnOriginalIsRestoredOnlyWhereTheSequenceIsNow()
{
// The folder the sequence left holds another sequence now, or nothing.
foreach (var taken in new[] { true, false })
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var stranger = taken ? await f.LocalAsync("Stranger", f.NewPhotos(2)) : null;
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
// Another index moves the folder away, then the other sequence (first parked elsewhere: its folder is the same
// name with a number) into the folder it left; this index has scanned neither.
var folder = f.Archive.Paths.Sequence(local.Folder).Folder;
var (other, archive) = await f.Archive.OtherWriterAsync();
var moves = stranger is null ? [(local.Id, "moved/{title}")] : new[] { (local.Id, "moved/{title}"), (stranger.Id, "parked/{title}"), (stranger.Id, local.Folder) };
foreach (var (id, template) in moves)
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], template, null, f.Ct), null, f.Ct);
var before = taken ? ArchiveFixture.FilesOf(folder) : null;
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error, taken.ToString());
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched");
else Assert.False(Directory.Exists(folder), "the folder it left is not created again");
// Once this index's scan found it there, the original goes back into the folder the sequence is in now.
await f.Archive.ScanAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var moved = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Folder;
Assert.Equal("moved/Harbour", moved);
var restored = (await f.Db.Frames.GetAsync(lost.Id, f.Ct))!;
Assert.Equal(moved + lost.RelativePath[local.Folder.Length..], restored.RelativePath);
Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(restored.RelativePath), f.Ct)));
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => (fr.Flags & FrameFlags.Missing) == 0), "nothing missing");
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched");
else Assert.False(Directory.Exists(folder));
}
}
[Test]
public async Task OriginalsTheServerHasNotHashedAreDownloadedAndHashedHere()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// A folder the server adopted and never verified: its manifest lists no SHA-256.
var remote = f.Remote.Add("Adopted on the server", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Adopted on the server");
var frames = await f.Db.Frames.ListBySequenceAsync(entry.Local!.Id, cancellationToken: f.Ct);
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived, hashed here");
// Matched by size and capture time, the copies are the server's originals.
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
// One more photo adopted there: only that one travels.
remote.AddFrame(f.NewPhotos(1)[0]);
await f.RefreshAsync();
Assert.Equal(1, (await f.EntryAsync("Adopted on the server")).Link!.MissingLocal);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(4, (await f.Db.Frames.ListBySequenceAsync(entry.Local.Id, cancellationToken: f.Ct)).Count);
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Adopted on the server")).Status);
}
[Test]
public async Task AnUploadDoesNotSendAgainWhatTheServerHoldsWithoutAHash()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Harbour", photos);
// The server adopted the first three photos and never verified them: it cannot recognise them by content.
var remote = f.Remote.Add("Harbour", photos[..3]);
remote.Unhashed = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = (await f.EntryAsync("Harbour")).Link!;
Assert.Equal((1, 0), (ahead.MissingRemote, ahead.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal((4L, 4L), (done.ProgressDone, done.ProgressTotal));
Assert.SequenceEqual(photos.Select(Sha), remote.Frames.Select(r => r.Sha256));
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task ProxiesOfFramesTheServerHasNotHashedAreSentToo()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server adopted the same originals, cannot decode them and has not hashed them.
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
remote.Unhashed = true;
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
Assert.Equal(3, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Equal((SyncStatus.InSync, 0), ((await f.EntryAsync("Holy grail")).Status, (await f.EntryAsync("Holy grail")).Link!.MissingRemote));
}
[Test]
public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(6);
var local = await f.LocalAsync("Harbour", photos[1..5]);
var remote = f.Remote.Add("Harbour", photos[1..5]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
// Another PC sends the server an earlier and a later photo; then an editor there excludes two photos, picks the
// hero and keys the camera, by the server's frames.
remote.InsertFrame(0, photos[0]);
remote.AddFrame(photos[5]);
f.Remote.EditSettings(remote.Id, new ProjectSettings
{
ExcludedFrames = [0, 3],
HeroFrame = 5,
Camera = new CameraSettings { Keyframes = [new CameraKey(2, 0.4, 0.5, 1.5, 0)] },
}.ToJson());
await f.RefreshAsync();
var taken = await NamedAsync(f, local.Id);
Assert.SequenceEqual([Sha(photos[3])], taken.Excluded);
Assert.Null(taken.Hero);
Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes);
Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them");
// Each waits with its capture time too, whichever side hashed what: the same project reads the same either way.
var waiting = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).PendingFrames!;
Assert.True(waiting.All(p => p.CapturedAt is not null), "a photo waits with its capture time");
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The same photos as on the server, and nothing waits any more.
var arrived = await NamedAsync(f, local.Id);
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], arrived.Excluded);
Assert.Equal(Sha(photos[5]), arrived.Hero);
Assert.SequenceEqual([Sha(photos[2])], arrived.Keyframes);
Assert.Empty(arrived.Pending);
var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(Sha(photos[5]), frames.Single(fr => fr.Id == row.HeroFrameId).Sha256);
Assert.True(SyncRules.SameSettings(row.SettingsJson, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(row.Folder).ProjectPath, f.Ct)),
"project.json mirrors the index");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
// An edit here later sends the server the same photos, by its frames.
var edited = ProjectSettings.FromJson(row.SettingsJson!) with { Exposure = new ExposureSettings { DeflickerWindow = new Auto<int>().WithManual(41) } };
await f.EditLocalSettingsAsync(local.Id, edited.ToJson());
await f.RefreshAsync();
var server = ProjectSettings.FromJson(remote.SettingsJson!);
Assert.Equal(41, server.Exposure.DeflickerWindow.Manual);
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], server.ExcludedFrames.Select(i => remote.Frames[i].Sha256));
Assert.Equal(Sha(photos[5]), remote.Frames[server.HeroFrame!.Value].Sha256);
Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256));
}
[Test]
public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies");
// Proxies arrive on the server (a client or render node sent them); the frames did not change.
remote.Status |= SequenceStatus.ProxiesBuilt;
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
}
[Test]
public async Task ACacheMadeBeforeTheServerDrewItsNewFramesStaysStaleUntilItHasTheirProxies()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Harbour", f.NewPhotos(2));
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
// Two photos reach the server. Its proxy flag stays set until its next proxy pass, which has not drawn them yet:
// the first has its thumbnail only, the second nothing at all.
var added = f.NewPhotos(2).Select(photo => remote.AddFrame(photo)).ToList();
f.Remote.Undrawn.UnionWith([(added[0].Id, "proxy"), (added[1].Id, "proxy"), (added[1].Id, "thumb")]);
await f.RefreshAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Take(3).Select(file => (int)new FileInfo(file!).Length));
Assert.Null(opened.Files[3]);
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale, "a thumbnail or a missing file stands in for a proxy the server offers");
// The pass draws them: the frames stay as they were, the server's sequence records the pass.
f.Remote.Undrawn.Clear();
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
await f.RefreshAsync();
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
}
[Test]
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Star trails", photos);
var remote = f.Remote.Add("Star trails", photos);
var render = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 5000).Select(i => (byte)i)]);
remote.Renders.Add(render);
var still = new FakeRender(Guid.CreateVersion7(), "Star trails - 0002.png", "PNG", [.. Enumerable.Range(0, 700).Select(i => (byte)(i * 5))], RenderKind.Still,
Recorded: string.Empty);
remote.Renders.Add(still);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Star trails");
Assert.Equal((SyncStatus.InSync, 2, 5700L), (entry.Status, entry.Link!.ExportsToGet, entry.Link.ExportsToGetBytes));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var fetched = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == render.Id);
Assert.Equal("4K HEVC", fetched.PresetName);
var file = f.Archive.Paths.FromRelative(fetched.OutputPath);
Assert.Equal(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 4K.mp4"), file);
Assert.SequenceEqual(render.Bytes, await File.ReadAllBytesAsync(file, f.Ct));
Assert.Equal((render.Sha256, (string?)null), (fetched.Sha256, fetched.Kind), "the server's hash, checked on arrival");
// A still the server has not hashed: its kind, and the hash of what arrived.
var saved = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == still.Id);
Assert.Equal((Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(still.Bytes)), RenderKind.Still), (saved.Sha256, saved.Kind));
Assert.Equal(0, (await f.EntryAsync("Star trails")).Link!.ExportsToGet);
// A file unlike the hash the server records is fetched again, then refused: nothing is filed.
var damaged = new FakeRender(Guid.CreateVersion7(), "Star trails 1080p.mp4", "1080p", [1, 2, 3, 4], Recorded: new string('a', 64));
remote.Renders.Add(damaged);
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Sync.Hash", "Star trails 1080p.mp4"), failed.Error);
Assert.Null(await f.Db.Renders.GetAsync(damaged.Id, f.Ct));
Assert.False(File.Exists(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 1080p.mp4")));
}
[Test]
public async Task ASentRenderKeepsItsKind()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var exports = f.Archive.Paths.Sequence(local.Folder).Exports;
Directory.CreateDirectory(exports);
byte[] bytes = [.. Enumerable.Range(0, 900).Select(i => (byte)(i * 11))];
await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour - 0001.png"), bytes, f.Ct);
var still = await AddRenderAsync(f, local, "Harbour - 0001.png", bytes.Length, RenderKind.Still);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var sent = Assert.Contains(remote.Renders, r => r.Id == still.Id);
Assert.Equal(RenderKind.Still, sent.Kind);
}
[Test]
public async Task TheServersRendersGoWhereTheSequenceIsNow()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Star trails", photos);
var remote = f.Remote.Add("Star trails", photos);
var first = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 3000).Select(i => (byte)i)]);
remote.Renders.Add(first);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var left = f.Archive.Paths.Sequence(local.Folder).Folder;
// A Reorganize here moves the folder while the server's renders are listed: the render goes where the sequence is.
string? moved = null;
f.Remote.OnceBeforeManifest = async () =>
{
var plan = await f.Archive.Service.PlanRenameAsync(f.Archive.Archive.Id, [local.Id], "moved/{title}", null, f.Ct);
await f.Archive.Service.ApplyRenameAsync(plan, null, f.Ct);
moved = plan.Steps.Single().ToFolder;
};
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal($"{moved}/exports/Star trails 4K.mp4", (await f.Db.Renders.GetAsync(first.Id, f.Ct))!.OutputPath);
Assert.FileExists(f.Archive.Paths.FromRelative($"{moved}/exports/Star trails 4K.mp4"));
Assert.False(Directory.Exists(left), "the folder it left is not created again");
// Another index moves it: nothing is filed until this index's scan finds it there.
var second = new FakeRender(Guid.CreateVersion7(), "Star trails 1080p.mp4", "1080p", [.. Enumerable.Range(0, 2000).Select(i => (byte)(i * 3))]);
remote.Renders.Add(second);
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
var (other, archive) = await f.Archive.OtherWriterAsync();
var away = await other.PlanRenameAsync(archive.Id, [local.Id], "elsewhere/{title}", null, f.Ct);
await other.ApplyRenameAsync(away, null, f.Ct);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error);
Assert.False(Directory.Exists(f.Archive.Paths.Sequence(moved!).Folder), "the folder it left is not created again");
Assert.Null(await f.Db.Renders.GetAsync(second.Id, f.Ct));
await f.Archive.ScanAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var fetched = (await f.Db.Renders.GetAsync(second.Id, f.Ct))!;
Assert.Equal("elsewhere/Star trails/exports/Star trails 1080p.mp4", fetched.OutputPath);
Assert.SequenceEqual(second.Bytes, await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(fetched.OutputPath), f.Ct));
}
[Test]
public async Task RendersMadeHereAreSentOnRequestAndAGoneFileIsReported()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var exports = f.Archive.Paths.Sequence(local.Folder).Exports;
Directory.CreateDirectory(exports);
var bytes = Enumerable.Range(0, 3000).Select(i => (byte)(i * 7)).ToArray();
await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour 1080p.mp4"), bytes, f.Ct);
var made = await AddRenderAsync(f, local, "Harbour 1080p.mp4", bytes.Length);
var gone = await AddRenderAsync(f, local, "Harbour draft.mp4", 10);
await f.RefreshAsync();
Assert.Equal((1, 3000L), ((await f.EntryAsync("Harbour")).Link!.ExportsToSend, (await f.EntryAsync("Harbour")).Link!.ExportsToSendBytes));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local);
Assert.Equal(SyncTransferKind.SendExports, SyncService.TransferKind(job));
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains("1", failed.Error);
var sent = Assert.Contains(remote.Renders, r => r.Id == made.Id);
Assert.SequenceEqual(bytes, sent.Bytes);
Assert.DoesNotContain(gone.Id, remote.Renders.Select(r => r.Id));
Assert.Equal(0, (await f.EntryAsync("Harbour")).Link!.ExportsToSend);
}
[Test]
public async Task ProxiesOpenARemoteSequenceForTheEditor()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Milan skyline", f.NewPhotos(3), SyncFixture.Window33);
await f.RefreshAsync();
Assert.Null(f.Sync.OpenRemote(f.Server, remote.Id));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Milan skyline", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var entry = await f.EntryAsync("Milan skyline");
Assert.Equal(SyncStatus.ProxiesOnly, entry.Status);
Assert.Equal(3, entry.Link!.ProxiesCached);
Assert.False(entry.Link.ProxiesStale);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.Equal(remote.Id, opened.Sequence.Id);
Assert.Equal(SequenceOrigin.Server, opened.Sequence.Origin);
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, opened.Sequence.SettingsJson));
Assert.Equal(3, opened.CachedCount);
var source = opened.OpenFrames(DecoderFactory.CreateServer());
await using (source as IAsyncDisposable)
{
Assert.Equal(3, source.FrameCount);
using var frame = await source.GetFrameAsync(2, 0, DecodePurpose.Render, f.Ct);
Assert.Equal((48, 32), (frame.Width, frame.Height));
}
// Nothing local was created, and a frame added on the server makes the cache stale.
Assert.Null(entry.Local);
remote.AddFrame(f.NewPhotos(1)[0]);
await f.RefreshAsync();
Assert.True((await f.EntryAsync("Milan skyline")).Link!.ProxiesStale);
}
[Test]
public async Task ThumbnailsStandInWhenTheServerHasNoProxies()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.Equal(2, opened.CachedCount);
Assert.True(opened.Files.All(file => new FileInfo(file!).Length == 64), "thumbnails");
}
[Test]
public async Task TransfersFailWithAReasonWhenThisPcIsNotConnected()
{
await using var f = await SyncFixture.CreateAsync(context);
var local = await f.LocalAsync("Comet", f.NewPhotos(1));
var engine = await f.StartEngineAsync(new SyncFixture.Connector(null));
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Sync.NotConnected"), failed.Error);
Assert.Empty(f.Remote.Sequences);
}
[Test]
public async Task APhotoEachSideConvertedIsNeitherSentNorFetchedAgain()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
// Each side converted its own copy of the first photo, to other bytes (another decoder, another quality).
var (mine, theirs) = (f.NewPhotos(1)[0], f.NewPhotos(1)[0]);
await f.ReplaceLocalAsync(local.Id, 0, mine);
var remote = f.Remote.Add("Comet", [theirs, photos[1]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[0]), Sha(theirs), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
var engine = await f.StartEngineAsync();
await f.Archive.WaitForAsync((await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id, JobState.Done);
await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null)).Id,
JobState.Done);
Assert.SequenceEqual([Sha(theirs), Sha(photos[1])], remote.Frames.Select(r => r.Sha256));
Assert.SequenceEqual([Sha(mine), Sha(photos[1])], (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Select(r => r.Sha256!));
}
[Test]
public async Task CopiesConvertedOnOneSideBeforeTheyWereLinkedAreLinkedByContent()
{
await using var f = await SyncFixture.CreateAsync(context);
// The Uploader sent the card to the server, which converted it before this PC compared the two.
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
var converted = f.NewPhotos(3);
var remote = f.Remote.Add("2026-09-28 Canon", converted);
foreach (var (original, replacement) in photos.Zip(converted))
remote.Removed.Add(new RemovedPhotoDto(Sha(original), Sha(replacement), "replaced", DateTimeOffset.UtcNow));
remote.FirstCapture = local.FirstCapture;
remote.CameraModel = local.CameraModel;
// And the other way round: this PC converted its copy of another card the server holds as it came.
var cards = f.NewPhotos(2);
var mine = await f.LocalAsync("Harbour", cards);
foreach (var (index, replacement) in f.NewPhotos(2).Index()) await f.ReplaceLocalAsync(mine.Id, index, replacement);
mine = (await f.Db.Sequences.GetAsync(mine.Id, f.Ct))!;
var theirs = f.Remote.Add("2026-09-28 Canon (2)", cards);
theirs.FirstCapture = mine.FirstCapture;
theirs.CameraModel = mine.CameraModel;
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared });
var comet = await f.EntryAsync("Comet");
Assert.Equal(remote.Id, comet.Link?.RemoteId);
Assert.Equal((0, 0), (comet.Link!.MissingRemote, comet.Link.MissingLocal));
var harbour = await f.EntryAsync("Harbour");
Assert.Equal(theirs.Id, harbour.Link?.RemoteId);
Assert.Equal((0, 0), (harbour.Link!.MissingRemote, harbour.Link.MissingLocal));
}
[Test]
public async Task AFrameMissingHereTakesBackItsPhotoAsTheServerConvertedIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
// The server converted that photo.
var converted = f.NewPhotos(1)[0];
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var behind = await f.EntryAsync("Harbour");
Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The same frame, holding the server's conversion, and no frame beside it.
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256));
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the frame is no longer missing");
Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct)));
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct), "the original is an alias of its conversion here too");
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task AFrameThatTookBackItsPhotoInAnotherEncodingIsDrawnAgain()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var engine = await f.StartEngineAsync();
var drawn = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([local.Id], "Harbour"), cancellationToken: f.Ct);
await f.Archive.WaitForAsync(drawn.Id, JobState.Done);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn before");
// The server converted that photo.
var converted = f.NewPhotos(1)[0];
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// Nothing was filed, only a frame swapped, which cleared what covered the sequence: the thumbnails it had are drawn again.
Assert.Equal(lost.Id, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1].Id);
var again = Assert.Contains((await f.Db.Jobs.ListAsync(null, JobKind.Thumbs, 50, null, f.Ct)).Items, j => j.Id != drawn.Id && j.TargetIds.Contains(local.Id));
await f.Archive.WaitForAsync(again.Id, JobState.Done);
Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn again");
}
[Test]
public async Task AFrameTakesBackItsPhotoInAnotherEncodingOnlyWhereTheSequenceIsNow()
{
// The folder the sequence left holds another sequence now, or nothing.
foreach (var taken in new[] { true, false })
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var stranger = taken ? await f.LocalAsync("Stranger", f.NewPhotos(2)) : null;
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
// The server converted that photo.
var converted = f.NewPhotos(1)[0];
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
// Another index moves the folder away, then the other sequence into the folder it left; this index has scanned neither.
var folder = f.Archive.Paths.Sequence(local.Folder).Folder;
var (other, archive) = await f.Archive.OtherWriterAsync();
var moves = stranger is null ? [(local.Id, "moved/{title}")] : new[] { (local.Id, "moved/{title}"), (stranger.Id, "parked/{title}"), (stranger.Id, local.Folder) };
foreach (var (id, template) in moves)
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], template, null, f.Ct), null, f.Ct);
var before = taken ? ArchiveFixture.FilesOf(folder) : null;
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error, taken.ToString());
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched");
else Assert.False(Directory.Exists(folder), "the folder it left is not created again");
Assert.Equal(lost.Sha256, (await f.Db.Frames.GetAsync(lost.Id, f.Ct))!.Sha256);
// Once this index's scan found it there, the converted photo takes the frame's place in the folder the sequence is in now.
await f.Archive.ScanAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256));
Assert.True(frames[1].RelativePath.StartsWith("moved/Harbour/", StringComparison.Ordinal), frames[1].RelativePath);
Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct)));
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "nothing missing");
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched");
else Assert.False(Directory.Exists(folder));
}
}
[Test]
public async Task AConversionMissingHereTakesBackTheOriginalTheServerHolds()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
// This PC converted the second photo, and that file is gone since.
var converted = f.NewPhotos(1)[0];
await f.ReplaceLocalAsync(local.Id, 1, converted);
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
await f.RefreshAsync();
Assert.Equal(1, (await f.EntryAsync("Harbour")).Link!.MissingLocal);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.Equal(3, frames.Count);
Assert.Equal((lost.Id, Sha(photos[1])), (frames[1].Id, frames[1].Sha256));
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(converted), f.Ct), "the conversion is an alias of the original");
Assert.False(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct));
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
}
[Test]
public async Task AnUploadSendsTheProxyOfAPhotoThisPcConverted()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Holy grail", photos);
// The server holds the originals and cannot decode them (HEIF, BRIEF §2.3); this PC converted its copy of the first.
var remote = f.Remote.Add("Holy grail", photos);
remote.Undecodable = true;
await f.ReplaceLocalAsync(local.Id, 0, f.NewPhotos(1)[0]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var ahead = await f.EntryAsync("Holy grail");
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
Assert.Equal(3, remote.Frames.Count);
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
}
private static async Task<Render> AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes, string? kind = null)
{
var render = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = local.Id,
PresetName = "1080p H.264",
OutputPath = local.Folder + "/exports/" + name,
Codec = "h264",
Width = 1920,
Height = 1080,
Fps = 30,
DurationSeconds = 10,
Bytes = bytes,
Created = DateTimeOffset.UtcNow,
ProducedBy = "studio",
Kind = kind,
};
await f.Db.Renders.InsertAsync(render, f.Ct);
return render;
}
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes));
/// <summary>Waits until no job is waiting or running (the thumbnails and analyses a download queues after it).</summary>
private static async Task UntilIdleAsync(SyncFixture f)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30));
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await f.Db.Jobs.ListAsync(state, cancellationToken: f.Ct)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50, f.Ct);
}
}
}
/// <summary>The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for.</summary>
private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId)
{
var project = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(localId, f.Ct))!.SettingsJson!);
var frames = await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct);
return ([.. project.ExcludedFrames.Select(i => frames[i].Sha256!)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null,
[.. project.Camera.Keyframes.Select(k => frames[k.Frame].Sha256!)], [.. (project.PendingFrames ?? []).Select(p => p.Sha256!)]);
}
}