Positions crossing copies could still land on other photos in three windows: - A push translated a row snapshot's settings against local frames listed later, after a manifest round trip: an import of earlier photos meanwhile moved the row's positions, and the server stored shifted exclusions, hero and keyframes. SyncPositions now reads this copy's row with its frames as one state (frames listed between two reads of the row at the same frames version) and translates that row's settings, at the settings version being pushed. The conflict comparison and ApplyLocallyAsync compare against that same row. When the copy was edited since, or its frames keep changing, nothing is sent or applied and the next comparison translates again (Keep mine reports a changed conflict; PushAll sends an edit saved meanwhile). - The manifest read the sequence row, then paged its frames: an upload committed in between paired settings of one frames version with frames of the next. The server now lists them again while the frames version moved during the listing, then inside a transaction holding the row (on SQLite, on that transaction's connection, so no second pooled connection waits behind the write lock). The client also refuses to translate against a manifest whose frames are not each listed once at 0..n-1. - Server capture times were fetched only when something was unhashed, so the same parked photo was written with or without capturedAt. They are now fetched whenever the two frame lists differ; an equal project then compares equal, and an entry can be placed on a photo that arrives unhashed. CompareOriginals counted unhashed server frames one-to-one against this PC's photos, so two copies of one photo in a folder the server adopted left Download on offer forever while the download fetched nothing. A photo on either side is now there when any photo of the other side matches it, as downloads and uploads find them. Paged frame listings keep each frame once (an insertion between pages lists one again). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
576 lines
31 KiB
C#
576 lines
31 KiB
C#
using Titano.Core.Codecs.Jpeg;
|
|
using Titano.Core.Imaging;
|
|
using Titano.Core.Imaging.Decoders;
|
|
using Titano.Core.Localization;
|
|
using Titano.Core.Model;
|
|
using Titano.Core.Project;
|
|
using Titano.Core.Render;
|
|
using Titano.Data.Sync;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Data.Tests.Sync;
|
|
|
|
/// <summary>Upload, download and proxy jobs on the job engine against an in-memory server.</summary>
|
|
[Category("sync")]
|
|
public sealed class SyncTransferTests(TestContext context)
|
|
{
|
|
[Test]
|
|
public async Task UploadSendsTheOriginalsAndSettingsOfALocalSequence()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Comet", photos, SyncFixture.Window41);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
Assert.Equal(job.Id, (await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id);
|
|
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal));
|
|
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
|
|
Assert.True(remote.Frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived");
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson), "the settings travel with the originals");
|
|
var entry = await f.EntryAsync("Comet");
|
|
Assert.Equal(SyncStatus.InSync, entry.Status);
|
|
Assert.Equal(remote.Id, entry.Link!.RemoteId);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUploadSendsTheProxiesTheServerCannotMakeItself()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Holy grail", photos);
|
|
// The server has the originals already (the Uploader sent them) but cannot decode them (HEIF, BRIEF §2.3).
|
|
var remote = f.Remote.Add("Holy grail", photos);
|
|
remote.Undecodable = true;
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var ahead = await f.EntryAsync("Holy grail");
|
|
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
|
|
foreach (var proxy in f.Remote.SentProxies.Values)
|
|
{
|
|
var info = JpegDecoder.ReadInfo(proxy);
|
|
Assert.True(info.Width <= 1280 && info.Height <= 1280, "made as the server's policy says");
|
|
}
|
|
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
// An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds
|
|
// under the names it gave them, and cannot decode (HEIF, BRIEF §2.3).
|
|
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
|
|
Directory.CreateDirectory(folder);
|
|
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
|
|
await f.Archive.ScanAsync();
|
|
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Titano.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
|
|
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
|
|
var remote = f.Remote.Add("Adopted", photos);
|
|
remote.Undecodable = true;
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var ahead = await f.EntryAsync(local.DisplayTitle);
|
|
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
|
|
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Holy grail", photos);
|
|
var remote = f.Remote.Add("Holy grail", photos);
|
|
remote.Undecodable = true;
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
|
|
var calls = f.Remote.ManifestCalls;
|
|
await f.RefreshAsync();
|
|
Assert.Equal(calls, f.Remote.ManifestCalls);
|
|
|
|
// A render node sends them after a job: the server's frame counters stand still, its sequence changes.
|
|
foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy;
|
|
remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1);
|
|
await f.RefreshAsync();
|
|
|
|
var entry = await f.EntryAsync("Holy grail");
|
|
Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AResumedUploadSendsOnlyWhatTheServerLacks()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(4);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
// A crashed attempt had created the server sequence and sent two originals.
|
|
var remote = f.Remote.Add("Comet", photos[..2]);
|
|
var engine = await f.StartEngineAsync();
|
|
var job = await f.Archive.EnqueueWithCheckpointAsync(engine,
|
|
new(JobKind.UploadCommit, [local.Id], "Comet", """{"server":"https://nas.test:8443"}""", JobPriority.User), remote.Id.ToString());
|
|
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Count(1, f.Remote.Sequences.Values);
|
|
Assert.Equal(4, remote.Frames.Count);
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Comet")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OriginalsChangedSinceTheyWereArchivedAreNeverUploaded()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var local = await f.LocalAsync("Comet", f.NewPhotos(3));
|
|
var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
|
|
var path = f.Archive.Paths.FromRelative(damaged.RelativePath);
|
|
var bytes = await File.ReadAllBytesAsync(path, f.Ct);
|
|
bytes[^3] ^= 0xFF;
|
|
await File.WriteAllBytesAsync(path, bytes, f.Ct);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Contains("1", failed.Error);
|
|
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
|
|
Assert.Equal(2, remote.Frames.Count);
|
|
Assert.DoesNotContain(damaged.Sha256!, remote.Frames.Select(r => r.Sha256));
|
|
Assert.Equal(SyncStatus.LocalAhead, (await f.EntryAsync("Comet")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadedOriginalsAreVerifiedAndCommittedThroughTheArchive()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var remote = f.Remote.Add("Construction cam", photos, SyncFixture.Window33);
|
|
remote.Tags = ["site"];
|
|
f.Remote.CorruptOnce = remote.Frames[1].Id;
|
|
await f.RefreshAsync();
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var entry = await f.EntryAsync("Construction cam");
|
|
Assert.Equal(SyncStatus.InSync, entry.Status);
|
|
var local = Assert.NotNull(entry.Local);
|
|
Assert.Equal(SequenceOrigin.Server, local.Origin);
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, local.SettingsJson), "the server's settings came along");
|
|
Assert.SequenceEqual(["site"], local.Tags);
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.Equal(3, frames.Count);
|
|
foreach (var frame in frames)
|
|
{
|
|
var bytes = await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frame.RelativePath), f.Ct);
|
|
Assert.Equal(frame.Sha256, Sha(bytes));
|
|
}
|
|
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "the damaged transfer was fetched again");
|
|
Assert.FileExists(f.Archive.Paths.Sequence(local.Folder).ManifestPath);
|
|
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadFillsInWhatALocalCopyLacks()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Harbour", photos[..1], SyncFixture.Window41);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
await f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.KeepMine, f.Ct);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Equal(3, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Count);
|
|
var entry = await f.EntryAsync("Harbour");
|
|
Assert.Equal(SyncStatus.InSync, entry.Status);
|
|
Assert.Equal(SequenceOrigin.Local, entry.Local!.Origin);
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local.SettingsJson), "an existing copy keeps its settings");
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadingIntoAnUnverifiedCopyHashesItFirstAndFetchesOnlyWhatItLacks()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(5);
|
|
// An adopted folder: indexed by a scan, never verified, so its frames have no hash yet.
|
|
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
|
|
Directory.CreateDirectory(folder);
|
|
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"IMG_{i + 1:0000}.JPG"), photos[i], f.Ct);
|
|
await f.Archive.ScanAsync();
|
|
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Titano.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
|
|
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
|
|
var remote = f.Remote.Add("Adopted", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
Assert.Equal(2, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingLocal);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.Equal(5, frames.Count);
|
|
Assert.True(frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "no original arrived twice");
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadingOriginalsRestoresTheOnesMissingHere()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
|
|
var place = f.Archive.Paths.FromRelative(lost.RelativePath);
|
|
File.Delete(place);
|
|
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var behind = await f.EntryAsync("Harbour");
|
|
Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(place, f.Ct)));
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.Equal(3, frames.Count);
|
|
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the restored original is no longer missing");
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OriginalsTheServerHasNotHashedAreDownloadedAndHashedHere()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
// A folder the server adopted and never verified: its manifest lists no SHA-256.
|
|
var remote = f.Remote.Add("Adopted on the server", photos);
|
|
remote.Unhashed = true;
|
|
await f.RefreshAsync();
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var entry = await f.EntryAsync("Adopted on the server");
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(entry.Local!.Id, cancellationToken: f.Ct);
|
|
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived, hashed here");
|
|
// Matched by size and capture time, the copies are the server's originals.
|
|
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
|
|
|
|
// One more photo adopted there: only that one travels.
|
|
remote.AddFrame(f.NewPhotos(1)[0]);
|
|
await f.RefreshAsync();
|
|
Assert.Equal(1, (await f.EntryAsync("Adopted on the server")).Link!.MissingLocal);
|
|
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Equal(4, (await f.Db.Frames.ListBySequenceAsync(entry.Local.Id, cancellationToken: f.Ct)).Count);
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Adopted on the server")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUploadDoesNotSendAgainWhatTheServerHoldsWithoutAHash()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(4);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
// The server adopted the first three photos and never verified them: it cannot recognise them by content.
|
|
var remote = f.Remote.Add("Harbour", photos[..3]);
|
|
remote.Unhashed = true;
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var ahead = (await f.EntryAsync("Harbour")).Link!;
|
|
Assert.Equal((1, 0), (ahead.MissingRemote, ahead.MissingLocal));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Equal((4L, 4L), (done.ProgressDone, done.ProgressTotal));
|
|
Assert.SequenceEqual(photos.Select(Sha), remote.Frames.Select(r => r.Sha256));
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProxiesOfFramesTheServerHasNotHashedAreSentToo()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Holy grail", photos);
|
|
// The server adopted the same originals, cannot decode them and has not hashed them.
|
|
var remote = f.Remote.Add("Holy grail", photos);
|
|
remote.Undecodable = true;
|
|
remote.Unhashed = true;
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
Assert.Equal(3, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
|
|
Assert.Equal((SyncStatus.InSync, 0), ((await f.EntryAsync("Holy grail")).Status, (await f.EntryAsync("Holy grail")).Link!.MissingRemote));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(6);
|
|
var local = await f.LocalAsync("Harbour", photos[1..5]);
|
|
var remote = f.Remote.Add("Harbour", photos[1..5]);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
// Another PC sends the server an earlier and a later photo; then an editor there excludes two photos, picks the
|
|
// hero and keys the camera, by the server's frames.
|
|
remote.InsertFrame(0, photos[0]);
|
|
remote.AddFrame(photos[5]);
|
|
f.Remote.EditSettings(remote.Id, new ProjectSettings
|
|
{
|
|
ExcludedFrames = [0, 3],
|
|
HeroFrame = 5,
|
|
Camera = new CameraSettings { Keyframes = [new CameraKey(2, 0.4, 0.5, 1.5, 0)] },
|
|
}.ToJson());
|
|
|
|
await f.RefreshAsync();
|
|
var taken = await NamedAsync(f, local.Id);
|
|
Assert.SequenceEqual([Sha(photos[3])], taken.Excluded);
|
|
Assert.Null(taken.Hero);
|
|
Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes);
|
|
Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them");
|
|
// Each waits with its capture time too, whichever side hashed what: the same project reads the same either way.
|
|
var waiting = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).PendingFrames!;
|
|
Assert.True(waiting.All(p => p.CapturedAt is not null), "a photo waits with its capture time");
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
// The same photos as on the server, and nothing waits any more.
|
|
var arrived = await NamedAsync(f, local.Id);
|
|
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], arrived.Excluded);
|
|
Assert.Equal(Sha(photos[5]), arrived.Hero);
|
|
Assert.SequenceEqual([Sha(photos[2])], arrived.Keyframes);
|
|
Assert.Empty(arrived.Pending);
|
|
var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.Equal(Sha(photos[5]), frames.Single(fr => fr.Id == row.HeroFrameId).Sha256);
|
|
Assert.True(SyncRules.SameSettings(row.SettingsJson, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(row.Folder).ProjectPath, f.Ct)),
|
|
"project.json mirrors the index");
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
|
|
|
|
// An edit here later sends the server the same photos, by its frames.
|
|
var edited = ProjectSettings.FromJson(row.SettingsJson!) with { Exposure = new ExposureSettings { DeflickerWindow = new Auto<int>().WithManual(41) } };
|
|
await f.EditLocalSettingsAsync(local.Id, edited.ToJson());
|
|
await f.RefreshAsync();
|
|
var server = ProjectSettings.FromJson(remote.SettingsJson!);
|
|
Assert.Equal(41, server.Exposure.DeflickerWindow.Manual);
|
|
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], server.ExcludedFrames.Select(i => remote.Frames[i].Sha256));
|
|
Assert.Equal(Sha(photos[5]), remote.Frames[server.HeroFrame!.Value].Sha256);
|
|
Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
|
|
await f.RefreshAsync();
|
|
var engine = await f.StartEngineAsync();
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
|
|
Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies");
|
|
|
|
// Proxies arrive on the server (a client or render node sent them); the frames did not change.
|
|
remote.Status |= SequenceStatus.ProxiesBuilt;
|
|
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
|
|
Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Star trails", photos);
|
|
var remote = f.Remote.Add("Star trails", photos);
|
|
var render = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 5000).Select(i => (byte)i)]);
|
|
remote.Renders.Add(render);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var entry = await f.EntryAsync("Star trails");
|
|
Assert.Equal((SyncStatus.InSync, 1, 5000L), (entry.Status, entry.Link!.ExportsToGet, entry.Link.ExportsToGetBytes));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var fetched = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == render.Id);
|
|
Assert.Equal("4K HEVC", fetched.PresetName);
|
|
var file = f.Archive.Paths.FromRelative(fetched.OutputPath);
|
|
Assert.Equal(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 4K.mp4"), file);
|
|
Assert.SequenceEqual(render.Bytes, await File.ReadAllBytesAsync(file, f.Ct));
|
|
Assert.Equal(0, (await f.EntryAsync("Star trails")).Link!.ExportsToGet);
|
|
}
|
|
|
|
[Test]
|
|
public async Task RendersMadeHereAreSentOnRequestAndAGoneFileIsReported()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
var exports = f.Archive.Paths.Sequence(local.Folder).Exports;
|
|
Directory.CreateDirectory(exports);
|
|
var bytes = Enumerable.Range(0, 3000).Select(i => (byte)(i * 7)).ToArray();
|
|
await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour 1080p.mp4"), bytes, f.Ct);
|
|
var made = await AddRenderAsync(f, local, "Harbour 1080p.mp4", bytes.Length);
|
|
var gone = await AddRenderAsync(f, local, "Harbour draft.mp4", 10);
|
|
await f.RefreshAsync();
|
|
Assert.Equal((1, 3000L), ((await f.EntryAsync("Harbour")).Link!.ExportsToSend, (await f.EntryAsync("Harbour")).Link!.ExportsToSendBytes));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local);
|
|
Assert.Equal(SyncTransferKind.SendExports, SyncService.TransferKind(job));
|
|
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Contains("1", failed.Error);
|
|
var sent = Assert.Contains(remote.Renders, r => r.Id == made.Id);
|
|
Assert.SequenceEqual(bytes, sent.Bytes);
|
|
Assert.DoesNotContain(gone.Id, remote.Renders.Select(r => r.Id));
|
|
Assert.Equal(0, (await f.EntryAsync("Harbour")).Link!.ExportsToSend);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProxiesOpenARemoteSequenceForTheEditor()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Milan skyline", f.NewPhotos(3), SyncFixture.Window33);
|
|
await f.RefreshAsync();
|
|
Assert.Null(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Milan skyline", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var entry = await f.EntryAsync("Milan skyline");
|
|
Assert.Equal(SyncStatus.ProxiesOnly, entry.Status);
|
|
Assert.Equal(3, entry.Link!.ProxiesCached);
|
|
Assert.False(entry.Link.ProxiesStale);
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.Equal(remote.Id, opened.Sequence.Id);
|
|
Assert.Equal(SequenceOrigin.Server, opened.Sequence.Origin);
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, opened.Sequence.SettingsJson));
|
|
Assert.Equal(3, opened.CachedCount);
|
|
|
|
var source = opened.OpenFrames(DecoderFactory.CreateServer());
|
|
await using (source as IAsyncDisposable)
|
|
{
|
|
Assert.Equal(3, source.FrameCount);
|
|
using var frame = await source.GetFrameAsync(2, 0, DecodePurpose.Render, f.Ct);
|
|
Assert.Equal((48, 32), (frame.Width, frame.Height));
|
|
}
|
|
|
|
// Nothing local was created, and a frame added on the server makes the cache stale.
|
|
Assert.Null(entry.Local);
|
|
remote.AddFrame(f.NewPhotos(1)[0]);
|
|
await f.RefreshAsync();
|
|
Assert.True((await f.EntryAsync("Milan skyline")).Link!.ProxiesStale);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThumbnailsStandInWhenTheServerHasNoProxies()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.Equal(2, opened.CachedCount);
|
|
Assert.True(opened.Files.All(file => new FileInfo(file!).Length == 64), "thumbnails");
|
|
}
|
|
|
|
[Test]
|
|
public async Task TransfersFailWithAReasonWhenThisPcIsNotConnected()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var local = await f.LocalAsync("Comet", f.NewPhotos(1));
|
|
var engine = await f.StartEngineAsync(new SyncFixture.Connector(null));
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Equal(Titano.Core.Localization.Loc.L("Errors.Sync.NotConnected"), failed.Error);
|
|
Assert.Empty(f.Remote.Sequences);
|
|
}
|
|
|
|
private static async Task<Render> AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes)
|
|
{
|
|
var render = new Render
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
SequenceId = local.Id,
|
|
PresetName = "1080p H.264",
|
|
OutputPath = local.Folder + "/exports/" + name,
|
|
Codec = "h264",
|
|
Width = 1920,
|
|
Height = 1080,
|
|
Fps = 30,
|
|
DurationSeconds = 10,
|
|
Bytes = bytes,
|
|
Created = DateTimeOffset.UtcNow,
|
|
ProducedBy = "studio",
|
|
};
|
|
await f.Db.Renders.InsertAsync(render, f.Ct);
|
|
return render;
|
|
}
|
|
|
|
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes));
|
|
|
|
/// <summary>The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for.</summary>
|
|
private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId)
|
|
{
|
|
var project = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(localId, f.Ct))!.SettingsJson!);
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct);
|
|
return ([.. project.ExcludedFrames.Select(i => frames[i].Sha256!)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null,
|
|
[.. project.Camera.Keyframes.Select(k => frames[k.Frame].Sha256!)], [.. (project.PendingFrames ?? []).Select(p => p.Sha256!)]);
|
|
}
|
|
}
|