A proxy download marked the cache filled whenever the server's sequence had ProxiesBuilt, even when frames added since the server's last proxy pass got a thumbnail or nothing (the flag offers their proxies, the endpoints answer 404 until the pass draws them), and frames without any cached file were not counted at all. The agent never compared such a cache again, so the editor kept thumbnails or blank frames until the frames changed. The cache now counts as filled only when every frame, missing or damaged ones aside, is cached as a proxy. On a server with ProxiesBuilt, a download that left stand-ins is queued again once the server's update time moves, so a frame the server cannot serve yet is not fetched at every refresh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
611 lines
33 KiB
C#
611 lines
33 KiB
C#
using Titano.Core.Codecs.Jpeg;
|
|
using Titano.Core.Imaging;
|
|
using Titano.Core.Imaging.Decoders;
|
|
using Titano.Core.Localization;
|
|
using Titano.Core.Model;
|
|
using Titano.Core.Project;
|
|
using Titano.Core.Render;
|
|
using Titano.Data.Sync;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Data.Tests.Sync;
|
|
|
|
/// <summary>Upload, download and proxy jobs on the job engine against an in-memory server.</summary>
|
|
[Category("sync")]
|
|
public sealed class SyncTransferTests(TestContext context)
|
|
{
|
|
[Test]
|
|
public async Task UploadSendsTheOriginalsAndSettingsOfALocalSequence()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Comet", photos, SyncFixture.Window41);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
Assert.Equal(job.Id, (await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id);
|
|
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal));
|
|
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
|
|
Assert.True(remote.Frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived");
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson), "the settings travel with the originals");
|
|
var entry = await f.EntryAsync("Comet");
|
|
Assert.Equal(SyncStatus.InSync, entry.Status);
|
|
Assert.Equal(remote.Id, entry.Link!.RemoteId);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUploadSendsTheProxiesTheServerCannotMakeItself()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Holy grail", photos);
|
|
// The server has the originals already (the Uploader sent them) but cannot decode them (HEIF, BRIEF §2.3).
|
|
var remote = f.Remote.Add("Holy grail", photos);
|
|
remote.Undecodable = true;
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var ahead = await f.EntryAsync("Holy grail");
|
|
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
|
|
foreach (var proxy in f.Remote.SentProxies.Values)
|
|
{
|
|
var info = JpegDecoder.ReadInfo(proxy);
|
|
Assert.True(info.Width <= 1280 && info.Height <= 1280, "made as the server's policy says");
|
|
}
|
|
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
// An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds
|
|
// under the names it gave them, and cannot decode (HEIF, BRIEF §2.3).
|
|
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
|
|
Directory.CreateDirectory(folder);
|
|
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
|
|
await f.Archive.ScanAsync();
|
|
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Titano.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
|
|
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
|
|
var remote = f.Remote.Add("Adopted", photos);
|
|
remote.Undecodable = true;
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var ahead = await f.EntryAsync(local.DisplayTitle);
|
|
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
|
|
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Holy grail", photos);
|
|
var remote = f.Remote.Add("Holy grail", photos);
|
|
remote.Undecodable = true;
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
|
|
var calls = f.Remote.ManifestCalls;
|
|
await f.RefreshAsync();
|
|
Assert.Equal(calls, f.Remote.ManifestCalls);
|
|
|
|
// A render node sends them after a job: the server's frame counters stand still, its sequence changes.
|
|
foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy;
|
|
remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1);
|
|
await f.RefreshAsync();
|
|
|
|
var entry = await f.EntryAsync("Holy grail");
|
|
Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AResumedUploadSendsOnlyWhatTheServerLacks()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(4);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
// A crashed attempt had created the server sequence and sent two originals.
|
|
var remote = f.Remote.Add("Comet", photos[..2]);
|
|
var engine = await f.StartEngineAsync();
|
|
var job = await f.Archive.EnqueueWithCheckpointAsync(engine,
|
|
new(JobKind.UploadCommit, [local.Id], "Comet", """{"server":"https://nas.test:8443"}""", JobPriority.User), remote.Id.ToString());
|
|
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Count(1, f.Remote.Sequences.Values);
|
|
Assert.Equal(4, remote.Frames.Count);
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Comet")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OriginalsChangedSinceTheyWereArchivedAreNeverUploaded()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var local = await f.LocalAsync("Comet", f.NewPhotos(3));
|
|
var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
|
|
var path = f.Archive.Paths.FromRelative(damaged.RelativePath);
|
|
var bytes = await File.ReadAllBytesAsync(path, f.Ct);
|
|
bytes[^3] ^= 0xFF;
|
|
await File.WriteAllBytesAsync(path, bytes, f.Ct);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Contains("1", failed.Error);
|
|
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
|
|
Assert.Equal(2, remote.Frames.Count);
|
|
Assert.DoesNotContain(damaged.Sha256!, remote.Frames.Select(r => r.Sha256));
|
|
Assert.Equal(SyncStatus.LocalAhead, (await f.EntryAsync("Comet")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadedOriginalsAreVerifiedAndCommittedThroughTheArchive()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var remote = f.Remote.Add("Construction cam", photos, SyncFixture.Window33);
|
|
remote.Tags = ["site"];
|
|
f.Remote.CorruptOnce = remote.Frames[1].Id;
|
|
await f.RefreshAsync();
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var entry = await f.EntryAsync("Construction cam");
|
|
Assert.Equal(SyncStatus.InSync, entry.Status);
|
|
var local = Assert.NotNull(entry.Local);
|
|
Assert.Equal(SequenceOrigin.Server, local.Origin);
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, local.SettingsJson), "the server's settings came along");
|
|
Assert.SequenceEqual(["site"], local.Tags);
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.Equal(3, frames.Count);
|
|
foreach (var frame in frames)
|
|
{
|
|
var bytes = await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frame.RelativePath), f.Ct);
|
|
Assert.Equal(frame.Sha256, Sha(bytes));
|
|
}
|
|
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "the damaged transfer was fetched again");
|
|
Assert.FileExists(f.Archive.Paths.Sequence(local.Folder).ManifestPath);
|
|
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadFillsInWhatALocalCopyLacks()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Harbour", photos[..1], SyncFixture.Window41);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
await f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.KeepMine, f.Ct);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Equal(3, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Count);
|
|
var entry = await f.EntryAsync("Harbour");
|
|
Assert.Equal(SyncStatus.InSync, entry.Status);
|
|
Assert.Equal(SequenceOrigin.Local, entry.Local!.Origin);
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local.SettingsJson), "an existing copy keeps its settings");
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadingIntoAnUnverifiedCopyHashesItFirstAndFetchesOnlyWhatItLacks()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(5);
|
|
// An adopted folder: indexed by a scan, never verified, so its frames have no hash yet.
|
|
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
|
|
Directory.CreateDirectory(folder);
|
|
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"IMG_{i + 1:0000}.JPG"), photos[i], f.Ct);
|
|
await f.Archive.ScanAsync();
|
|
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Titano.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
|
|
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
|
|
var remote = f.Remote.Add("Adopted", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
Assert.Equal(2, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingLocal);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.Equal(5, frames.Count);
|
|
Assert.True(frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "no original arrived twice");
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadingOriginalsRestoresTheOnesMissingHere()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
|
|
var place = f.Archive.Paths.FromRelative(lost.RelativePath);
|
|
File.Delete(place);
|
|
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var behind = await f.EntryAsync("Harbour");
|
|
Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(place, f.Ct)));
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.Equal(3, frames.Count);
|
|
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the restored original is no longer missing");
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OriginalsTheServerHasNotHashedAreDownloadedAndHashedHere()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
// A folder the server adopted and never verified: its manifest lists no SHA-256.
|
|
var remote = f.Remote.Add("Adopted on the server", photos);
|
|
remote.Unhashed = true;
|
|
await f.RefreshAsync();
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var entry = await f.EntryAsync("Adopted on the server");
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(entry.Local!.Id, cancellationToken: f.Ct);
|
|
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived, hashed here");
|
|
// Matched by size and capture time, the copies are the server's originals.
|
|
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
|
|
|
|
// One more photo adopted there: only that one travels.
|
|
remote.AddFrame(f.NewPhotos(1)[0]);
|
|
await f.RefreshAsync();
|
|
Assert.Equal(1, (await f.EntryAsync("Adopted on the server")).Link!.MissingLocal);
|
|
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Equal(4, (await f.Db.Frames.ListBySequenceAsync(entry.Local.Id, cancellationToken: f.Ct)).Count);
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Adopted on the server")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUploadDoesNotSendAgainWhatTheServerHoldsWithoutAHash()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(4);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
// The server adopted the first three photos and never verified them: it cannot recognise them by content.
|
|
var remote = f.Remote.Add("Harbour", photos[..3]);
|
|
remote.Unhashed = true;
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var ahead = (await f.EntryAsync("Harbour")).Link!;
|
|
Assert.Equal((1, 0), (ahead.MissingRemote, ahead.MissingLocal));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Equal((4L, 4L), (done.ProgressDone, done.ProgressTotal));
|
|
Assert.SequenceEqual(photos.Select(Sha), remote.Frames.Select(r => r.Sha256));
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProxiesOfFramesTheServerHasNotHashedAreSentToo()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Holy grail", photos);
|
|
// The server adopted the same originals, cannot decode them and has not hashed them.
|
|
var remote = f.Remote.Add("Holy grail", photos);
|
|
remote.Undecodable = true;
|
|
remote.Unhashed = true;
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
Assert.Equal(3, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
|
|
Assert.Equal((SyncStatus.InSync, 0), ((await f.EntryAsync("Holy grail")).Status, (await f.EntryAsync("Holy grail")).Link!.MissingRemote));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PositionsTakenFromTheServerStayOnTheirPhotosWhenTheOriginalsArrive()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(6);
|
|
var local = await f.LocalAsync("Harbour", photos[1..5]);
|
|
var remote = f.Remote.Add("Harbour", photos[1..5]);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
// Another PC sends the server an earlier and a later photo; then an editor there excludes two photos, picks the
|
|
// hero and keys the camera, by the server's frames.
|
|
remote.InsertFrame(0, photos[0]);
|
|
remote.AddFrame(photos[5]);
|
|
f.Remote.EditSettings(remote.Id, new ProjectSettings
|
|
{
|
|
ExcludedFrames = [0, 3],
|
|
HeroFrame = 5,
|
|
Camera = new CameraSettings { Keyframes = [new CameraKey(2, 0.4, 0.5, 1.5, 0)] },
|
|
}.ToJson());
|
|
|
|
await f.RefreshAsync();
|
|
var taken = await NamedAsync(f, local.Id);
|
|
Assert.SequenceEqual([Sha(photos[3])], taken.Excluded);
|
|
Assert.Null(taken.Hero);
|
|
Assert.SequenceEqual([Sha(photos[2])], taken.Keyframes);
|
|
Assert.True(taken.Pending.ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[5])]), "the photos this copy lacks wait for them");
|
|
// Each waits with its capture time too, whichever side hashed what: the same project reads the same either way.
|
|
var waiting = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson!).PendingFrames!;
|
|
Assert.True(waiting.All(p => p.CapturedAt is not null), "a photo waits with its capture time");
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
// The same photos as on the server, and nothing waits any more.
|
|
var arrived = await NamedAsync(f, local.Id);
|
|
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], arrived.Excluded);
|
|
Assert.Equal(Sha(photos[5]), arrived.Hero);
|
|
Assert.SequenceEqual([Sha(photos[2])], arrived.Keyframes);
|
|
Assert.Empty(arrived.Pending);
|
|
var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.Equal(Sha(photos[5]), frames.Single(fr => fr.Id == row.HeroFrameId).Sha256);
|
|
Assert.True(SyncRules.SameSettings(row.SettingsJson, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(row.Folder).ProjectPath, f.Ct)),
|
|
"project.json mirrors the index");
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
|
|
|
|
// An edit here later sends the server the same photos, by its frames.
|
|
var edited = ProjectSettings.FromJson(row.SettingsJson!) with { Exposure = new ExposureSettings { DeflickerWindow = new Auto<int>().WithManual(41) } };
|
|
await f.EditLocalSettingsAsync(local.Id, edited.ToJson());
|
|
await f.RefreshAsync();
|
|
var server = ProjectSettings.FromJson(remote.SettingsJson!);
|
|
Assert.Equal(41, server.Exposure.DeflickerWindow.Manual);
|
|
Assert.SequenceEqual([Sha(photos[0]), Sha(photos[3])], server.ExcludedFrames.Select(i => remote.Frames[i].Sha256));
|
|
Assert.Equal(Sha(photos[5]), remote.Frames[server.HeroFrame!.Value].Sha256);
|
|
Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
|
|
await f.RefreshAsync();
|
|
var engine = await f.StartEngineAsync();
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
|
|
Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies");
|
|
|
|
// Proxies arrive on the server (a client or render node sent them); the frames did not change.
|
|
remote.Status |= SequenceStatus.ProxiesBuilt;
|
|
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
|
|
Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ACacheMadeBeforeTheServerDrewItsNewFramesStaysStaleUntilItHasTheirProxies()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Harbour", f.NewPhotos(2));
|
|
await f.RefreshAsync();
|
|
var engine = await f.StartEngineAsync();
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
|
|
|
|
// Two photos reach the server. Its proxy flag stays set until its next proxy pass, which has not drawn them yet:
|
|
// the first has its thumbnail only, the second nothing at all.
|
|
var added = f.NewPhotos(2).Select(photo => remote.AddFrame(photo)).ToList();
|
|
f.Remote.Undrawn.UnionWith([(added[0].Id, "proxy"), (added[1].Id, "proxy"), (added[1].Id, "thumb")]);
|
|
await f.RefreshAsync();
|
|
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Take(3).Select(file => (int)new FileInfo(file!).Length));
|
|
Assert.Null(opened.Files[3]);
|
|
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale, "a thumbnail or a missing file stands in for a proxy the server offers");
|
|
|
|
// The pass draws them: the frames stay as they were, the server's sequence records the pass.
|
|
f.Remote.Undrawn.Clear();
|
|
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
|
await f.RefreshAsync();
|
|
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
|
|
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
|
|
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Star trails", photos);
|
|
var remote = f.Remote.Add("Star trails", photos);
|
|
var render = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 5000).Select(i => (byte)i)]);
|
|
remote.Renders.Add(render);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var entry = await f.EntryAsync("Star trails");
|
|
Assert.Equal((SyncStatus.InSync, 1, 5000L), (entry.Status, entry.Link!.ExportsToGet, entry.Link.ExportsToGetBytes));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var fetched = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == render.Id);
|
|
Assert.Equal("4K HEVC", fetched.PresetName);
|
|
var file = f.Archive.Paths.FromRelative(fetched.OutputPath);
|
|
Assert.Equal(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 4K.mp4"), file);
|
|
Assert.SequenceEqual(render.Bytes, await File.ReadAllBytesAsync(file, f.Ct));
|
|
Assert.Equal(0, (await f.EntryAsync("Star trails")).Link!.ExportsToGet);
|
|
}
|
|
|
|
[Test]
|
|
public async Task RendersMadeHereAreSentOnRequestAndAGoneFileIsReported()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
var exports = f.Archive.Paths.Sequence(local.Folder).Exports;
|
|
Directory.CreateDirectory(exports);
|
|
var bytes = Enumerable.Range(0, 3000).Select(i => (byte)(i * 7)).ToArray();
|
|
await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour 1080p.mp4"), bytes, f.Ct);
|
|
var made = await AddRenderAsync(f, local, "Harbour 1080p.mp4", bytes.Length);
|
|
var gone = await AddRenderAsync(f, local, "Harbour draft.mp4", 10);
|
|
await f.RefreshAsync();
|
|
Assert.Equal((1, 3000L), ((await f.EntryAsync("Harbour")).Link!.ExportsToSend, (await f.EntryAsync("Harbour")).Link!.ExportsToSendBytes));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local);
|
|
Assert.Equal(SyncTransferKind.SendExports, SyncService.TransferKind(job));
|
|
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Contains("1", failed.Error);
|
|
var sent = Assert.Contains(remote.Renders, r => r.Id == made.Id);
|
|
Assert.SequenceEqual(bytes, sent.Bytes);
|
|
Assert.DoesNotContain(gone.Id, remote.Renders.Select(r => r.Id));
|
|
Assert.Equal(0, (await f.EntryAsync("Harbour")).Link!.ExportsToSend);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProxiesOpenARemoteSequenceForTheEditor()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Milan skyline", f.NewPhotos(3), SyncFixture.Window33);
|
|
await f.RefreshAsync();
|
|
Assert.Null(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Milan skyline", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var entry = await f.EntryAsync("Milan skyline");
|
|
Assert.Equal(SyncStatus.ProxiesOnly, entry.Status);
|
|
Assert.Equal(3, entry.Link!.ProxiesCached);
|
|
Assert.False(entry.Link.ProxiesStale);
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.Equal(remote.Id, opened.Sequence.Id);
|
|
Assert.Equal(SequenceOrigin.Server, opened.Sequence.Origin);
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, opened.Sequence.SettingsJson));
|
|
Assert.Equal(3, opened.CachedCount);
|
|
|
|
var source = opened.OpenFrames(DecoderFactory.CreateServer());
|
|
await using (source as IAsyncDisposable)
|
|
{
|
|
Assert.Equal(3, source.FrameCount);
|
|
using var frame = await source.GetFrameAsync(2, 0, DecodePurpose.Render, f.Ct);
|
|
Assert.Equal((48, 32), (frame.Width, frame.Height));
|
|
}
|
|
|
|
// Nothing local was created, and a frame added on the server makes the cache stale.
|
|
Assert.Null(entry.Local);
|
|
remote.AddFrame(f.NewPhotos(1)[0]);
|
|
await f.RefreshAsync();
|
|
Assert.True((await f.EntryAsync("Milan skyline")).Link!.ProxiesStale);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThumbnailsStandInWhenTheServerHasNoProxies()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.Equal(2, opened.CachedCount);
|
|
Assert.True(opened.Files.All(file => new FileInfo(file!).Length == 64), "thumbnails");
|
|
}
|
|
|
|
[Test]
|
|
public async Task TransfersFailWithAReasonWhenThisPcIsNotConnected()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var local = await f.LocalAsync("Comet", f.NewPhotos(1));
|
|
var engine = await f.StartEngineAsync(new SyncFixture.Connector(null));
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Equal(Titano.Core.Localization.Loc.L("Errors.Sync.NotConnected"), failed.Error);
|
|
Assert.Empty(f.Remote.Sequences);
|
|
}
|
|
|
|
private static async Task<Render> AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes)
|
|
{
|
|
var render = new Render
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
SequenceId = local.Id,
|
|
PresetName = "1080p H.264",
|
|
OutputPath = local.Folder + "/exports/" + name,
|
|
Codec = "h264",
|
|
Width = 1920,
|
|
Height = 1080,
|
|
Fps = 30,
|
|
DurationSeconds = 10,
|
|
Bytes = bytes,
|
|
Created = DateTimeOffset.UtcNow,
|
|
ProducedBy = "studio",
|
|
};
|
|
await f.Db.Renders.InsertAsync(render, f.Ct);
|
|
return render;
|
|
}
|
|
|
|
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes));
|
|
|
|
/// <summary>The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for.</summary>
|
|
private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId)
|
|
{
|
|
var project = ProjectSettings.FromJson((await f.Db.Sequences.GetAsync(localId, f.Ct))!.SettingsJson!);
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct);
|
|
return ([.. project.ExcludedFrames.Select(i => frames[i].Sha256!)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null,
|
|
[.. project.Camera.Keyframes.Select(k => frames[k.Frame].Sha256!)], [.. (project.PendingFrames ?? []).Select(p => p.Sha256!)]);
|
|
}
|
|
}
|