- Conflicts are settled against the versions the user was shown; a later change on
either side refuses the choice and compares again. Title, tags, rating and notes
changed differently on both sides are a conflict too (no silent tag union).
- Links are written under the engine's gate and forgotten only after a 404; linked
local copies are read by id; unlinked copies with the same originals are linked
by content.
- A failure on one sequence is recorded on its link; the others are still compared.
- Downloads hash unverified local copies first and restore originals missing here.
- Background proxy downloads give way to the user's requests; "Keep proxies offline"
refreshes the caches of sequences opened with proxies, checks free space and backs
off after a failure.
- Renders travel on request: Send exports (chunked, hash-verified upload keeping the
render id; new POST /sequences/{id}/renders endpoints) and Get exports.
- Server screen: stable row buttons, mockup column grid, Tab through rows, conflict
rows first on the card colour, localised conflict detail, Save both confirmation,
Pause metered uploads, culture switch, Library one-word sync glyphs.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
328 lines
16 KiB
C#
328 lines
16 KiB
C#
using Titano.Core.Imaging;
|
|
using Titano.Core.Imaging.Decoders;
|
|
using Titano.Core.Model;
|
|
using Titano.Core.Render;
|
|
using Titano.Data.Sync;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Data.Tests.Sync;
|
|
|
|
/// <summary>Upload, download and proxy jobs on the job engine against an in-memory server.</summary>
|
|
[Category("sync")]
|
|
public sealed class SyncTransferTests(TestContext context)
|
|
{
|
|
[Test]
|
|
public async Task UploadSendsTheOriginalsAndSettingsOfALocalSequence()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Comet", photos, SyncFixture.Window41);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
Assert.Equal(job.Id, (await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id);
|
|
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Equal((3L, 3L), (done.ProgressDone, done.ProgressTotal));
|
|
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
|
|
Assert.True(remote.Frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "every original arrived");
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson), "the settings travel with the originals");
|
|
var entry = await f.EntryAsync("Comet");
|
|
Assert.Equal(SyncStatus.InSync, entry.Status);
|
|
Assert.Equal(remote.Id, entry.Link!.RemoteId);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AResumedUploadSendsOnlyWhatTheServerLacks()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(4);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
// A crashed attempt had created the server sequence and sent two originals.
|
|
var remote = f.Remote.Add("Comet", photos[..2]);
|
|
var engine = await f.StartEngineAsync();
|
|
var job = await f.Archive.EnqueueWithCheckpointAsync(engine,
|
|
new(JobKind.UploadCommit, [local.Id], "Comet", """{"server":"https://nas.test:8443"}""", JobPriority.User), remote.Id.ToString());
|
|
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Count(1, f.Remote.Sequences.Values);
|
|
Assert.Equal(4, remote.Frames.Count);
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Comet")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OriginalsChangedSinceTheyWereArchivedAreNeverUploaded()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var local = await f.LocalAsync("Comet", f.NewPhotos(3));
|
|
var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
|
|
var path = f.Archive.Paths.FromRelative(damaged.RelativePath);
|
|
var bytes = await File.ReadAllBytesAsync(path, f.Ct);
|
|
bytes[^3] ^= 0xFF;
|
|
await File.WriteAllBytesAsync(path, bytes, f.Ct);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Contains("1", failed.Error);
|
|
var remote = Assert.Contains(f.Remote.Sequences.Values, s => s.Title == "Comet");
|
|
Assert.Equal(2, remote.Frames.Count);
|
|
Assert.DoesNotContain(damaged.Sha256!, remote.Frames.Select(r => r.Sha256));
|
|
Assert.Equal(SyncStatus.LocalAhead, (await f.EntryAsync("Comet")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadedOriginalsAreVerifiedAndCommittedThroughTheArchive()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var remote = f.Remote.Add("Construction cam", photos, SyncFixture.Window33);
|
|
remote.Tags = ["site"];
|
|
f.Remote.CorruptOnce = remote.Frames[1].Id;
|
|
await f.RefreshAsync();
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var entry = await f.EntryAsync("Construction cam");
|
|
Assert.Equal(SyncStatus.InSync, entry.Status);
|
|
var local = Assert.NotNull(entry.Local);
|
|
Assert.Equal(SequenceOrigin.Server, local.Origin);
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, local.SettingsJson), "the server's settings came along");
|
|
Assert.SequenceEqual(["site"], local.Tags);
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.Equal(3, frames.Count);
|
|
foreach (var frame in frames)
|
|
{
|
|
var bytes = await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frame.RelativePath), f.Ct);
|
|
Assert.Equal(frame.Sha256, Sha(bytes));
|
|
}
|
|
Assert.True(frames.Select(r => r.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "the damaged transfer was fetched again");
|
|
Assert.FileExists(f.Archive.Paths.Sequence(local.Folder).ManifestPath);
|
|
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadFillsInWhatALocalCopyLacks()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Harbour", photos[..1], SyncFixture.Window41);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
await f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.KeepMine, f.Ct);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Equal(3, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Count);
|
|
var entry = await f.EntryAsync("Harbour");
|
|
Assert.Equal(SyncStatus.InSync, entry.Status);
|
|
Assert.Equal(SequenceOrigin.Local, entry.Local!.Origin);
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local.SettingsJson), "an existing copy keeps its settings");
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadingIntoAnUnverifiedCopyHashesItFirstAndFetchesOnlyWhatItLacks()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(5);
|
|
// An adopted folder: indexed by a scan, never verified, so its frames have no hash yet.
|
|
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
|
|
Directory.CreateDirectory(folder);
|
|
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"IMG_{i + 1:0000}.JPG"), photos[i], f.Ct);
|
|
await f.Archive.ScanAsync();
|
|
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Titano.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
|
|
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
|
|
var remote = f.Remote.Add("Adopted", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
Assert.Equal(2, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingLocal);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.Equal(5, frames.Count);
|
|
Assert.True(frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)), "no original arrived twice");
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task DownloadingOriginalsRestoresTheOnesMissingHere()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
|
|
var place = f.Archive.Paths.FromRelative(lost.RelativePath);
|
|
File.Delete(place);
|
|
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var behind = await f.EntryAsync("Harbour");
|
|
Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.Equal(lost.Sha256, Sha(await File.ReadAllBytesAsync(place, f.Ct)));
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.Equal(3, frames.Count);
|
|
Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the restored original is no longer missing");
|
|
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Star trails", photos);
|
|
var remote = f.Remote.Add("Star trails", photos);
|
|
var render = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 5000).Select(i => (byte)i)]);
|
|
remote.Renders.Add(render);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync();
|
|
var entry = await f.EntryAsync("Star trails");
|
|
Assert.Equal((SyncStatus.InSync, 1, 5000L), (entry.Status, entry.Link!.ExportsToGet, entry.Link.ExportsToGetBytes));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var fetched = Assert.Contains(await f.Db.Renders.ListBySequenceAsync(local.Id, f.Ct), r => r.Id == render.Id);
|
|
Assert.Equal("4K HEVC", fetched.PresetName);
|
|
var file = f.Archive.Paths.FromRelative(fetched.OutputPath);
|
|
Assert.Equal(Path.Combine(f.Archive.Paths.Sequence(local.Folder).Exports, "Star trails 4K.mp4"), file);
|
|
Assert.SequenceEqual(render.Bytes, await File.ReadAllBytesAsync(file, f.Ct));
|
|
Assert.Equal(0, (await f.EntryAsync("Star trails")).Link!.ExportsToGet);
|
|
}
|
|
|
|
[Test]
|
|
public async Task RendersMadeHereAreSentOnRequestAndAGoneFileIsReported()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
var exports = f.Archive.Paths.Sequence(local.Folder).Exports;
|
|
Directory.CreateDirectory(exports);
|
|
var bytes = Enumerable.Range(0, 3000).Select(i => (byte)(i * 7)).ToArray();
|
|
await File.WriteAllBytesAsync(Path.Combine(exports, "Harbour 1080p.mp4"), bytes, f.Ct);
|
|
var made = await AddRenderAsync(f, local, "Harbour 1080p.mp4", bytes.Length);
|
|
var gone = await AddRenderAsync(f, local, "Harbour draft.mp4", 10);
|
|
await f.RefreshAsync();
|
|
Assert.Equal((1, 3000L), ((await f.EntryAsync("Harbour")).Link!.ExportsToSend, (await f.EntryAsync("Harbour")).Link!.ExportsToSendBytes));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueSendExportsAsync(engine, f.Server, local);
|
|
Assert.Equal(SyncTransferKind.SendExports, SyncService.TransferKind(job));
|
|
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Contains("1", failed.Error);
|
|
var sent = Assert.Contains(remote.Renders, r => r.Id == made.Id);
|
|
Assert.SequenceEqual(bytes, sent.Bytes);
|
|
Assert.DoesNotContain(gone.Id, remote.Renders.Select(r => r.Id));
|
|
Assert.Equal(0, (await f.EntryAsync("Harbour")).Link!.ExportsToSend);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProxiesOpenARemoteSequenceForTheEditor()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Milan skyline", f.NewPhotos(3), SyncFixture.Window33);
|
|
await f.RefreshAsync();
|
|
Assert.Null(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Milan skyline", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var entry = await f.EntryAsync("Milan skyline");
|
|
Assert.Equal(SyncStatus.ProxiesOnly, entry.Status);
|
|
Assert.Equal(3, entry.Link!.ProxiesCached);
|
|
Assert.False(entry.Link.ProxiesStale);
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.Equal(remote.Id, opened.Sequence.Id);
|
|
Assert.Equal(SequenceOrigin.Server, opened.Sequence.Origin);
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, opened.Sequence.SettingsJson));
|
|
Assert.Equal(3, opened.CachedCount);
|
|
|
|
var source = opened.OpenFrames(DecoderFactory.CreateServer());
|
|
await using (source as IAsyncDisposable)
|
|
{
|
|
Assert.Equal(3, source.FrameCount);
|
|
using var frame = await source.GetFrameAsync(2, 0, DecodePurpose.Render, f.Ct);
|
|
Assert.Equal((48, 32), (frame.Width, frame.Height));
|
|
}
|
|
|
|
// Nothing local was created, and a frame added on the server makes the cache stale.
|
|
Assert.Null(entry.Local);
|
|
remote.AddFrame(f.NewPhotos(1)[0]);
|
|
await f.RefreshAsync();
|
|
Assert.True((await f.EntryAsync("Milan skyline")).Link!.ProxiesStale);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThumbnailsStandInWhenTheServerHasNoProxies()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.Equal(2, opened.CachedCount);
|
|
Assert.True(opened.Files.All(file => new FileInfo(file!).Length == 64), "thumbnails");
|
|
}
|
|
|
|
[Test]
|
|
public async Task TransfersFailWithAReasonWhenThisPcIsNotConnected()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var local = await f.LocalAsync("Comet", f.NewPhotos(1));
|
|
var engine = await f.StartEngineAsync(new SyncFixture.Connector(null));
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Equal(Titano.Core.Localization.Loc.L("Errors.Sync.NotConnected"), failed.Error);
|
|
Assert.Empty(f.Remote.Sequences);
|
|
}
|
|
|
|
private static async Task<Render> AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes)
|
|
{
|
|
var render = new Render
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
SequenceId = local.Id,
|
|
PresetName = "1080p H.264",
|
|
OutputPath = local.Folder + "/exports/" + name,
|
|
Codec = "h264",
|
|
Width = 1920,
|
|
Height = 1080,
|
|
Fps = 30,
|
|
DurationSeconds = 10,
|
|
Bytes = bytes,
|
|
Created = DateTimeOffset.UtcNow,
|
|
ProducedBy = "studio",
|
|
};
|
|
await f.Db.Renders.InsertAsync(render, f.Ct);
|
|
return render;
|
|
}
|
|
|
|
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes));
|
|
}
|