Files
Chronos/tests/Titano.Server.Tests/Sync/SyncServerTests.cs
T
Claude c87357b02c fix(sync): finish stuck downloads and keep edits saved during a comparison
A download whose .part already held the whole original (the job stopped while
hashing it) or more asked the server to resume past the end, got 416 and
failed on every retry; such a part is now hashed again, and one the server
cannot resume starts over (CD-1).

Auto-sync pulled the server's settings with the copy of the sequence read when
the comparison began, so an edit saved on this PC meanwhile was overwritten
unseen. The payload is now applied in one step under the sequence lease and
only while the local settings version is the one read; otherwise nothing is
written and the change is settled as a conflict (CD-3).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 09:58:36 +00:00

280 lines
18 KiB
C#

using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Imaging;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Repositories;
using Titano.Data.Sync;
using Titano.Protocol;
using Titano.Server.Events;
using Titano.Server.Storage;
using Titano.Testing;
namespace Titano.Server.Tests.Sync;
/// <summary>The client's sync engine against the real server: uploads, downloads, proxies and live events through the API.</summary>
[Category("server")]
[Timeout(180_000)]
public sealed class SyncServerTests(TestContext context)
{
private const string Window41 = """{"exposure":{"deflickerWindow":{"isAuto":false,"manual":41}}}""";
private const string Window33 = """{"exposure":{"deflickerWindow":{"isAuto":false,"manual":33}}}""";
/// <summary>How long any wait may take on a loaded runner; every wait ends as soon as what it waits for happened.</summary>
private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(60);
[Test]
public async Task ASequenceUploadedFromOnePcDownloadsIdenticallyOnAnother()
{
await using var server = await StartServerAsync();
var (_, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
var photos = Enumerable.Range(1, 3).Select(SyncClient.Photo).ToList();
var local = await studio.LocalAsync("Comet over the ridge", photos, Window41);
var upload = await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local);
await studio.WaitAsync(upload.Id);
var uploaded = await studio.EntryAsync("Comet over the ridge");
Assert.Equal((0, 0), (uploaded.Link!.MissingRemote, uploaded.Link.MissingLocal));
var remoteId = uploaded.Link!.RemoteId;
var onServer = Assert.NotNull(await server.Database.Sequences.GetAsync(remoteId));
Assert.Equal(41, ProjectSettings.FromJson(onServer.SettingsJson!).Exposure.DeflickerWindow.Manual, "settings travel with the originals");
var serverFrames = await server.Database.Frames.ListBySequenceAsync(remoteId);
Assert.True(serverFrames.Select(f => f.Sha256).ToHashSet().SetEquals(photos.Select(SyncClient.Sha)), "every original was committed");
await ServerIdleAsync(server);
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
Assert.Equal(SyncStatus.RemoteOnly, (await laptop.EntryAsync("Comet over the ridge")).Status);
var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Comet over the ridge", SyncDownloadMode.Originals,
laptop.Archive.Id);
await laptop.WaitAsync(download.Id);
var copy = await laptop.EntryAsync("Comet over the ridge");
Assert.Equal(SyncStatus.InSync, copy.Status);
Assert.Equal(SequenceOrigin.Server, copy.Local!.Origin);
// The server's analysis may have added its advice meanwhile; the copy has what the server has, the user's edit included.
Assert.True(SyncRules.SameSettings((await server.Database.Sequences.GetAsync(remoteId))!.SettingsJson, copy.Local.SettingsJson));
Assert.Equal(41, ProjectSettings.FromJson(copy.Local.SettingsJson!).Exposure.DeflickerWindow.Manual);
var files = (await laptop.Db.Frames.ListBySequenceAsync(copy.Local.Id))
.Select(f => File.ReadAllBytes(Path.Combine(laptop.Archive.RootPath, f.RelativePath))).ToList();
Assert.True(files.Select(SyncClient.Sha).ToHashSet().SetEquals(photos.Select(SyncClient.Sha)), "byte-identical originals");
await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation);
Assert.Equal(SyncStatus.InSync, (await studio.EntryAsync("Comet over the ridge")).Status);
}
[Test]
public async Task AnUploadInterruptedMidFileResumesTheServersPartialUpload()
{
await using var server = await StartServerAsync();
var (http, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
var photos = Enumerable.Range(10, 2).Select(SyncClient.Photo).ToList();
var local = await studio.LocalAsync("Clouds", photos);
// The crashed attempt had sent the first original (which created the server sequence) and one chunk of the second.
var first = await Uploads.UploadAsync(http, photos[0], "IMG_0001.JPG", sessionKey: "crashed-attempt");
var partial = await Uploads.StartAsync(http, photos[1], "IMG_0002.JPG", first.SequenceId);
Assert.True(photos[1].Length > partial.ChunkSize, "the photo spans several chunks");
using (var chunk = await Uploads.PutChunkAsync(http, partial.UploadId!.Value, 0, Uploads.Chunk(photos[1], partial.ChunkSize, 0)))
await chunk.ExpectAsync(HttpStatusCode.OK);
var job = new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.UploadCommit,
TargetIds = [local.Id],
TargetLabel = "Clouds",
ParametersJson = $$"""{"server":"{{studio.Server}}"}""",
Checkpoint = first.SequenceId.ToString(),
State = JobState.Queued,
Created = DateTimeOffset.UtcNow,
};
await studio.Db.Jobs.EnqueueAsync(job, context.Cancellation);
studio.Engine.Wake();
await studio.WaitAsync(job.Id);
var resumed = Assert.NotNull(await server.Database.Uploads.GetAsync(partial.UploadId!.Value));
Assert.Equal(Titano.Data.Repositories.UploadState.Committed, resumed.State);
Assert.Equal(2, (await server.Database.Frames.ListBySequenceAsync(first.SequenceId)).Count);
// Every original is on both sides (the server's analysis may still be editing the settings).
var link = (await studio.EntryAsync("Clouds")).Link!;
Assert.Equal((first.SequenceId, 0, 0), (link.RemoteId, link.MissingRemote, link.MissingLocal));
}
[Test]
public async Task ADownloadResumesPartialFilesAndRefusesDamagedOnes()
{
await using var server = await StartServerAsync();
var (_, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
var photos = Enumerable.Range(20, 4).Select(SyncClient.Photo).ToList();
var local = await studio.LocalAsync("Aurora", photos);
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
var remoteId = (await studio.EntryAsync("Aurora")).Link!.RemoteId;
// An earlier attempt left half of one original, a damaged half of another, a third complete (stopped while it was
// being hashed) and a fourth longer than the file: the server cannot resume either of the last two (416).
var staging = Path.Combine(laptop.Archive.RootPath, ".titano", "staging", "sync-" + remoteId.ToString("N"));
Directory.CreateDirectory(staging);
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[0]) + ".jpg.part"), photos[0][..(photos[0].Length / 2)]);
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[1]) + ".jpg.part"), new byte[photos[1].Length / 2]);
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[2]) + ".jpg.part"), photos[2]);
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[3]) + ".jpg.part"), [.. photos[3], .. new byte[64]]);
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Aurora", SyncDownloadMode.Originals, laptop.Archive.Id);
await laptop.WaitAsync(download.Id);
var copy = Assert.NotNull((await laptop.EntryAsync("Aurora")).Local);
var hashes = (await laptop.Db.Frames.ListBySequenceAsync(copy.Id))
.Select(f => SyncClient.Sha(File.ReadAllBytes(Path.Combine(laptop.Archive.RootPath, f.RelativePath)))).ToHashSet();
Assert.True(hashes.SetEquals(photos.Select(SyncClient.Sha)), "every original arrived intact");
Assert.False(Directory.Exists(staging), "staging cleaned up");
}
[Test]
public async Task ServerEventsUpdateThisPcWithoutWaitingForTheNextComparison()
{
await using var server = await StartServerAsync();
var (http, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
var local = await studio.LocalAsync("Harbour sunset", [SyncClient.Photo(30)]);
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
var remoteId = (await studio.EntryAsync("Harbour sunset")).Link!.RemoteId;
await ServerIdleAsync(server);
await using var agent = new SyncAgent(studio.Sync, studio.Remote, () => new SyncPolicy(), studio.Engine,
new SyncAgentOptions { RefreshInterval = TimeSpan.FromHours(1), EventDelay = TimeSpan.FromMilliseconds(50) });
agent.Start();
var hub = server.Services.GetRequiredService<EventHub>();
await UntilAsync(() => Task.FromResult(agent.Snapshot is not null && hub.SubscriberCount > 0), "the agent's first comparison and event socket");
// Someone edits the project on the server (web or another PC). The comparison its event starts writes the
// settings on this PC before the link that records the agreement: wait for the comparison, not for the settings.
var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
studio.Sync.Changed += _ => compared.TrySetResult();
using (var patch = await http.PatchAsJsonAsync($"/api/v1/sequences/{remoteId}",
new UpdateSequenceRequest(null, null, null, null, Window33, null), ProtocolJson.Default.UpdateSequenceRequest))
await patch.ExpectAsync(HttpStatusCode.OK);
await SignalAsync(compared.Task, "the comparison of the edit");
Assert.Equal(33, ProjectSettings.FromJson((await studio.Db.Sequences.GetAsync(local.Id))!.SettingsJson ?? "{}").Exposure.DeflickerWindow.Manual,
"the edit reached this PC");
Assert.Equal(SyncStatus.InSync, (await studio.EntryAsync("Harbour sunset")).Status);
}
[Test]
public async Task ARemoteSequenceOpensWithProxiesOnly()
{
await using var server = await StartServerAsync();
var (_, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
var photos = Enumerable.Range(40, 2).Select(SyncClient.Photo).ToList();
var local = await studio.LocalAsync("Milan skyline dusk", photos, Window41);
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
var remoteId = (await studio.EntryAsync("Milan skyline dusk")).Link!.RemoteId;
await ServerIdleAsync(server);
Assert.True(((await server.Database.Sequences.GetAsync(remoteId))!.Status & SequenceStatus.ThumbsBuilt) != 0, "thumbnails on the server");
// A remote card without a hero frame shows its first frame, read as one row instead of the whole list.
ISyncRemote remote = laptop.Remote;
Assert.Equal((await remote.ListFramesAsync(remoteId, context.Cancellation))[0].Id, (await remote.FirstFrameAsync(remoteId, context.Cancellation))?.Id);
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
var job = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Milan skyline dusk", SyncDownloadMode.Proxies, null);
await laptop.WaitAsync(job.Id);
Assert.Equal(SyncStatus.ProxiesOnly, (await laptop.EntryAsync("Milan skyline dusk")).Status);
Assert.Empty((await laptop.Db.Sequences.ListAsync(new SequenceQuery())).Items);
var opened = Assert.NotNull(laptop.Sync.OpenRemote(laptop.Server, remoteId));
Assert.Equal(2, opened.CachedCount);
Assert.Equal(41, ProjectSettings.FromJson(opened.Sequence.SettingsJson!).Exposure.DeflickerWindow.Manual);
var source = opened.OpenFrames(DecoderFactory.CreateServer());
await using (source as IAsyncDisposable)
{
using var frame = await source.GetFrameAsync(0, 0, DecodePurpose.Render, context.Cancellation);
Assert.True(frame.Width > 0 && frame.Height > 0, "a proxy decodes");
}
}
[Test]
public async Task RendersTravelOnRequestWithTheirIdAndAreVerified()
{
await using var server = await StartServerAsync();
var (_, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
var photos = Enumerable.Range(50, 2).Select(SyncClient.Photo).ToList();
var local = await studio.LocalAsync("Star trails", photos);
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
var remoteId = (await studio.EntryAsync("Star trails")).Link!.RemoteId;
await ServerIdleAsync(server);
// A render made on the studio PC, larger than one chunk, is sent on request and keeps its id.
var output = Enumerable.Range(0, 200_000).Select(i => (byte)(i * 31)).ToArray();
var render = await studio.RenderAsync(local, "Star trails 4K.mp4", output);
await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation);
Assert.Equal(1, (await studio.EntryAsync("Star trails")).Link!.ExportsToSend);
await studio.WaitAsync((await studio.Sync.EnqueueSendExportsAsync(studio.Engine, studio.Server, local)).Id);
var filed = Assert.NotNull(await server.Database.Renders.GetAsync(render.Id));
Assert.Equal(remoteId, filed.SequenceId);
Assert.SequenceEqual(output, await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, filed.OutputPath)));
Assert.Equal(0, (await studio.EntryAsync("Star trails")).Link!.ExportsToSend);
Assert.Null(await studio.Remote.UploadRenderAsync(remoteId, studio.RenderFile(render), new RenderUploadRequest(render.Id, "Star trails 4K.mp4",
output.Length, SyncClient.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, "studio"), context.Cancellation));
// A damaged transfer is refused at completion.
var bad = new RenderUploadRequest(Guid.CreateVersion7(), "Damaged.mp4", output.Length, SyncClient.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null);
var damaged = Path.Combine(context.CreateTempDirectory("damaged"), "Damaged.mp4");
await File.WriteAllBytesAsync(damaged, [.. output.Select(b => (byte)~b)]);
var refused = await Assert.ThrowsAsync<Titano.Protocol.Client.TitanoApiException>(() =>
studio.Remote.UploadRenderAsync(remoteId, damaged, bad, context.Cancellation));
Assert.Equal(HttpStatusCode.UnprocessableEntity, refused.Status);
Assert.Null(await server.Database.Renders.GetAsync(bad.RenderId));
// Another PC with the originals gets the render into its own exports, byte-identical.
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Star trails", SyncDownloadMode.Originals,
laptop.Archive.Id)).Id);
Assert.Equal(1, (await laptop.EntryAsync("Star trails")).Link!.ExportsToGet);
await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Star trails", SyncDownloadMode.Exports, null)).Id);
var copy = (await laptop.EntryAsync("Star trails")).Local!;
var fetched = Assert.Contains(await laptop.Db.Renders.ListBySequenceAsync(copy.Id), r => r.Id == render.Id);
Assert.SequenceEqual(output, await File.ReadAllBytesAsync(Path.Combine(laptop.Archive.RootPath, fetched.OutputPath)));
Assert.Equal(0, (await laptop.EntryAsync("Star trails")).Link!.ExportsToGet);
}
private Task<TestServer> StartServerAsync() => TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register);
/// <summary>Waits until the server's engines finished what the commits queued (thumbnails, proxies, analysis, which may edit settings).</summary>
private Task ServerIdleAsync(TestServer server) =>
UntilAsync(async () => await server.Database.Jobs.CountAsync(DateTimeOffset.UtcNow) is { Queued: 0, Running: 0 }, "the server's jobs");
private async Task UntilAsync(Func<Task<bool>> condition, string what)
{
var until = DateTime.UtcNow + Deadline;
while (!await condition())
{
if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what);
await Task.Delay(50, context.Cancellation);
}
}
private async Task SignalAsync(Task signal, string what)
{
try
{
await signal.WaitAsync(Deadline, context.Cancellation);
}
catch (TimeoutException)
{
Assert.Fail("Timed out waiting for " + what);
}
}
}