ServerEventsUpdateThisPcWithoutWaitingForTheNextComparison polled until the server's settings reached the local copy, then expected the row in sync; the link that records the agreement is stored after the local copy, so the row could still read LocalAhead (1 failure in 15 loaded runs). It now waits for SyncService.Changed, raised once the comparison stored every link and state. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
274 lines
17 KiB
C#
274 lines
17 KiB
C#
using System.Net;
|
|
using System.Net.Http.Json;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Titano.Core.Imaging;
|
|
using Titano.Core.Imaging.Decoders;
|
|
using Titano.Core.Model;
|
|
using Titano.Core.Project;
|
|
using Titano.Data.Repositories;
|
|
using Titano.Data.Sync;
|
|
using Titano.Protocol;
|
|
using Titano.Server.Events;
|
|
using Titano.Server.Storage;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Server.Tests.Sync;
|
|
|
|
/// <summary>The client's sync engine against the real server: uploads, downloads, proxies and live events through the API.</summary>
|
|
[Category("server")]
|
|
[Timeout(180_000)]
|
|
public sealed class SyncServerTests(TestContext context)
|
|
{
|
|
private const string Window41 = """{"exposure":{"deflickerWindow":{"isAuto":false,"manual":41}}}""";
|
|
private const string Window33 = """{"exposure":{"deflickerWindow":{"isAuto":false,"manual":33}}}""";
|
|
|
|
/// <summary>How long any wait may take on a loaded runner; every wait ends as soon as what it waits for happened.</summary>
|
|
private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(60);
|
|
|
|
[Test]
|
|
public async Task ASequenceUploadedFromOnePcDownloadsIdenticallyOnAnother()
|
|
{
|
|
await using var server = await StartServerAsync();
|
|
var (_, _, token) = await server.UserAsync("alberto", "editor");
|
|
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
|
|
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
|
|
var photos = Enumerable.Range(1, 3).Select(SyncClient.Photo).ToList();
|
|
var local = await studio.LocalAsync("Comet over the ridge", photos, Window41);
|
|
|
|
var upload = await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local);
|
|
await studio.WaitAsync(upload.Id);
|
|
|
|
var uploaded = await studio.EntryAsync("Comet over the ridge");
|
|
Assert.Equal((0, 0), (uploaded.Link!.MissingRemote, uploaded.Link.MissingLocal));
|
|
var remoteId = uploaded.Link!.RemoteId;
|
|
var onServer = Assert.NotNull(await server.Database.Sequences.GetAsync(remoteId));
|
|
Assert.Equal(41, ProjectSettings.FromJson(onServer.SettingsJson!).Exposure.DeflickerWindow.Manual, "settings travel with the originals");
|
|
var serverFrames = await server.Database.Frames.ListBySequenceAsync(remoteId);
|
|
Assert.True(serverFrames.Select(f => f.Sha256).ToHashSet().SetEquals(photos.Select(SyncClient.Sha)), "every original was committed");
|
|
|
|
await ServerIdleAsync(server);
|
|
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
|
|
Assert.Equal(SyncStatus.RemoteOnly, (await laptop.EntryAsync("Comet over the ridge")).Status);
|
|
var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Comet over the ridge", SyncDownloadMode.Originals,
|
|
laptop.Archive.Id);
|
|
await laptop.WaitAsync(download.Id);
|
|
|
|
var copy = await laptop.EntryAsync("Comet over the ridge");
|
|
Assert.Equal(SyncStatus.InSync, copy.Status);
|
|
Assert.Equal(SequenceOrigin.Server, copy.Local!.Origin);
|
|
// The server's analysis may have added its advice meanwhile; the copy has what the server has, the user's edit included.
|
|
Assert.True(SyncRules.SameSettings((await server.Database.Sequences.GetAsync(remoteId))!.SettingsJson, copy.Local.SettingsJson));
|
|
Assert.Equal(41, ProjectSettings.FromJson(copy.Local.SettingsJson!).Exposure.DeflickerWindow.Manual);
|
|
var files = (await laptop.Db.Frames.ListBySequenceAsync(copy.Local.Id))
|
|
.Select(f => File.ReadAllBytes(Path.Combine(laptop.Archive.RootPath, f.RelativePath))).ToList();
|
|
Assert.True(files.Select(SyncClient.Sha).ToHashSet().SetEquals(photos.Select(SyncClient.Sha)), "byte-identical originals");
|
|
|
|
await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation);
|
|
Assert.Equal(SyncStatus.InSync, (await studio.EntryAsync("Comet over the ridge")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUploadInterruptedMidFileResumesTheServersPartialUpload()
|
|
{
|
|
await using var server = await StartServerAsync();
|
|
var (http, _, token) = await server.UserAsync("alberto", "editor");
|
|
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
|
|
var photos = Enumerable.Range(10, 2).Select(SyncClient.Photo).ToList();
|
|
var local = await studio.LocalAsync("Clouds", photos);
|
|
// The crashed attempt had sent the first original (which created the server sequence) and one chunk of the second.
|
|
var first = await Uploads.UploadAsync(http, photos[0], "IMG_0001.JPG", sessionKey: "crashed-attempt");
|
|
var partial = await Uploads.StartAsync(http, photos[1], "IMG_0002.JPG", first.SequenceId);
|
|
Assert.True(photos[1].Length > partial.ChunkSize, "the photo spans several chunks");
|
|
using (var chunk = await Uploads.PutChunkAsync(http, partial.UploadId!.Value, 0, Uploads.Chunk(photos[1], partial.ChunkSize, 0)))
|
|
await chunk.ExpectAsync(HttpStatusCode.OK);
|
|
|
|
var job = new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = JobKind.UploadCommit,
|
|
TargetIds = [local.Id],
|
|
TargetLabel = "Clouds",
|
|
ParametersJson = $$"""{"server":"{{studio.Server}}"}""",
|
|
Checkpoint = first.SequenceId.ToString(),
|
|
State = JobState.Queued,
|
|
Created = DateTimeOffset.UtcNow,
|
|
};
|
|
await studio.Db.Jobs.EnqueueAsync(job, context.Cancellation);
|
|
studio.Engine.Wake();
|
|
await studio.WaitAsync(job.Id);
|
|
|
|
var resumed = Assert.NotNull(await server.Database.Uploads.GetAsync(partial.UploadId!.Value));
|
|
Assert.Equal(Titano.Data.Repositories.UploadState.Committed, resumed.State);
|
|
Assert.Equal(2, (await server.Database.Frames.ListBySequenceAsync(first.SequenceId)).Count);
|
|
// Every original is on both sides (the server's analysis may still be editing the settings).
|
|
var link = (await studio.EntryAsync("Clouds")).Link!;
|
|
Assert.Equal((first.SequenceId, 0, 0), (link.RemoteId, link.MissingRemote, link.MissingLocal));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADownloadResumesPartialFilesAndRefusesDamagedOnes()
|
|
{
|
|
await using var server = await StartServerAsync();
|
|
var (_, _, token) = await server.UserAsync("alberto", "editor");
|
|
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
|
|
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
|
|
var photos = Enumerable.Range(20, 3).Select(SyncClient.Photo).ToList();
|
|
var local = await studio.LocalAsync("Aurora", photos);
|
|
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
|
|
var remoteId = (await studio.EntryAsync("Aurora")).Link!.RemoteId;
|
|
|
|
// An earlier attempt left half of one original, and a damaged half of another.
|
|
var staging = Path.Combine(laptop.Archive.RootPath, ".titano", "staging", "sync-" + remoteId.ToString("N"));
|
|
Directory.CreateDirectory(staging);
|
|
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[0]) + ".jpg.part"), photos[0][..(photos[0].Length / 2)]);
|
|
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[1]) + ".jpg.part"), new byte[photos[1].Length / 2]);
|
|
|
|
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
|
|
var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Aurora", SyncDownloadMode.Originals, laptop.Archive.Id);
|
|
await laptop.WaitAsync(download.Id);
|
|
|
|
var copy = Assert.NotNull((await laptop.EntryAsync("Aurora")).Local);
|
|
var hashes = (await laptop.Db.Frames.ListBySequenceAsync(copy.Id))
|
|
.Select(f => SyncClient.Sha(File.ReadAllBytes(Path.Combine(laptop.Archive.RootPath, f.RelativePath)))).ToHashSet();
|
|
Assert.True(hashes.SetEquals(photos.Select(SyncClient.Sha)), "every original arrived intact");
|
|
Assert.False(Directory.Exists(staging), "staging cleaned up");
|
|
}
|
|
|
|
[Test]
|
|
public async Task ServerEventsUpdateThisPcWithoutWaitingForTheNextComparison()
|
|
{
|
|
await using var server = await StartServerAsync();
|
|
var (http, _, token) = await server.UserAsync("alberto", "editor");
|
|
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
|
|
var local = await studio.LocalAsync("Harbour sunset", [SyncClient.Photo(30)]);
|
|
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
|
|
var remoteId = (await studio.EntryAsync("Harbour sunset")).Link!.RemoteId;
|
|
await ServerIdleAsync(server);
|
|
await using var agent = new SyncAgent(studio.Sync, studio.Remote, () => new SyncPolicy(), studio.Engine,
|
|
new SyncAgentOptions { RefreshInterval = TimeSpan.FromHours(1), EventDelay = TimeSpan.FromMilliseconds(50) });
|
|
agent.Start();
|
|
var hub = server.Services.GetRequiredService<EventHub>();
|
|
await UntilAsync(() => Task.FromResult(agent.Snapshot is not null && hub.SubscriberCount > 0), "the agent's first comparison and event socket");
|
|
|
|
// Someone edits the project on the server (web or another PC). The comparison its event starts writes the
|
|
// settings on this PC before the link that records the agreement: wait for the comparison, not for the settings.
|
|
var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
studio.Sync.Changed += _ => compared.TrySetResult();
|
|
using (var patch = await http.PatchAsJsonAsync($"/api/v1/sequences/{remoteId}",
|
|
new UpdateSequenceRequest(null, null, null, null, Window33, null), ProtocolJson.Default.UpdateSequenceRequest))
|
|
await patch.ExpectAsync(HttpStatusCode.OK);
|
|
|
|
await SignalAsync(compared.Task, "the comparison of the edit");
|
|
Assert.Equal(33, ProjectSettings.FromJson((await studio.Db.Sequences.GetAsync(local.Id))!.SettingsJson ?? "{}").Exposure.DeflickerWindow.Manual,
|
|
"the edit reached this PC");
|
|
Assert.Equal(SyncStatus.InSync, (await studio.EntryAsync("Harbour sunset")).Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ARemoteSequenceOpensWithProxiesOnly()
|
|
{
|
|
await using var server = await StartServerAsync();
|
|
var (_, _, token) = await server.UserAsync("alberto", "editor");
|
|
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
|
|
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
|
|
var photos = Enumerable.Range(40, 2).Select(SyncClient.Photo).ToList();
|
|
var local = await studio.LocalAsync("Milan skyline dusk", photos, Window41);
|
|
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
|
|
var remoteId = (await studio.EntryAsync("Milan skyline dusk")).Link!.RemoteId;
|
|
await ServerIdleAsync(server);
|
|
Assert.True(((await server.Database.Sequences.GetAsync(remoteId))!.Status & SequenceStatus.ThumbsBuilt) != 0, "thumbnails on the server");
|
|
|
|
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
|
|
var job = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Milan skyline dusk", SyncDownloadMode.Proxies, null);
|
|
await laptop.WaitAsync(job.Id);
|
|
|
|
Assert.Equal(SyncStatus.ProxiesOnly, (await laptop.EntryAsync("Milan skyline dusk")).Status);
|
|
Assert.Empty((await laptop.Db.Sequences.ListAsync(new SequenceQuery())).Items);
|
|
var opened = Assert.NotNull(laptop.Sync.OpenRemote(laptop.Server, remoteId));
|
|
Assert.Equal(2, opened.CachedCount);
|
|
Assert.Equal(41, ProjectSettings.FromJson(opened.Sequence.SettingsJson!).Exposure.DeflickerWindow.Manual);
|
|
var source = opened.OpenFrames(DecoderFactory.CreateServer());
|
|
await using (source as IAsyncDisposable)
|
|
{
|
|
using var frame = await source.GetFrameAsync(0, 0, DecodePurpose.Render, context.Cancellation);
|
|
Assert.True(frame.Width > 0 && frame.Height > 0, "a proxy decodes");
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task RendersTravelOnRequestWithTheirIdAndAreVerified()
|
|
{
|
|
await using var server = await StartServerAsync();
|
|
var (_, _, token) = await server.UserAsync("alberto", "editor");
|
|
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
|
|
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
|
|
var photos = Enumerable.Range(50, 2).Select(SyncClient.Photo).ToList();
|
|
var local = await studio.LocalAsync("Star trails", photos);
|
|
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
|
|
var remoteId = (await studio.EntryAsync("Star trails")).Link!.RemoteId;
|
|
await ServerIdleAsync(server);
|
|
|
|
// A render made on the studio PC, larger than one chunk, is sent on request and keeps its id.
|
|
var output = Enumerable.Range(0, 200_000).Select(i => (byte)(i * 31)).ToArray();
|
|
var render = await studio.RenderAsync(local, "Star trails 4K.mp4", output);
|
|
await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation);
|
|
Assert.Equal(1, (await studio.EntryAsync("Star trails")).Link!.ExportsToSend);
|
|
await studio.WaitAsync((await studio.Sync.EnqueueSendExportsAsync(studio.Engine, studio.Server, local)).Id);
|
|
|
|
var filed = Assert.NotNull(await server.Database.Renders.GetAsync(render.Id));
|
|
Assert.Equal(remoteId, filed.SequenceId);
|
|
Assert.SequenceEqual(output, await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, filed.OutputPath)));
|
|
Assert.Equal(0, (await studio.EntryAsync("Star trails")).Link!.ExportsToSend);
|
|
Assert.Null(await studio.Remote.UploadRenderAsync(remoteId, studio.RenderFile(render), new RenderUploadRequest(render.Id, "Star trails 4K.mp4",
|
|
output.Length, SyncClient.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, "studio"), context.Cancellation));
|
|
|
|
// A damaged transfer is refused at completion.
|
|
var bad = new RenderUploadRequest(Guid.CreateVersion7(), "Damaged.mp4", output.Length, SyncClient.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null);
|
|
var damaged = Path.Combine(context.CreateTempDirectory("damaged"), "Damaged.mp4");
|
|
await File.WriteAllBytesAsync(damaged, [.. output.Select(b => (byte)~b)]);
|
|
var refused = await Assert.ThrowsAsync<Titano.Protocol.Client.TitanoApiException>(() =>
|
|
studio.Remote.UploadRenderAsync(remoteId, damaged, bad, context.Cancellation));
|
|
Assert.Equal(HttpStatusCode.UnprocessableEntity, refused.Status);
|
|
Assert.Null(await server.Database.Renders.GetAsync(bad.RenderId));
|
|
|
|
// Another PC with the originals gets the render into its own exports, byte-identical.
|
|
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
|
|
await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Star trails", SyncDownloadMode.Originals,
|
|
laptop.Archive.Id)).Id);
|
|
Assert.Equal(1, (await laptop.EntryAsync("Star trails")).Link!.ExportsToGet);
|
|
await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Star trails", SyncDownloadMode.Exports, null)).Id);
|
|
var copy = (await laptop.EntryAsync("Star trails")).Local!;
|
|
var fetched = Assert.Contains(await laptop.Db.Renders.ListBySequenceAsync(copy.Id), r => r.Id == render.Id);
|
|
Assert.SequenceEqual(output, await File.ReadAllBytesAsync(Path.Combine(laptop.Archive.RootPath, fetched.OutputPath)));
|
|
Assert.Equal(0, (await laptop.EntryAsync("Star trails")).Link!.ExportsToGet);
|
|
}
|
|
|
|
private Task<TestServer> StartServerAsync() => TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register);
|
|
|
|
/// <summary>Waits until the server's engines finished what the commits queued (thumbnails, proxies, analysis, which may edit settings).</summary>
|
|
private Task ServerIdleAsync(TestServer server) =>
|
|
UntilAsync(async () => await server.Database.Jobs.CountAsync(DateTimeOffset.UtcNow) is { Queued: 0, Running: 0 }, "the server's jobs");
|
|
|
|
private async Task UntilAsync(Func<Task<bool>> condition, string what)
|
|
{
|
|
var until = DateTime.UtcNow + Deadline;
|
|
while (!await condition())
|
|
{
|
|
if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what);
|
|
await Task.Delay(50, context.Cancellation);
|
|
}
|
|
}
|
|
|
|
private async Task SignalAsync(Task signal, string what)
|
|
{
|
|
try
|
|
{
|
|
await signal.WaitAsync(Deadline, context.Cancellation);
|
|
}
|
|
catch (TimeoutException)
|
|
{
|
|
Assert.Fail("Timed out waiting for " + what);
|
|
}
|
|
}
|
|
}
|