using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Imaging;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Repositories;
using Titano.Data.Sync;
using Titano.Protocol;
using Titano.Server.Events;
using Titano.Server.Storage;
using Titano.Testing;
namespace Titano.Server.Tests.Sync;
/// The client's sync engine against the real server: uploads, downloads, proxies and live events through the API.
[Category("server")]
[Timeout(180_000)]
public sealed class SyncServerTests(TestContext context)
{
private const string Window41 = """{"exposure":{"deflickerWindow":{"isAuto":false,"manual":41}}}""";
private const string Window33 = """{"exposure":{"deflickerWindow":{"isAuto":false,"manual":33}}}""";
/// How long any wait may take on a loaded runner; every wait ends as soon as what it waits for happened.
private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(60);
[Test]
public async Task ASequenceUploadedFromOnePcDownloadsIdenticallyOnAnother()
{
await using var server = await StartServerAsync();
var (_, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
var photos = Enumerable.Range(1, 3).Select(SyncClient.Photo).ToList();
var local = await studio.LocalAsync("Comet over the ridge", photos, Window41);
var upload = await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local);
await studio.WaitAsync(upload.Id);
var uploaded = await studio.EntryAsync("Comet over the ridge");
Assert.Equal((0, 0), (uploaded.Link!.MissingRemote, uploaded.Link.MissingLocal));
var remoteId = uploaded.Link!.RemoteId;
var onServer = Assert.NotNull(await server.Database.Sequences.GetAsync(remoteId));
Assert.Equal(41, ProjectSettings.FromJson(onServer.SettingsJson!).Exposure.DeflickerWindow.Manual, "settings travel with the originals");
var serverFrames = await server.Database.Frames.ListBySequenceAsync(remoteId);
Assert.True(serverFrames.Select(f => f.Sha256).ToHashSet().SetEquals(photos.Select(SyncClient.Sha)), "every original was committed");
await ServerIdleAsync(server);
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
Assert.Equal(SyncStatus.RemoteOnly, (await laptop.EntryAsync("Comet over the ridge")).Status);
var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Comet over the ridge", SyncDownloadMode.Originals,
laptop.Archive.Id);
await laptop.WaitAsync(download.Id);
var copy = await laptop.EntryAsync("Comet over the ridge");
Assert.Equal(SyncStatus.InSync, copy.Status);
Assert.Equal(SequenceOrigin.Server, copy.Local!.Origin);
// The server's analysis may have added its advice meanwhile; the copy has what the server has, the user's edit included.
Assert.True(SyncRules.SameSettings((await server.Database.Sequences.GetAsync(remoteId))!.SettingsJson, copy.Local.SettingsJson));
Assert.Equal(41, ProjectSettings.FromJson(copy.Local.SettingsJson!).Exposure.DeflickerWindow.Manual);
var files = (await laptop.Db.Frames.ListBySequenceAsync(copy.Local.Id))
.Select(f => File.ReadAllBytes(Path.Combine(laptop.Archive.RootPath, f.RelativePath))).ToList();
Assert.True(files.Select(SyncClient.Sha).ToHashSet().SetEquals(photos.Select(SyncClient.Sha)), "byte-identical originals");
await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation);
Assert.Equal(SyncStatus.InSync, (await studio.EntryAsync("Comet over the ridge")).Status);
}
[Test]
public async Task AnUploadInterruptedMidFileResumesTheServersPartialUpload()
{
await using var server = await StartServerAsync();
var (http, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
var photos = Enumerable.Range(10, 2).Select(SyncClient.Photo).ToList();
var local = await studio.LocalAsync("Clouds", photos);
// The crashed attempt had sent the first original (which created the server sequence) and one chunk of the second.
var first = await Uploads.UploadAsync(http, photos[0], "IMG_0001.JPG", sessionKey: "crashed-attempt");
var partial = await Uploads.StartAsync(http, photos[1], "IMG_0002.JPG", first.SequenceId);
Assert.True(photos[1].Length > partial.ChunkSize, "the photo spans several chunks");
using (var chunk = await Uploads.PutChunkAsync(http, partial.UploadId!.Value, 0, Uploads.Chunk(photos[1], partial.ChunkSize, 0)))
await chunk.ExpectAsync(HttpStatusCode.OK);
var job = new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.UploadCommit,
TargetIds = [local.Id],
TargetLabel = "Clouds",
ParametersJson = $$"""{"server":"{{studio.Server}}"}""",
Checkpoint = first.SequenceId.ToString(),
State = JobState.Queued,
Created = DateTimeOffset.UtcNow,
};
await studio.Db.Jobs.EnqueueAsync(job, context.Cancellation);
studio.Engine.Wake();
await studio.WaitAsync(job.Id);
var resumed = Assert.NotNull(await server.Database.Uploads.GetAsync(partial.UploadId!.Value));
Assert.Equal(Titano.Data.Repositories.UploadState.Committed, resumed.State);
Assert.Equal(2, (await server.Database.Frames.ListBySequenceAsync(first.SequenceId)).Count);
// Every original is on both sides (the server's analysis may still be editing the settings).
var link = (await studio.EntryAsync("Clouds")).Link!;
Assert.Equal((first.SequenceId, 0, 0), (link.RemoteId, link.MissingRemote, link.MissingLocal));
}
[Test]
public async Task ADownloadResumesPartialFilesAndRefusesDamagedOnes()
{
await using var server = await StartServerAsync();
var (_, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
var photos = Enumerable.Range(20, 3).Select(SyncClient.Photo).ToList();
var local = await studio.LocalAsync("Aurora", photos);
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
var remoteId = (await studio.EntryAsync("Aurora")).Link!.RemoteId;
// An earlier attempt left half of one original, and a damaged half of another.
var staging = Path.Combine(laptop.Archive.RootPath, ".titano", "staging", "sync-" + remoteId.ToString("N"));
Directory.CreateDirectory(staging);
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[0]) + ".jpg.part"), photos[0][..(photos[0].Length / 2)]);
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[1]) + ".jpg.part"), new byte[photos[1].Length / 2]);
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Aurora", SyncDownloadMode.Originals, laptop.Archive.Id);
await laptop.WaitAsync(download.Id);
var copy = Assert.NotNull((await laptop.EntryAsync("Aurora")).Local);
var hashes = (await laptop.Db.Frames.ListBySequenceAsync(copy.Id))
.Select(f => SyncClient.Sha(File.ReadAllBytes(Path.Combine(laptop.Archive.RootPath, f.RelativePath)))).ToHashSet();
Assert.True(hashes.SetEquals(photos.Select(SyncClient.Sha)), "every original arrived intact");
Assert.False(Directory.Exists(staging), "staging cleaned up");
}
[Test]
public async Task ServerEventsUpdateThisPcWithoutWaitingForTheNextComparison()
{
await using var server = await StartServerAsync();
var (http, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
var local = await studio.LocalAsync("Harbour sunset", [SyncClient.Photo(30)]);
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
var remoteId = (await studio.EntryAsync("Harbour sunset")).Link!.RemoteId;
await ServerIdleAsync(server);
await using var agent = new SyncAgent(studio.Sync, studio.Remote, () => new SyncPolicy(), studio.Engine,
new SyncAgentOptions { RefreshInterval = TimeSpan.FromHours(1), EventDelay = TimeSpan.FromMilliseconds(50) });
agent.Start();
var hub = server.Services.GetRequiredService();
await UntilAsync(() => Task.FromResult(agent.Snapshot is not null && hub.SubscriberCount > 0), "the agent's first comparison and event socket");
// Someone edits the project on the server (web or another PC). The comparison its event starts writes the
// settings on this PC before the link that records the agreement: wait for the comparison, not for the settings.
var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
studio.Sync.Changed += _ => compared.TrySetResult();
using (var patch = await http.PatchAsJsonAsync($"/api/v1/sequences/{remoteId}",
new UpdateSequenceRequest(null, null, null, null, Window33, null), ProtocolJson.Default.UpdateSequenceRequest))
await patch.ExpectAsync(HttpStatusCode.OK);
await SignalAsync(compared.Task, "the comparison of the edit");
Assert.Equal(33, ProjectSettings.FromJson((await studio.Db.Sequences.GetAsync(local.Id))!.SettingsJson ?? "{}").Exposure.DeflickerWindow.Manual,
"the edit reached this PC");
Assert.Equal(SyncStatus.InSync, (await studio.EntryAsync("Harbour sunset")).Status);
}
[Test]
public async Task ARemoteSequenceOpensWithProxiesOnly()
{
await using var server = await StartServerAsync();
var (_, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
var photos = Enumerable.Range(40, 2).Select(SyncClient.Photo).ToList();
var local = await studio.LocalAsync("Milan skyline dusk", photos, Window41);
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
var remoteId = (await studio.EntryAsync("Milan skyline dusk")).Link!.RemoteId;
await ServerIdleAsync(server);
Assert.True(((await server.Database.Sequences.GetAsync(remoteId))!.Status & SequenceStatus.ThumbsBuilt) != 0, "thumbnails on the server");
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
var job = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Milan skyline dusk", SyncDownloadMode.Proxies, null);
await laptop.WaitAsync(job.Id);
Assert.Equal(SyncStatus.ProxiesOnly, (await laptop.EntryAsync("Milan skyline dusk")).Status);
Assert.Empty((await laptop.Db.Sequences.ListAsync(new SequenceQuery())).Items);
var opened = Assert.NotNull(laptop.Sync.OpenRemote(laptop.Server, remoteId));
Assert.Equal(2, opened.CachedCount);
Assert.Equal(41, ProjectSettings.FromJson(opened.Sequence.SettingsJson!).Exposure.DeflickerWindow.Manual);
var source = opened.OpenFrames(DecoderFactory.CreateServer());
await using (source as IAsyncDisposable)
{
using var frame = await source.GetFrameAsync(0, 0, DecodePurpose.Render, context.Cancellation);
Assert.True(frame.Width > 0 && frame.Height > 0, "a proxy decodes");
}
}
[Test]
public async Task RendersTravelOnRequestWithTheirIdAndAreVerified()
{
await using var server = await StartServerAsync();
var (_, _, token) = await server.UserAsync("alberto", "editor");
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
var photos = Enumerable.Range(50, 2).Select(SyncClient.Photo).ToList();
var local = await studio.LocalAsync("Star trails", photos);
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
var remoteId = (await studio.EntryAsync("Star trails")).Link!.RemoteId;
await ServerIdleAsync(server);
// A render made on the studio PC, larger than one chunk, is sent on request and keeps its id.
var output = Enumerable.Range(0, 200_000).Select(i => (byte)(i * 31)).ToArray();
var render = await studio.RenderAsync(local, "Star trails 4K.mp4", output);
await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation);
Assert.Equal(1, (await studio.EntryAsync("Star trails")).Link!.ExportsToSend);
await studio.WaitAsync((await studio.Sync.EnqueueSendExportsAsync(studio.Engine, studio.Server, local)).Id);
var filed = Assert.NotNull(await server.Database.Renders.GetAsync(render.Id));
Assert.Equal(remoteId, filed.SequenceId);
Assert.SequenceEqual(output, await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, filed.OutputPath)));
Assert.Equal(0, (await studio.EntryAsync("Star trails")).Link!.ExportsToSend);
Assert.Null(await studio.Remote.UploadRenderAsync(remoteId, studio.RenderFile(render), new RenderUploadRequest(render.Id, "Star trails 4K.mp4",
output.Length, SyncClient.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, "studio"), context.Cancellation));
// A damaged transfer is refused at completion.
var bad = new RenderUploadRequest(Guid.CreateVersion7(), "Damaged.mp4", output.Length, SyncClient.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null);
var damaged = Path.Combine(context.CreateTempDirectory("damaged"), "Damaged.mp4");
await File.WriteAllBytesAsync(damaged, [.. output.Select(b => (byte)~b)]);
var refused = await Assert.ThrowsAsync(() =>
studio.Remote.UploadRenderAsync(remoteId, damaged, bad, context.Cancellation));
Assert.Equal(HttpStatusCode.UnprocessableEntity, refused.Status);
Assert.Null(await server.Database.Renders.GetAsync(bad.RenderId));
// Another PC with the originals gets the render into its own exports, byte-identical.
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Star trails", SyncDownloadMode.Originals,
laptop.Archive.Id)).Id);
Assert.Equal(1, (await laptop.EntryAsync("Star trails")).Link!.ExportsToGet);
await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Star trails", SyncDownloadMode.Exports, null)).Id);
var copy = (await laptop.EntryAsync("Star trails")).Local!;
var fetched = Assert.Contains(await laptop.Db.Renders.ListBySequenceAsync(copy.Id), r => r.Id == render.Id);
Assert.SequenceEqual(output, await File.ReadAllBytesAsync(Path.Combine(laptop.Archive.RootPath, fetched.OutputPath)));
Assert.Equal(0, (await laptop.EntryAsync("Star trails")).Link!.ExportsToGet);
}
private Task StartServerAsync() => TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register);
/// Waits until the server's engines finished what the commits queued (thumbnails, proxies, analysis, which may edit settings).
private Task ServerIdleAsync(TestServer server) =>
UntilAsync(async () => await server.Database.Jobs.CountAsync(DateTimeOffset.UtcNow) is { Queued: 0, Running: 0 }, "the server's jobs");
private async Task UntilAsync(Func> condition, string what)
{
var until = DateTime.UtcNow + Deadline;
while (!await condition())
{
if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what);
await Task.Delay(50, context.Cancellation);
}
}
private async Task SignalAsync(Task signal, string what)
{
try
{
await signal.WaitAsync(Deadline, context.Cancellation);
}
catch (TimeoutException)
{
Assert.Fail("Timed out waiting for " + what);
}
}
}