using System.Net; using System.Net.Http.Json; using Microsoft.Extensions.DependencyInjection; using Chronos.Core.Imaging; using Chronos.Core.Imaging.Decoders; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Repositories; using Chronos.Data.Sync; using Chronos.Protocol; using Chronos.Server.Events; using Chronos.Server.Storage; using Chronos.Testing; namespace Chronos.Server.Tests.Sync; /// The client's sync engine against the real server: uploads, downloads, proxies and live events through the API. [Category("server")] [Timeout(180_000)] public sealed class SyncServerTests(TestContext context) { private const string Window41 = """{"exposure":{"deflickerWindow":{"isAuto":false,"manual":41}}}"""; private const string Window33 = """{"exposure":{"deflickerWindow":{"isAuto":false,"manual":33}}}"""; /// How long any wait may take on a loaded runner; every wait ends as soon as what it waits for happened. private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(60); [Test] public async Task ASequenceUploadedFromOnePcDownloadsIdenticallyOnAnother() { await using var server = await StartServerAsync(); var (_, _, token) = await server.UserAsync("alberto", "editor"); await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop"); var photos = Enumerable.Range(1, 3).Select(SyncClient.Photo).ToList(); var local = await studio.LocalAsync("Comet over the ridge", photos, Window41); var upload = await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local); await studio.WaitAsync(upload.Id); var uploaded = await studio.EntryAsync("Comet over the ridge"); Assert.Equal((0, 0), (uploaded.Link!.MissingRemote, uploaded.Link.MissingLocal)); var remoteId = uploaded.Link!.RemoteId; var onServer = Assert.NotNull(await server.Database.Sequences.GetAsync(remoteId)); Assert.Equal(41, ProjectSettings.FromJson(onServer.SettingsJson!).Exposure.DeflickerWindow.Manual, "settings travel with the originals"); var serverFrames = await server.Database.Frames.ListBySequenceAsync(remoteId); Assert.True(serverFrames.Select(f => f.Sha256).ToHashSet().SetEquals(photos.Select(SyncClient.Sha)), "every original was committed"); await ServerIdleAsync(server); await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); Assert.Equal(SyncStatus.RemoteOnly, (await laptop.EntryAsync("Comet over the ridge")).Status); var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Comet over the ridge", SyncDownloadMode.Originals, laptop.Archive.Id); await laptop.WaitAsync(download.Id); var copy = await laptop.EntryAsync("Comet over the ridge"); Assert.Equal(SyncStatus.InSync, copy.Status); Assert.Equal(SequenceOrigin.Server, copy.Local!.Origin); // The server's analysis may have added its advice meanwhile; the copy has what the server has, the user's edit included. Assert.True(SyncRules.SameSettings((await server.Database.Sequences.GetAsync(remoteId))!.SettingsJson, copy.Local.SettingsJson)); Assert.Equal(41, ProjectSettings.FromJson(copy.Local.SettingsJson!).Exposure.DeflickerWindow.Manual); var files = (await laptop.Db.Frames.ListBySequenceAsync(copy.Local.Id)) .Select(f => File.ReadAllBytes(Path.Combine(laptop.Archive.RootPath, f.RelativePath))).ToList(); Assert.True(files.Select(SyncClient.Sha).ToHashSet().SetEquals(photos.Select(SyncClient.Sha)), "byte-identical originals"); await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation); Assert.Equal(SyncStatus.InSync, (await studio.EntryAsync("Comet over the ridge")).Status); } [Test] public async Task AnUploadInterruptedMidFileResumesTheServersPartialUpload() { await using var server = await StartServerAsync(); var (http, _, token) = await server.UserAsync("alberto", "editor"); await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); var photos = Enumerable.Range(10, 2).Select(SyncClient.Photo).ToList(); var local = await studio.LocalAsync("Clouds", photos); // The crashed attempt had sent the first original (which created the server sequence) and one chunk of the second. var first = await Uploads.UploadAsync(http, photos[0], "IMG_0001.JPG", sessionKey: "crashed-attempt"); var partial = await Uploads.StartAsync(http, photos[1], "IMG_0002.JPG", first.SequenceId); Assert.True(photos[1].Length > partial.ChunkSize, "the photo spans several chunks"); using (var chunk = await Uploads.PutChunkAsync(http, partial.UploadId!.Value, 0, Uploads.Chunk(photos[1], partial.ChunkSize, 0))) await chunk.ExpectAsync(HttpStatusCode.OK); var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.UploadCommit, TargetIds = [local.Id], TargetLabel = "Clouds", ParametersJson = $$"""{"server":"{{studio.Server}}"}""", Checkpoint = first.SequenceId.ToString(), State = JobState.Queued, Created = DateTimeOffset.UtcNow, }; await studio.Db.Jobs.EnqueueAsync(job, context.Cancellation); studio.Engine.Wake(); await studio.WaitAsync(job.Id); var resumed = Assert.NotNull(await server.Database.Uploads.GetAsync(partial.UploadId!.Value)); Assert.Equal(Chronos.Data.Repositories.UploadState.Committed, resumed.State); Assert.Equal(2, (await server.Database.Frames.ListBySequenceAsync(first.SequenceId)).Count); // Every original is on both sides (the server's analysis may still be editing the settings). var link = (await studio.EntryAsync("Clouds")).Link!; Assert.Equal((first.SequenceId, 0, 0), (link.RemoteId, link.MissingRemote, link.MissingLocal)); } [Test] public async Task ADownloadResumesPartialFilesAndRefusesDamagedOnes() { await using var server = await StartServerAsync(); var (_, _, token) = await server.UserAsync("alberto", "editor"); await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop"); var photos = Enumerable.Range(20, 4).Select(SyncClient.Photo).ToList(); var local = await studio.LocalAsync("Aurora", photos); await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); var remoteId = (await studio.EntryAsync("Aurora")).Link!.RemoteId; // An earlier attempt left half of one original, a damaged half of another, a third complete (stopped while it was // being hashed) and a fourth longer than the file: the server cannot resume either of the last two (416). var staging = Path.Combine(laptop.Archive.RootPath, ".chronos", "staging", "sync-" + remoteId.ToString("N")); Directory.CreateDirectory(staging); await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[0]) + ".jpg.part"), photos[0][..(photos[0].Length / 2)]); await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[1]) + ".jpg.part"), new byte[photos[1].Length / 2]); await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[2]) + ".jpg.part"), photos[2]); await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[3]) + ".jpg.part"), [.. photos[3], .. new byte[64]]); await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Aurora", SyncDownloadMode.Originals, laptop.Archive.Id); await laptop.WaitAsync(download.Id); var copy = Assert.NotNull((await laptop.EntryAsync("Aurora")).Local); var hashes = (await laptop.Db.Frames.ListBySequenceAsync(copy.Id)) .Select(f => SyncClient.Sha(File.ReadAllBytes(Path.Combine(laptop.Archive.RootPath, f.RelativePath)))).ToHashSet(); Assert.True(hashes.SetEquals(photos.Select(SyncClient.Sha)), "every original arrived intact"); Assert.False(Directory.Exists(staging), "staging cleaned up"); } [Test] public async Task ServerEventsUpdateThisPcWithoutWaitingForTheNextComparison() { await using var server = await StartServerAsync(); var (http, _, token) = await server.UserAsync("alberto", "editor"); await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); var local = await studio.LocalAsync("Harbour sunset", [SyncClient.Photo(30)]); await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); var remoteId = (await studio.EntryAsync("Harbour sunset")).Link!.RemoteId; await ServerIdleAsync(server); await using var agent = new SyncAgent(studio.Sync, studio.Remote, () => new SyncPolicy(), studio.Engine, new SyncAgentOptions { RefreshInterval = TimeSpan.FromHours(1), EventDelay = TimeSpan.FromMilliseconds(50) }); agent.Start(); var hub = server.Services.GetRequiredService(); await UntilAsync(() => Task.FromResult(agent.Snapshot is not null && hub.SubscriberCount > 0), "the agent's first comparison and event socket"); // Someone edits the project on the server (web or another PC). The comparison its event starts writes the // settings on this PC before the link that records the agreement: wait for the comparison, not for the settings. var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); studio.Sync.Changed += _ => compared.TrySetResult(); using (var patch = await http.PatchAsJsonAsync($"/api/v1/sequences/{remoteId}", new UpdateSequenceRequest(null, null, null, null, Window33, null), ProtocolJson.Default.UpdateSequenceRequest)) await patch.ExpectAsync(HttpStatusCode.OK); await SignalAsync(compared.Task, "the comparison of the edit"); Assert.Equal(33, ProjectSettings.FromJson((await studio.Db.Sequences.GetAsync(local.Id))!.SettingsJson ?? "{}").Exposure.DeflickerWindow.Manual, "the edit reached this PC"); Assert.Equal(SyncStatus.InSync, (await studio.EntryAsync("Harbour sunset")).Status); } [Test] public async Task ARemoteSequenceOpensWithProxiesOnly() { await using var server = await StartServerAsync(); var (_, _, token) = await server.UserAsync("alberto", "editor"); await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop"); var photos = Enumerable.Range(40, 2).Select(SyncClient.Photo).ToList(); var local = await studio.LocalAsync("Milan skyline dusk", photos, Window41); await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); var remoteId = (await studio.EntryAsync("Milan skyline dusk")).Link!.RemoteId; await ServerIdleAsync(server); Assert.True(((await server.Database.Sequences.GetAsync(remoteId))!.Status & SequenceStatus.ThumbsBuilt) != 0, "thumbnails on the server"); // A remote card without a hero frame shows its first frame, read as one row instead of the whole list. ISyncRemote remote = laptop.Remote; Assert.Equal((await remote.ListFramesAsync(remoteId, context.Cancellation))[0].Id, (await remote.FirstFrameAsync(remoteId, context.Cancellation))?.Id); await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); var job = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Milan skyline dusk", SyncDownloadMode.Proxies, null); await laptop.WaitAsync(job.Id); Assert.Equal(SyncStatus.ProxiesOnly, (await laptop.EntryAsync("Milan skyline dusk")).Status); Assert.Empty((await laptop.Db.Sequences.ListAsync(new SequenceQuery())).Items); var opened = Assert.NotNull(laptop.Sync.OpenRemote(laptop.Server, remoteId)); Assert.Equal(2, opened.CachedCount); Assert.Equal(41, ProjectSettings.FromJson(opened.Sequence.SettingsJson!).Exposure.DeflickerWindow.Manual); var source = opened.OpenFrames(DecoderFactory.CreateServer()); await using (source as IAsyncDisposable) { using var frame = await source.GetFrameAsync(0, 0, DecodePurpose.Render, context.Cancellation); Assert.True(frame.Width > 0 && frame.Height > 0, "a proxy decodes"); } } [Test] public async Task RendersTravelOnRequestWithTheirIdAndAreVerified() { await using var server = await StartServerAsync(); var (_, _, token) = await server.UserAsync("alberto", "editor"); await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop"); var photos = Enumerable.Range(50, 2).Select(SyncClient.Photo).ToList(); var local = await studio.LocalAsync("Star trails", photos); await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); var remoteId = (await studio.EntryAsync("Star trails")).Link!.RemoteId; await ServerIdleAsync(server); // A render made on the studio PC, larger than one chunk, is sent on request and keeps its id. var output = Enumerable.Range(0, 200_000).Select(i => (byte)(i * 31)).ToArray(); var render = await studio.RenderAsync(local, "Star trails 4K.mp4", output); await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation); Assert.Equal(1, (await studio.EntryAsync("Star trails")).Link!.ExportsToSend); await studio.WaitAsync((await studio.Sync.EnqueueSendExportsAsync(studio.Engine, studio.Server, local)).Id); var filed = Assert.NotNull(await server.Database.Renders.GetAsync(render.Id)); Assert.Equal(remoteId, filed.SequenceId); Assert.SequenceEqual(output, await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, filed.OutputPath))); Assert.Equal(0, (await studio.EntryAsync("Star trails")).Link!.ExportsToSend); Assert.Null(await studio.Remote.UploadRenderAsync(remoteId, studio.RenderFile(render), new RenderUploadRequest(render.Id, "Star trails 4K.mp4", output.Length, SyncClient.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, "studio"), context.Cancellation)); // A damaged transfer is refused at completion. var bad = new RenderUploadRequest(Guid.CreateVersion7(), "Damaged.mp4", output.Length, SyncClient.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null); var damaged = Path.Combine(context.CreateTempDirectory("damaged"), "Damaged.mp4"); await File.WriteAllBytesAsync(damaged, [.. output.Select(b => (byte)~b)]); var refused = await Assert.ThrowsAsync(() => studio.Remote.UploadRenderAsync(remoteId, damaged, bad, context.Cancellation)); Assert.Equal(HttpStatusCode.UnprocessableEntity, refused.Status); Assert.Null(await server.Database.Renders.GetAsync(bad.RenderId)); // Another PC with the originals gets the render into its own exports, byte-identical. await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Star trails", SyncDownloadMode.Originals, laptop.Archive.Id)).Id); Assert.Equal(1, (await laptop.EntryAsync("Star trails")).Link!.ExportsToGet); await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Star trails", SyncDownloadMode.Exports, null)).Id); var copy = (await laptop.EntryAsync("Star trails")).Local!; var fetched = Assert.Contains(await laptop.Db.Renders.ListBySequenceAsync(copy.Id), r => r.Id == render.Id); Assert.SequenceEqual(output, await File.ReadAllBytesAsync(Path.Combine(laptop.Archive.RootPath, fetched.OutputPath))); Assert.Equal(0, (await laptop.EntryAsync("Star trails")).Link!.ExportsToGet); } [Test] public async Task PositionsEditedOnTheServerStayOnTheirPhotosWhenAnotherPcDownloadsTheInsertedFrames() { await using var server = await StartServerAsync(); var (http, _, token) = await server.UserAsync("alberto", "editor"); await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop"); var photos = Enumerable.Range(60, 5).Select(SyncClient.Photo).ToList(); // Both PCs hold the last four photos, the third of them excluded. var local = await studio.LocalAsync("Harbour at dawn", photos[1..], new ProjectSettings { ExcludedFrames = [2] }.ToJson()); await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); var remoteId = (await studio.EntryAsync("Harbour at dawn")).Link!.RemoteId; await ServerIdleAsync(server); await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Harbour at dawn", SyncDownloadMode.Originals, laptop.Archive.Id)).Id); var copy = (await laptop.EntryAsync("Harbour at dawn")).Local!; // The studio files an earlier photo and uploads it: the server inserts it first and moves its own positions. await studio.AddPhotoAsync(local.Id, photos[0], new DateTime(2026, 9, 28, 17, 59, 50)); await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); await ServerIdleAsync(server); Assert.SequenceEqual([SyncClient.Sha(photos[3])], (await ServerNamedAsync(server, remoteId)).Excluded); // An editor on the server excludes the new photo too, by the server's frames, and changes the deflicker window. var row = (await server.Database.Sequences.GetAsync(remoteId))!; var current = ProjectSettings.FromJson(row.SettingsJson!); var edit = current with { ExcludedFrames = [0, .. current.ExcludedFrames], Exposure = current.Exposure with { DeflickerWindow = new Auto().WithManual(33) }, }; using (var patch = await http.PatchAsJsonAsync($"/api/v1/sequences/{remoteId}", new UpdateSequenceRequest(null, null, null, null, edit.ToJson(), row.SettingsVersion), ProtocolJson.Default.UpdateSequenceRequest)) await patch.ExpectAsync(HttpStatusCode.OK); // The laptop takes the edit while it still lacks the new photo, then downloads it. await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); var taken = await LocalNamedAsync(laptop, copy.Id); Assert.Equal(33, taken.Project.Exposure.DeflickerWindow.Manual); Assert.SequenceEqual([SyncClient.Sha(photos[3])], taken.Excluded); Assert.True(taken.Project.PendingFrames is [{ Excluded: true } waiting] && waiting.Sha256 == SyncClient.Sha(photos[0]), "the new photo's exclusion waits for it"); await laptop.WaitAsync((await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Harbour at dawn", SyncDownloadMode.Originals, null)).Id); var onServer = await ServerNamedAsync(server, remoteId); var here = await LocalNamedAsync(laptop, copy.Id); Assert.SequenceEqual([SyncClient.Sha(photos[0]), SyncClient.Sha(photos[3])], here.Excluded); Assert.SequenceEqual(onServer.Excluded, here.Excluded); Assert.Equal(onServer.Hero, here.Hero); Assert.Null(here.Project.PendingFrames); await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation); Assert.Equal(SyncStatus.InSync, (await laptop.EntryAsync("Harbour at dawn")).Status); } [Test] public async Task PositionsSentBeforeTheirPhotosStayOnThemWhenTheUploadArrives() { await using var server = await StartServerAsync(); var (_, _, token) = await server.UserAsync("alberto", "editor"); await using var studio = await SyncClient.StartAsync(context, server, token, "studio"); var photos = Enumerable.Range(70, 5).Select(SyncClient.Photo).ToList(); var local = await studio.LocalAsync("Pier", photos[1..]); await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); var remoteId = (await studio.EntryAsync("Pier")).Link!.RemoteId; await ServerIdleAsync(server); await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation); // This PC files an earlier photo and, before uploading it, excludes it and a later one and picks the hero. await studio.AddPhotoAsync(local.Id, photos[0], new DateTime(2026, 9, 28, 17, 59, 50)); var project = ProjectSettings.FromJson((await studio.Db.Sequences.GetAsync(local.Id))!.SettingsJson ?? "{}"); await studio.EditSettingsAsync(local.Id, (project with { ExcludedFrames = [0, 3], HeroFrame = 1 }).ToJson()); await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation); var sent = await ServerNamedAsync(server, remoteId); Assert.SequenceEqual([SyncClient.Sha(photos[3])], sent.Excluded); Assert.Equal(SyncClient.Sha(photos[1]), sent.Hero); Assert.True(sent.Project.PendingFrames is [{ Excluded: true } waiting] && waiting.Sha256 == SyncClient.Sha(photos[0]), "the exclusion waits for its photo"); await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id); await ServerIdleAsync(server); var arrived = await ServerNamedAsync(server, remoteId); Assert.SequenceEqual([SyncClient.Sha(photos[0]), SyncClient.Sha(photos[3])], arrived.Excluded); Assert.Equal(SyncClient.Sha(photos[1]), arrived.Hero); Assert.Null(arrived.Project.PendingFrames); var heroFrame = (await server.Database.Sequences.GetAsync(remoteId))!.HeroFrameId; Assert.Equal(SyncClient.Sha(photos[1]), (await server.Database.Frames.ListBySequenceAsync(remoteId)).Single(f => f.Id == heroFrame).Sha256); var here = await LocalNamedAsync(studio, local.Id); Assert.SequenceEqual(arrived.Excluded, here.Excluded); Assert.Equal(arrived.Hero, here.Hero); await studio.Sync.RefreshAsync(studio.Remote, new SyncPolicy(), context.Cancellation); Assert.Equal(SyncStatus.InSync, (await studio.EntryAsync("Pier")).Status); } /// The server's project and the photos (SHA-256) it excludes and shows as hero. private static async Task<(ProjectSettings Project, string?[] Excluded, string? Hero)> ServerNamedAsync(TestServer server, Guid remoteId) { var project = ProjectSettings.FromJson((await server.Database.Sequences.GetAsync(remoteId))!.SettingsJson ?? "{}"); var frames = await server.Database.Frames.ListBySequenceAsync(remoteId); return (project, [.. project.ExcludedFrames.Select(i => frames[i].Sha256)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null); } /// A PC's project and the photos (SHA-256) it excludes and shows as hero. private static async Task<(ProjectSettings Project, string?[] Excluded, string? Hero)> LocalNamedAsync(SyncClient client, Guid localId) { var project = ProjectSettings.FromJson((await client.Db.Sequences.GetAsync(localId))!.SettingsJson ?? "{}"); var frames = await client.Db.Frames.ListBySequenceAsync(localId); return (project, [.. project.ExcludedFrames.Select(i => frames[i].Sha256)], project.HeroFrame is { } hero ? frames[hero].Sha256 : null); } private Task StartServerAsync() => TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register); /// Waits until the server's engines finished what the commits queued (thumbnails, proxies, analysis, which may edit settings). private Task ServerIdleAsync(TestServer server) => UntilAsync(async () => await server.Database.Jobs.CountAsync(DateTimeOffset.UtcNow) is { Queued: 0, Running: 0 }, "the server's jobs"); private async Task UntilAsync(Func> condition, string what) { var until = DateTime.UtcNow + Deadline; while (!await condition()) { if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what); await Task.Delay(50, context.Cancellation); } } private async Task SignalAsync(Task signal, string what) { try { await signal.WaitAsync(Deadline, context.Cancellation); } catch (TimeoutException) { Assert.Fail("Timed out waiting for " + what); } } }