diff --git a/src/Titano.Data/Processing/RenderNodeClient.cs b/src/Titano.Data/Processing/RenderNodeClient.cs index db13ffc1..aa53c5df 100644 --- a/src/Titano.Data/Processing/RenderNodeClient.cs +++ b/src/Titano.Data/Processing/RenderNodeClient.cs @@ -308,8 +308,9 @@ public sealed class RenderNodeClient : IAsyncDisposable } /// - /// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it); the original - /// when the server has none for the frame. + /// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it, and a retry + /// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server + /// has none for the frame. /// private async Task DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) { @@ -317,7 +318,7 @@ public sealed class RenderNodeClient : IAsyncDisposable Directory.CreateDirectory(Path.GetDirectoryName(path)!); try { - await FetchAsync(frame, "proxy", path, cancellationToken).ConfigureAwait(false); + await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false); return path; } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) @@ -343,7 +344,7 @@ public sealed class RenderNodeClient : IAsyncDisposable for (var attempt = 1; ; attempt++) { // A part as long as the file was complete when the last attempt stopped: only its hash is left to check. - if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, cancellationToken).ConfigureAwait(false); + if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, resume: true, cancellationToken).ConfigureAwait(false); if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha) { File.Move(part, path, overwrite: true); @@ -355,17 +356,18 @@ public sealed class RenderNodeClient : IAsyncDisposable } /// - /// Fetches a frame resource into , appending to what an earlier attempt left there. A dropped - /// connection or a busy server is retried, resuming where the file ends (up to times, - /// the wait doubling from ). + /// Fetches a frame resource into . A dropped connection or a busy server is retried (up to + /// times, the wait doubling from ); + /// with (a download its hash verifies) each attempt appends to what the last one left, + /// otherwise it starts over. /// - private async Task FetchAsync(ManifestFrame frame, string kind, string path, CancellationToken cancellationToken) + private async Task FetchAsync(ManifestFrame frame, string kind, string path, bool resume, CancellationToken cancellationToken) { for (var attempt = 0; ; attempt++) { try { - await using var file = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + await using var file = new FileStream(path, resume ? FileMode.OpenOrCreate : FileMode.Create, FileAccess.ReadWrite, FileShare.None); await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false); return; } diff --git a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs index dea231c2..b4887ac4 100644 --- a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs +++ b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs @@ -160,7 +160,7 @@ public sealed class RenderNodeClientTests(TestContext context) var sequenceId = await UploadSequenceAsync(editor, Photos()); var job = await CreateRenderAsync(server, editor, sequenceId); - var cut = new CutFirstOriginal(new SocketsHttpHandler()); + var cut = new CutFirst("original", new SocketsHttpHandler()); using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token }; await using var node = Node(client, context.CreateTempDirectory("node")); Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); @@ -170,6 +170,33 @@ public sealed class RenderNodeClientTests(TestContext context) Assert.Equal(1, cut.Resumed); } + [Test] + public async Task ADroppedProxyIsFetchedWholeAgain() + { + await using var server = await TestServer.StartAsync(context); + var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); + var photos = Photos(); + var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, photos)))!; + // The archive has proxies of every frame (the test locator serves /proxies/NNNN.jpg). + var proxies = Path.Combine(server.Options.ArchivePath, sequence.Folder, "proxies"); + Directory.CreateDirectory(proxies); + foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id)) + await File.WriteAllBytesAsync(Path.Combine(proxies, $"{frame.Index:D4}.jpg"), photos[frame.Index]); + await server.Database.Sequences.UpsertAsync(sequence with { Status = sequence.Status | SequenceStatus.ProxiesBuilt }); + var job = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, Mjpeg, 96, 64, useProxies: true).ParametersJson); + + var cut = new CutFirst("proxy", new SocketsHttpHandler()); + using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token }; + var cache = context.CreateTempDirectory("node"); + await using var node = Node(client, cache); + Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); + + Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State); + // Nothing verifies a proxy against a hash, so a retry never appends to what the dropped attempt left. + Assert.Equal((1, 0), (cut.Cuts, cut.Resumed)); + Assert.False(Directory.Exists(Path.Combine(cache, "originals")), "no original needed"); + } + [Test] public async Task ANodeLeavesARenderTheServerStartedToTheServer() { @@ -260,8 +287,8 @@ public sealed class RenderNodeClientTests(TestContext context) } } - /// Drops the connection halfway through the first original it serves; counts the downloads resumed with a range. - private sealed class CutFirstOriginal(HttpMessageHandler inner) : DelegatingHandler(inner) + /// Drops the connection halfway through the first resource of it serves; counts the downloads resumed with a range. + private sealed class CutFirst(string kind, HttpMessageHandler inner) : DelegatingHandler(inner) { private int _cuts; private int _resumed; @@ -273,7 +300,7 @@ public sealed class RenderNodeClientTests(TestContext context) { if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed); var response = await base.SendAsync(request, cancellationToken); - if (!request.RequestUri!.AbsolutePath.EndsWith("/original", StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0) + if (!request.RequestUri!.AbsolutePath.EndsWith("/" + kind, StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0) return response; var body = await response.Content.ReadAsByteArrayAsync(cancellationToken); var content = new StreamContent(new CutStream(body, body.Length / 2));