fix(render-node): a dropped proxy download starts over instead of resuming

Originals are resumed from their partial file because their SHA-256 checks
the result. Proxies have no hash on the server and can be rebuilt between
two attempts, so appending to what a dropped attempt left could mix two
versions; a retry now fetches the proxy whole.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-03 09:42:19 +00:00
1 parent 4007293833
commit 5cac49efea
2 files changed
+42 -13

No files matched your search

+11 -9
View File
@@ -308,8 +308,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
} }
/// <summary> /// <summary>
/// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it); the original /// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it, and a retry
/// when the server has none for the frame. /// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server
/// has none for the frame.
/// </summary> /// </summary>
private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
{ {
@@ -317,7 +318,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
Directory.CreateDirectory(Path.GetDirectoryName(path)!); Directory.CreateDirectory(Path.GetDirectoryName(path)!);
try try
{ {
await FetchAsync(frame, "proxy", path, cancellationToken).ConfigureAwait(false); await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false);
return path; return path;
} }
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
@@ -343,7 +344,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
for (var attempt = 1; ; attempt++) for (var attempt = 1; ; attempt++)
{ {
// A part as long as the file was complete when the last attempt stopped: only its hash is left to check. // A part as long as the file was complete when the last attempt stopped: only its hash is left to check.
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, cancellationToken).ConfigureAwait(false); if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, resume: true, cancellationToken).ConfigureAwait(false);
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha) if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
{ {
File.Move(part, path, overwrite: true); File.Move(part, path, overwrite: true);
@@ -355,17 +356,18 @@ public sealed class RenderNodeClient : IAsyncDisposable
} }
/// <summary> /// <summary>
/// Fetches a frame resource into <paramref name="path"/>, appending to what an earlier attempt left there. A dropped /// Fetches a frame resource into <paramref name="path"/>. A dropped connection or a busy server is retried (up to
/// connection or a busy server is retried, resuming where the file ends (up to <see cref="RenderNodeOptions.DownloadRetries"/> times, /// <see cref="RenderNodeOptions.DownloadRetries"/> times, the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>);
/// the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>). /// with <paramref name="resume"/> (a download its hash verifies) each attempt appends to what the last one left,
/// otherwise it starts over.
/// </summary> /// </summary>
private async Task FetchAsync(ManifestFrame frame, string kind, string path, CancellationToken cancellationToken) private async Task FetchAsync(ManifestFrame frame, string kind, string path, bool resume, CancellationToken cancellationToken)
{ {
for (var attempt = 0; ; attempt++) for (var attempt = 0; ; attempt++)
{ {
try try
{ {
await using var file = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); await using var file = new FileStream(path, resume ? FileMode.OpenOrCreate : FileMode.Create, FileAccess.ReadWrite, FileShare.None);
await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false); await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false);
return; return;
} }
@@ -160,7 +160,7 @@ public sealed class RenderNodeClientTests(TestContext context)
var sequenceId = await UploadSequenceAsync(editor, Photos()); var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId); var job = await CreateRenderAsync(server, editor, sequenceId);
var cut = new CutFirstOriginal(new SocketsHttpHandler()); var cut = new CutFirst("original", new SocketsHttpHandler());
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token }; using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node")); await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
@@ -170,6 +170,33 @@ public sealed class RenderNodeClientTests(TestContext context)
Assert.Equal(1, cut.Resumed); Assert.Equal(1, cut.Resumed);
} }
[Test]
public async Task ADroppedProxyIsFetchedWholeAgain()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var photos = Photos();
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, photos)))!;
// The archive has proxies of every frame (the test locator serves <folder>/proxies/NNNN.jpg).
var proxies = Path.Combine(server.Options.ArchivePath, sequence.Folder, "proxies");
Directory.CreateDirectory(proxies);
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
await File.WriteAllBytesAsync(Path.Combine(proxies, $"{frame.Index:D4}.jpg"), photos[frame.Index]);
await server.Database.Sequences.UpsertAsync(sequence with { Status = sequence.Status | SequenceStatus.ProxiesBuilt });
var job = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, Mjpeg, 96, 64, useProxies: true).ParametersJson);
var cut = new CutFirst("proxy", new SocketsHttpHandler());
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
var cache = context.CreateTempDirectory("node");
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
// Nothing verifies a proxy against a hash, so a retry never appends to what the dropped attempt left.
Assert.Equal((1, 0), (cut.Cuts, cut.Resumed));
Assert.False(Directory.Exists(Path.Combine(cache, "originals")), "no original needed");
}
[Test] [Test]
public async Task ANodeLeavesARenderTheServerStartedToTheServer() public async Task ANodeLeavesARenderTheServerStartedToTheServer()
{ {
@@ -260,8 +287,8 @@ public sealed class RenderNodeClientTests(TestContext context)
} }
} }
/// <summary>Drops the connection halfway through the first original it serves; counts the downloads resumed with a range.</summary> /// <summary>Drops the connection halfway through the first resource of <paramref name="kind"/> it serves; counts the downloads resumed with a range.</summary>
private sealed class CutFirstOriginal(HttpMessageHandler inner) : DelegatingHandler(inner) private sealed class CutFirst(string kind, HttpMessageHandler inner) : DelegatingHandler(inner)
{ {
private int _cuts; private int _cuts;
private int _resumed; private int _resumed;
@@ -273,7 +300,7 @@ public sealed class RenderNodeClientTests(TestContext context)
{ {
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed); if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
var response = await base.SendAsync(request, cancellationToken); var response = await base.SendAsync(request, cancellationToken);
if (!request.RequestUri!.AbsolutePath.EndsWith("/original", StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0) if (!request.RequestUri!.AbsolutePath.EndsWith("/" + kind, StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
return response; return response;
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken); var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
var content = new StreamContent(new CutStream(body, body.Length / 2)); var content = new StreamContent(new CutStream(body, body.Length / 2));