fix(render-node): a dropped proxy download starts over instead of resuming

Originals are resumed from their partial file because their SHA-256 checks
the result. Proxies have no hash on the server and can be rebuilt between
two attempts, so appending to what a dropped attempt left could mix two
versions; a retry now fetches the proxy whole.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-03 09:42:19 +00:00
1 parent 4007293833
commit 5cac49efea
2 files changed
+42 -13

No files matched your search

+11 -9
View File
@@ -308,8 +308,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
}
/// <summary>
/// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it); the original
/// when the server has none for the frame.
/// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it, and a retry
/// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server
/// has none for the frame.
/// </summary>
private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
{
@@ -317,7 +318,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
try
{
await FetchAsync(frame, "proxy", path, cancellationToken).ConfigureAwait(false);
await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false);
return path;
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
@@ -343,7 +344,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
for (var attempt = 1; ; attempt++)
{
// A part as long as the file was complete when the last attempt stopped: only its hash is left to check.
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, cancellationToken).ConfigureAwait(false);
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, resume: true, cancellationToken).ConfigureAwait(false);
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
{
File.Move(part, path, overwrite: true);
@@ -355,17 +356,18 @@ public sealed class RenderNodeClient : IAsyncDisposable
}
/// <summary>
/// Fetches a frame resource into <paramref name="path"/>, appending to what an earlier attempt left there. A dropped
/// connection or a busy server is retried, resuming where the file ends (up to <see cref="RenderNodeOptions.DownloadRetries"/> times,
/// the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>).
/// Fetches a frame resource into <paramref name="path"/>. A dropped connection or a busy server is retried (up to
/// <see cref="RenderNodeOptions.DownloadRetries"/> times, the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>);
/// with <paramref name="resume"/> (a download its hash verifies) each attempt appends to what the last one left,
/// otherwise it starts over.
/// </summary>
private async Task FetchAsync(ManifestFrame frame, string kind, string path, CancellationToken cancellationToken)
private async Task FetchAsync(ManifestFrame frame, string kind, string path, bool resume, CancellationToken cancellationToken)
{
for (var attempt = 0; ; attempt++)
{
try
{
await using var file = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
await using var file = new FileStream(path, resume ? FileMode.OpenOrCreate : FileMode.Create, FileAccess.ReadWrite, FileShare.None);
await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false);
return;
}
@@ -160,7 +160,7 @@ public sealed class RenderNodeClientTests(TestContext context)
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
var cut = new CutFirstOriginal(new SocketsHttpHandler());
var cut = new CutFirst("original", new SocketsHttpHandler());
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
@@ -170,6 +170,33 @@ public sealed class RenderNodeClientTests(TestContext context)
Assert.Equal(1, cut.Resumed);
}
[Test]
public async Task ADroppedProxyIsFetchedWholeAgain()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var photos = Photos();
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, photos)))!;
// The archive has proxies of every frame (the test locator serves <folder>/proxies/NNNN.jpg).
var proxies = Path.Combine(server.Options.ArchivePath, sequence.Folder, "proxies");
Directory.CreateDirectory(proxies);
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
await File.WriteAllBytesAsync(Path.Combine(proxies, $"{frame.Index:D4}.jpg"), photos[frame.Index]);
await server.Database.Sequences.UpsertAsync(sequence with { Status = sequence.Status | SequenceStatus.ProxiesBuilt });
var job = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, Mjpeg, 96, 64, useProxies: true).ParametersJson);
var cut = new CutFirst("proxy", new SocketsHttpHandler());
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
var cache = context.CreateTempDirectory("node");
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
// Nothing verifies a proxy against a hash, so a retry never appends to what the dropped attempt left.
Assert.Equal((1, 0), (cut.Cuts, cut.Resumed));
Assert.False(Directory.Exists(Path.Combine(cache, "originals")), "no original needed");
}
[Test]
public async Task ANodeLeavesARenderTheServerStartedToTheServer()
{
@@ -260,8 +287,8 @@ public sealed class RenderNodeClientTests(TestContext context)
}
}
/// <summary>Drops the connection halfway through the first original it serves; counts the downloads resumed with a range.</summary>
private sealed class CutFirstOriginal(HttpMessageHandler inner) : DelegatingHandler(inner)
/// <summary>Drops the connection halfway through the first resource of <paramref name="kind"/> it serves; counts the downloads resumed with a range.</summary>
private sealed class CutFirst(string kind, HttpMessageHandler inner) : DelegatingHandler(inner)
{
private int _cuts;
private int _resumed;
@@ -273,7 +300,7 @@ public sealed class RenderNodeClientTests(TestContext context)
{
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
var response = await base.SendAsync(request, cancellationToken);
if (!request.RequestUri!.AbsolutePath.EndsWith("/original", StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
if (!request.RequestUri!.AbsolutePath.EndsWith("/" + kind, StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
return response;
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
var content = new StreamContent(new CutStream(body, body.Length / 2));