fix(render-node): a dropped proxy download starts over instead of resuming
Originals are resumed from their partial file because their SHA-256 checks the result. Proxies have no hash on the server and can be rebuilt between two attempts, so appending to what a dropped attempt left could mix two versions; a retry now fetches the proxy whole. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
2 files changed
+42
-13
No files matched your search
@@ -308,8 +308,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it); the original
|
||||
/// when the server has none for the frame.
|
||||
/// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it, and a retry
|
||||
/// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server
|
||||
/// has none for the frame.
|
||||
/// </summary>
|
||||
private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -317,7 +318,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
try
|
||||
{
|
||||
await FetchAsync(frame, "proxy", path, cancellationToken).ConfigureAwait(false);
|
||||
await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false);
|
||||
return path;
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
@@ -343,7 +344,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
// A part as long as the file was complete when the last attempt stopped: only its hash is left to check.
|
||||
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, cancellationToken).ConfigureAwait(false);
|
||||
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, resume: true, cancellationToken).ConfigureAwait(false);
|
||||
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
|
||||
{
|
||||
File.Move(part, path, overwrite: true);
|
||||
@@ -355,17 +356,18 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fetches a frame resource into <paramref name="path"/>, appending to what an earlier attempt left there. A dropped
|
||||
/// connection or a busy server is retried, resuming where the file ends (up to <see cref="RenderNodeOptions.DownloadRetries"/> times,
|
||||
/// the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>).
|
||||
/// Fetches a frame resource into <paramref name="path"/>. A dropped connection or a busy server is retried (up to
|
||||
/// <see cref="RenderNodeOptions.DownloadRetries"/> times, the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>);
|
||||
/// with <paramref name="resume"/> (a download its hash verifies) each attempt appends to what the last one left,
|
||||
/// otherwise it starts over.
|
||||
/// </summary>
|
||||
private async Task FetchAsync(ManifestFrame frame, string kind, string path, CancellationToken cancellationToken)
|
||||
private async Task FetchAsync(ManifestFrame frame, string kind, string path, bool resume, CancellationToken cancellationToken)
|
||||
{
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var file = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
|
||||
await using var file = new FileStream(path, resume ? FileMode.OpenOrCreate : FileMode.Create, FileAccess.ReadWrite, FileShare.None);
|
||||
await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -160,7 +160,7 @@ public sealed class RenderNodeClientTests(TestContext context)
|
||||
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
||||
var job = await CreateRenderAsync(server, editor, sequenceId);
|
||||
|
||||
var cut = new CutFirstOriginal(new SocketsHttpHandler());
|
||||
var cut = new CutFirst("original", new SocketsHttpHandler());
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
|
||||
await using var node = Node(client, context.CreateTempDirectory("node"));
|
||||
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
||||
@@ -170,6 +170,33 @@ public sealed class RenderNodeClientTests(TestContext context)
|
||||
Assert.Equal(1, cut.Resumed);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADroppedProxyIsFetchedWholeAgain()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
||||
var photos = Photos();
|
||||
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, photos)))!;
|
||||
// The archive has proxies of every frame (the test locator serves <folder>/proxies/NNNN.jpg).
|
||||
var proxies = Path.Combine(server.Options.ArchivePath, sequence.Folder, "proxies");
|
||||
Directory.CreateDirectory(proxies);
|
||||
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
|
||||
await File.WriteAllBytesAsync(Path.Combine(proxies, $"{frame.Index:D4}.jpg"), photos[frame.Index]);
|
||||
await server.Database.Sequences.UpsertAsync(sequence with { Status = sequence.Status | SequenceStatus.ProxiesBuilt });
|
||||
var job = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, Mjpeg, 96, 64, useProxies: true).ParametersJson);
|
||||
|
||||
var cut = new CutFirst("proxy", new SocketsHttpHandler());
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
|
||||
var cache = context.CreateTempDirectory("node");
|
||||
await using var node = Node(client, cache);
|
||||
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
||||
|
||||
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
||||
// Nothing verifies a proxy against a hash, so a retry never appends to what the dropped attempt left.
|
||||
Assert.Equal((1, 0), (cut.Cuts, cut.Resumed));
|
||||
Assert.False(Directory.Exists(Path.Combine(cache, "originals")), "no original needed");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANodeLeavesARenderTheServerStartedToTheServer()
|
||||
{
|
||||
@@ -260,8 +287,8 @@ public sealed class RenderNodeClientTests(TestContext context)
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Drops the connection halfway through the first original it serves; counts the downloads resumed with a range.</summary>
|
||||
private sealed class CutFirstOriginal(HttpMessageHandler inner) : DelegatingHandler(inner)
|
||||
/// <summary>Drops the connection halfway through the first resource of <paramref name="kind"/> it serves; counts the downloads resumed with a range.</summary>
|
||||
private sealed class CutFirst(string kind, HttpMessageHandler inner) : DelegatingHandler(inner)
|
||||
{
|
||||
private int _cuts;
|
||||
private int _resumed;
|
||||
@@ -273,7 +300,7 @@ public sealed class RenderNodeClientTests(TestContext context)
|
||||
{
|
||||
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
|
||||
var response = await base.SendAsync(request, cancellationToken);
|
||||
if (!request.RequestUri!.AbsolutePath.EndsWith("/original", StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
|
||||
if (!request.RequestUri!.AbsolutePath.EndsWith("/" + kind, StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
|
||||
return response;
|
||||
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
|
||||
var content = new StreamContent(new CutStream(body, body.Length / 2));
|
||||
|
||||
Reference in new issue
Block a user