fix(render-node): a dropped proxy download starts over instead of resuming
Originals are resumed from their partial file because their SHA-256 checks the result. Proxies have no hash on the server and can be rebuilt between two attempts, so appending to what a dropped attempt left could mix two versions; a retry now fetches the proxy whole. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
2 files changed
+42
-13
No files matched your search
@@ -308,8 +308,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it); the original
|
/// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it, and a retry
|
||||||
/// when the server has none for the frame.
|
/// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server
|
||||||
|
/// has none for the frame.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
@@ -317,7 +318,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
|||||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await FetchAsync(frame, "proxy", path, cancellationToken).ConfigureAwait(false);
|
await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false);
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||||
@@ -343,7 +344,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
|||||||
for (var attempt = 1; ; attempt++)
|
for (var attempt = 1; ; attempt++)
|
||||||
{
|
{
|
||||||
// A part as long as the file was complete when the last attempt stopped: only its hash is left to check.
|
// A part as long as the file was complete when the last attempt stopped: only its hash is left to check.
|
||||||
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, cancellationToken).ConfigureAwait(false);
|
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, resume: true, cancellationToken).ConfigureAwait(false);
|
||||||
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
|
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
|
||||||
{
|
{
|
||||||
File.Move(part, path, overwrite: true);
|
File.Move(part, path, overwrite: true);
|
||||||
@@ -355,17 +356,18 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Fetches a frame resource into <paramref name="path"/>, appending to what an earlier attempt left there. A dropped
|
/// Fetches a frame resource into <paramref name="path"/>. A dropped connection or a busy server is retried (up to
|
||||||
/// connection or a busy server is retried, resuming where the file ends (up to <see cref="RenderNodeOptions.DownloadRetries"/> times,
|
/// <see cref="RenderNodeOptions.DownloadRetries"/> times, the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>);
|
||||||
/// the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>).
|
/// with <paramref name="resume"/> (a download its hash verifies) each attempt appends to what the last one left,
|
||||||
|
/// otherwise it starts over.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private async Task FetchAsync(ManifestFrame frame, string kind, string path, CancellationToken cancellationToken)
|
private async Task FetchAsync(ManifestFrame frame, string kind, string path, bool resume, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
for (var attempt = 0; ; attempt++)
|
for (var attempt = 0; ; attempt++)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await using var file = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
|
await using var file = new FileStream(path, resume ? FileMode.OpenOrCreate : FileMode.Create, FileAccess.ReadWrite, FileShare.None);
|
||||||
await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false);
|
await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -160,7 +160,7 @@ public sealed class RenderNodeClientTests(TestContext context)
|
|||||||
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
||||||
var job = await CreateRenderAsync(server, editor, sequenceId);
|
var job = await CreateRenderAsync(server, editor, sequenceId);
|
||||||
|
|
||||||
var cut = new CutFirstOriginal(new SocketsHttpHandler());
|
var cut = new CutFirst("original", new SocketsHttpHandler());
|
||||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
|
||||||
await using var node = Node(client, context.CreateTempDirectory("node"));
|
await using var node = Node(client, context.CreateTempDirectory("node"));
|
||||||
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
||||||
@@ -170,6 +170,33 @@ public sealed class RenderNodeClientTests(TestContext context)
|
|||||||
Assert.Equal(1, cut.Resumed);
|
Assert.Equal(1, cut.Resumed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task ADroppedProxyIsFetchedWholeAgain()
|
||||||
|
{
|
||||||
|
await using var server = await TestServer.StartAsync(context);
|
||||||
|
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
||||||
|
var photos = Photos();
|
||||||
|
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, photos)))!;
|
||||||
|
// The archive has proxies of every frame (the test locator serves <folder>/proxies/NNNN.jpg).
|
||||||
|
var proxies = Path.Combine(server.Options.ArchivePath, sequence.Folder, "proxies");
|
||||||
|
Directory.CreateDirectory(proxies);
|
||||||
|
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
|
||||||
|
await File.WriteAllBytesAsync(Path.Combine(proxies, $"{frame.Index:D4}.jpg"), photos[frame.Index]);
|
||||||
|
await server.Database.Sequences.UpsertAsync(sequence with { Status = sequence.Status | SequenceStatus.ProxiesBuilt });
|
||||||
|
var job = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, Mjpeg, 96, 64, useProxies: true).ParametersJson);
|
||||||
|
|
||||||
|
var cut = new CutFirst("proxy", new SocketsHttpHandler());
|
||||||
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
|
||||||
|
var cache = context.CreateTempDirectory("node");
|
||||||
|
await using var node = Node(client, cache);
|
||||||
|
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
||||||
|
|
||||||
|
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
||||||
|
// Nothing verifies a proxy against a hash, so a retry never appends to what the dropped attempt left.
|
||||||
|
Assert.Equal((1, 0), (cut.Cuts, cut.Resumed));
|
||||||
|
Assert.False(Directory.Exists(Path.Combine(cache, "originals")), "no original needed");
|
||||||
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
public async Task ANodeLeavesARenderTheServerStartedToTheServer()
|
public async Task ANodeLeavesARenderTheServerStartedToTheServer()
|
||||||
{
|
{
|
||||||
@@ -260,8 +287,8 @@ public sealed class RenderNodeClientTests(TestContext context)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Drops the connection halfway through the first original it serves; counts the downloads resumed with a range.</summary>
|
/// <summary>Drops the connection halfway through the first resource of <paramref name="kind"/> it serves; counts the downloads resumed with a range.</summary>
|
||||||
private sealed class CutFirstOriginal(HttpMessageHandler inner) : DelegatingHandler(inner)
|
private sealed class CutFirst(string kind, HttpMessageHandler inner) : DelegatingHandler(inner)
|
||||||
{
|
{
|
||||||
private int _cuts;
|
private int _cuts;
|
||||||
private int _resumed;
|
private int _resumed;
|
||||||
@@ -273,7 +300,7 @@ public sealed class RenderNodeClientTests(TestContext context)
|
|||||||
{
|
{
|
||||||
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
|
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
|
||||||
var response = await base.SendAsync(request, cancellationToken);
|
var response = await base.SendAsync(request, cancellationToken);
|
||||||
if (!request.RequestUri!.AbsolutePath.EndsWith("/original", StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
|
if (!request.RequestUri!.AbsolutePath.EndsWith("/" + kind, StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
|
||||||
return response;
|
return response;
|
||||||
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
|
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
|
||||||
var content = new StreamContent(new CutStream(body, body.Length / 2));
|
var content = new StreamContent(new CutStream(body, body.Length / 2));
|
||||||
|
|||||||
Reference in new issue
Block a user