diff --git a/src/Titano.Data/Processing/RenderNodeClient.cs b/src/Titano.Data/Processing/RenderNodeClient.cs
index db13ffc1..aa53c5df 100644
--- a/src/Titano.Data/Processing/RenderNodeClient.cs
+++ b/src/Titano.Data/Processing/RenderNodeClient.cs
@@ -308,8 +308,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
}
///
- /// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it); the original
- /// when the server has none for the frame.
+ /// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it, and a retry
+ /// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server
+ /// has none for the frame.
///
private async Task DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
{
@@ -317,7 +318,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
try
{
- await FetchAsync(frame, "proxy", path, cancellationToken).ConfigureAwait(false);
+ await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false);
return path;
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
@@ -343,7 +344,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
for (var attempt = 1; ; attempt++)
{
// A part as long as the file was complete when the last attempt stopped: only its hash is left to check.
- if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, cancellationToken).ConfigureAwait(false);
+ if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, resume: true, cancellationToken).ConfigureAwait(false);
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
{
File.Move(part, path, overwrite: true);
@@ -355,17 +356,18 @@ public sealed class RenderNodeClient : IAsyncDisposable
}
///
- /// Fetches a frame resource into , appending to what an earlier attempt left there. A dropped
- /// connection or a busy server is retried, resuming where the file ends (up to times,
- /// the wait doubling from ).
+ /// Fetches a frame resource into . A dropped connection or a busy server is retried (up to
+ /// times, the wait doubling from );
+ /// with (a download its hash verifies) each attempt appends to what the last one left,
+ /// otherwise it starts over.
///
- private async Task FetchAsync(ManifestFrame frame, string kind, string path, CancellationToken cancellationToken)
+ private async Task FetchAsync(ManifestFrame frame, string kind, string path, bool resume, CancellationToken cancellationToken)
{
for (var attempt = 0; ; attempt++)
{
try
{
- await using var file = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
+ await using var file = new FileStream(path, resume ? FileMode.OpenOrCreate : FileMode.Create, FileAccess.ReadWrite, FileShare.None);
await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false);
return;
}
diff --git a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs
index dea231c2..b4887ac4 100644
--- a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs
+++ b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs
@@ -160,7 +160,7 @@ public sealed class RenderNodeClientTests(TestContext context)
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
- var cut = new CutFirstOriginal(new SocketsHttpHandler());
+ var cut = new CutFirst("original", new SocketsHttpHandler());
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
@@ -170,6 +170,33 @@ public sealed class RenderNodeClientTests(TestContext context)
Assert.Equal(1, cut.Resumed);
}
+ [Test]
+ public async Task ADroppedProxyIsFetchedWholeAgain()
+ {
+ await using var server = await TestServer.StartAsync(context);
+ var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
+ var photos = Photos();
+ var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, photos)))!;
+ // The archive has proxies of every frame (the test locator serves /proxies/NNNN.jpg).
+ var proxies = Path.Combine(server.Options.ArchivePath, sequence.Folder, "proxies");
+ Directory.CreateDirectory(proxies);
+ foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
+ await File.WriteAllBytesAsync(Path.Combine(proxies, $"{frame.Index:D4}.jpg"), photos[frame.Index]);
+ await server.Database.Sequences.UpsertAsync(sequence with { Status = sequence.Status | SequenceStatus.ProxiesBuilt });
+ var job = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, Mjpeg, 96, 64, useProxies: true).ParametersJson);
+
+ var cut = new CutFirst("proxy", new SocketsHttpHandler());
+ using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
+ var cache = context.CreateTempDirectory("node");
+ await using var node = Node(client, cache);
+ Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
+
+ Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
+ // Nothing verifies a proxy against a hash, so a retry never appends to what the dropped attempt left.
+ Assert.Equal((1, 0), (cut.Cuts, cut.Resumed));
+ Assert.False(Directory.Exists(Path.Combine(cache, "originals")), "no original needed");
+ }
+
[Test]
public async Task ANodeLeavesARenderTheServerStartedToTheServer()
{
@@ -260,8 +287,8 @@ public sealed class RenderNodeClientTests(TestContext context)
}
}
- /// Drops the connection halfway through the first original it serves; counts the downloads resumed with a range.
- private sealed class CutFirstOriginal(HttpMessageHandler inner) : DelegatingHandler(inner)
+ /// Drops the connection halfway through the first resource of it serves; counts the downloads resumed with a range.
+ private sealed class CutFirst(string kind, HttpMessageHandler inner) : DelegatingHandler(inner)
{
private int _cuts;
private int _resumed;
@@ -273,7 +300,7 @@ public sealed class RenderNodeClientTests(TestContext context)
{
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
var response = await base.SendAsync(request, cancellationToken);
- if (!request.RequestUri!.AbsolutePath.EndsWith("/original", StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
+ if (!request.RequestUri!.AbsolutePath.EndsWith("/" + kind, StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
return response;
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
var content = new StreamContent(new CutStream(body, body.Length / 2));