Files
Chronos/tests/Chronos.Server.Tests/Storage/UploadTests.cs
T
Alby96andClaude Opus 5.5 2e1c67f531 feat(workspace): lay the phase 5c foundation
Phase 5c (owner review 2, R8) is built by six packages in parallel. This
commit settles everything they share, so each one touches its own files:

- job kinds WorkspaceRender (21), WorkspaceSend (22) and WorkspaceMove (23)
  with their wire names, default priorities, notices and parameter records;
  their handlers are registered as skeletons that throw until their
  package lands
- WorkspaceJobOptions and a new CreateHandlers overload; the earlier ones
  stay and default the options
- originals in the media index (an index of phase 5b reads as proxies
  only), their file names, and the exports/renders.json ledger of local
  renders, written atomically
- the optional project and version of an uploaded render
- the workspace settings (size limit, default media, version on render,
  clean-up age, project to open) and when each project was opened, which
  Reset and Import keep and Export leaves out
- WorkspaceHost split into partial files, with stubs for usage, trash and
  the folder move, removal hooks for unsent renders and last originals,
  and a guard that holds work while the workspace moves
- every new string of the phase in English and Italian, with tooltips

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-04 15:05:09 +01:00

369 lines
22 KiB
C#

using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Chronos.Core.Model;
using Chronos.Protocol;
using Chronos.Protocol.Client;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Storage;
[Category("server")]
public sealed class UploadTests(TestContext context)
{
private const int Chunk = 64 * 1024;
[Test]
public async Task ChunksInAnyOrderCommitAVerifiedFrame()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk * 3 + 1234, 1);
var start = await Uploads.StartAsync(uploader, data, sessionKey: "roof-2026-09-28");
var id = Assert.NotNull(start.UploadId);
Assert.Equal(Chunk, start.ChunkSize);
Assert.Empty(start.Received);
foreach (var index in new[] { 3, 0, 2, 1, 0 })
{
using var put = await Uploads.PutChunkAsync(uploader, id, index, Uploads.Chunk(data, Chunk, index));
await put.ExpectAsync(HttpStatusCode.OK);
Assert.Equal(index, (await put.ReadAsync(ProtocolJson.Default.ChunkReceived)).Index);
}
using (var state = await uploader.GetAsync($"/api/v1/uploads/{id}"))
{
var resume = await state.ReadAsync(ProtocolJson.Default.UploadState);
Assert.SequenceEqual([0, 1, 2, 3], resume.Received);
Assert.Equal("receiving", resume.State);
}
using var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null);
await complete.ExpectAsync(HttpStatusCode.OK);
var frame = await complete.ReadAsync(ProtocolJson.Default.CompleteUploadResponse);
var ingest = server.Services.GetRequiredService<FakeArchiveIngest>();
var request = Assert.NotNull(ingest.Requests.LastOrDefault());
Assert.Equal("roof-2026-09-28", request.SessionKey);
Assert.Null(request.SequenceId);
Assert.Equal(Uploads.Sha(data), request.Sha256);
Assert.True(request.StagedPath.StartsWith(server.Options.StagingPath, StringComparison.Ordinal), request.StagedPath);
var stored = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
await Http.BytesEqualAsync(data, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, stored.RelativePath)));
Assert.False(server.Services.GetRequiredService<UploadStore>().Exists(id), "the partial file is gone");
Assert.Empty(Directory.EnumerateFiles(server.Options.StagingPath));
using (var me = await uploader.GetAsync("/api/v1/auth/me")) Assert.Equal(data.Length, (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes);
// A completion whose response was lost can be repeated.
using var again = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null);
await again.ExpectAsync(HttpStatusCode.OK);
Assert.Equal(frame.FrameId, (await again.ReadAsync(ProtocolJson.Default.CompleteUploadResponse)).FrameId);
}
[Test]
public async Task AnInterruptedUploadResumesWithItsReceivedChunks()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk * 4, 2);
var first = await Uploads.StartAsync(uploader, data);
foreach (var index in new[] { 0, 2 })
{
using var put = await Uploads.PutChunkAsync(uploader, first.UploadId!.Value, index, Uploads.Chunk(data, Chunk, index));
await put.ExpectAsync(HttpStatusCode.OK);
}
var resumed = await Uploads.StartAsync(uploader, data);
Assert.Equal(first.UploadId, resumed.UploadId);
Assert.SequenceEqual([0, 2], resumed.Received);
foreach (var index in new[] { 1, 3 })
{
using var put = await Uploads.PutChunkAsync(uploader, first.UploadId!.Value, index, Uploads.Chunk(data, Chunk, index));
await put.ExpectAsync(HttpStatusCode.OK);
}
using var complete = await uploader.PostAsync($"/api/v1/uploads/{first.UploadId}/complete", null);
await complete.ExpectAsync(HttpStatusCode.OK);
}
[Test]
public async Task AKnownHashShortCircuitsTheUpload()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(5000, 3);
var frame = await Uploads.UploadAsync(uploader, data);
var again = await Uploads.StartAsync(uploader, data, expected: HttpStatusCode.OK);
Assert.True(again.AlreadyExists);
Assert.Null(again.UploadId);
Assert.Equal(frame.FrameId, again.FrameId);
}
[Test]
public async Task ACopyFlaggedOrGoneIsNoShortcut()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
// A client may delete its file on "already exists": a damaged copy must ask for the bytes again.
var flagged = Uploads.Photo(5000, 11);
var first = await Uploads.UploadAsync(uploader, flagged);
await server.Database.Frames.UpdateFlagsAsync([first.FrameId], FrameFlags.Corrupt, FrameFlags.None);
Assert.False((await Uploads.StartAsync(uploader, flagged)).AlreadyExists);
var gone = Uploads.Photo(5000, 12);
var second = await Uploads.UploadAsync(uploader, gone);
File.Delete(Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(second.FrameId)).RelativePath));
Assert.False((await Uploads.StartAsync(uploader, gone)).AlreadyExists);
}
[Test]
public async Task ACommitLeftByACrashIsSettledOnlyByAnIntactFrame()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk + 100, 13);
var missing = await Uploads.UploadAsync(uploader, data);
await server.Database.Frames.UpdateFlagsAsync([missing.FrameId], FrameFlags.Missing, FrameFlags.None);
var start = await Uploads.StartAsync(uploader, data);
var id = Assert.NotNull(start.UploadId);
for (var i = 0; i < 2; i++)
{
using var put = await Uploads.PutChunkAsync(uploader, id, i, Uploads.Chunk(data, Chunk, i));
await put.ExpectAsync(HttpStatusCode.OK);
}
// The server stopped while committing it.
Assert.True(await server.Database.Uploads.TryTransitionAsync(id, Chronos.Data.Repositories.UploadState.Receiving,
Chronos.Data.Repositories.UploadState.Committing, cancellationToken: context.Cancellation));
await server.Services.GetRequiredService<UploadMaintenance>().RecoverAsync(context.Cancellation);
// The flagged frame does not stand for it: the client completes again with its part file intact.
using (var state = await uploader.GetAsync($"/api/v1/uploads/{id}")) Assert.Equal("receiving", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
Assert.True(server.Services.GetRequiredService<UploadStore>().Exists(id));
}
[Test]
public async Task AnUploadKeepsItsChunkSizeWhenTheSettingShrinks()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk * 2);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk * 3, 14);
var start = await Uploads.StartAsync(uploader, data);
var id = Assert.NotNull(start.UploadId);
Assert.Equal(Chunk * 2, start.ChunkSize);
// An administrator lowers the chunk size; the upload resumes with the size it started with.
server.Options.ChunkSizeBytes = Chunk;
var resumed = await Uploads.StartAsync(uploader, data);
Assert.Equal((id, Chunk * 2), (resumed.UploadId!.Value, resumed.ChunkSize));
for (var i = 0; i < 2; i++)
{
using var put = await Uploads.PutChunkAsync(uploader, id, i, Uploads.Chunk(data, Chunk * 2, i));
await put.ExpectAsync(HttpStatusCode.OK);
}
using var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null);
await complete.ExpectAsync(HttpStatusCode.OK);
// A new upload takes the new size, and a body above it is refused.
var next = Uploads.Photo(Chunk * 2, 15);
var small = await Uploads.StartAsync(uploader, next);
Assert.Equal(Chunk, small.ChunkSize);
using var oversized = await Uploads.PutChunkAsync(uploader, small.UploadId!.Value, 0, next);
await oversized.ExpectErrorAsync(HttpStatusCode.BadRequest, "chunk_size");
}
[Test]
public async Task AnswersThatLeaveALargeChunkUnreadReachTheClient()
{
// Chunks far above the 1 MiB other requests may send: more than the connection buffers while the server reads nothing.
const int large = 32 * 1024 * 1024;
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = large);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(24 * 1024 * 1024, 16);
var id = (await Uploads.StartAsync(uploader, data)).UploadId!.Value;
using (var first = await Uploads.PutChunkAsync(uploader, id, 0, data)) await first.ExpectAsync(HttpStatusCode.OK);
// The client lost the answer and sends the chunk again: the same answer, not a closed connection.
using (var again = await Uploads.PutChunkAsync(uploader, id, 0, data))
{
await again.ExpectAsync(HttpStatusCode.OK);
Assert.SequenceEqual([0], (await again.ReadAsync(ProtocolJson.Default.ChunkReceived)).Received);
}
using (var unknown = await Uploads.PutChunkAsync(uploader, Guid.CreateVersion7(), 0, data)) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
using (var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null)) await complete.ExpectAsync(HttpStatusCode.OK);
using (var late = await Uploads.PutChunkAsync(uploader, id, 0, data)) await late.ExpectErrorAsync(HttpStatusCode.Conflict, "upload_state");
// Authentication answers before the endpoint runs: a user who may not upload, and a session signed out meanwhile.
var (viewer, _, _) = await server.UserAsync("marco", "viewer");
using (var forbidden = await Uploads.PutChunkAsync(viewer, id, 0, data)) await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using (var logout = await uploader.PostAsync("/api/v1/auth/logout", null)) await logout.ExpectAsync(HttpStatusCode.NoContent);
using var revoked = await Uploads.PutChunkAsync(uploader, id, 0, data);
await revoked.ExpectErrorAsync(HttpStatusCode.Unauthorized, "unauthorized");
}
[Test]
public async Task RenderUploadsNeedTheEditPermission()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var frame = await Uploads.UploadAsync(uploader, Uploads.Photo(5000, 16), sessionKey: "roof");
var output = Uploads.Photo(3000, 17);
var request = new RenderUploadRequest(Guid.CreateVersion7(), "Roof.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null);
// Render outputs count against no quota: an upload-only account cannot add them to any sequence.
using (var refused = await uploader.PostAsJsonAsync($"/api/v1/sequences/{frame.SequenceId}/renders", request, ProtocolJson.Default.RenderUploadRequest))
await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using (var complete = await uploader.PostAsync($"/api/v1/sequences/{frame.SequenceId}/renders/{Guid.CreateVersion7()}/complete", null))
await complete.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using var accepted = await editor.PostAsJsonAsync($"/api/v1/sequences/{frame.SequenceId}/renders", request, ProtocolJson.Default.RenderUploadRequest);
await accepted.ExpectAsync(HttpStatusCode.Created);
}
[Test]
public async Task TheProtocolClientUploadsAndDownloadsLikeTheUploader()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (_, _, token) = await server.UserAsync("alberto", "editor");
var folder = context.CreateTempDirectory("source");
var path = Path.Combine(folder, "2026-09-28_18-02-11_0413.CR3");
var data = Uploads.Photo(Chunk * 2 + 77, 4);
await File.WriteAllBytesAsync(path, data);
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "Chronos Uploader (tests)") { Token = token };
var sequence = await client.CreateSequenceAsync(new CreateSequenceRequest("Harbour sunset", null));
var progress = new List<UploadProgress>();
var result = Assert.NotNull(await client.UploadFileAsync(path, sequence.Id, new DateTime(2026, 9, 28), progress: new SyncProgress<UploadProgress>(progress.Add)));
Assert.Equal(sequence.Id, result.SequenceId);
Assert.Equal(0, result.Index);
Assert.Equal(data.Length, progress[^1].BytesSent);
var duplicate = Assert.NotNull(await client.UploadFileAsync(path, sequence.Id, null));
Assert.Equal((result.FrameId, -1), (duplicate.FrameId, duplicate.Index));
using var downloaded = new MemoryStream();
await client.DownloadFrameAsync(result.FrameId, "original", downloaded);
Assert.BytesEqual(data, downloaded.ToArray());
var manifest = await client.GetManifestAsync(sequence.Id);
Assert.Equal(Uploads.Sha(data), manifest.Frames.Single().Sha256);
}
[Test]
public async Task ADamagedChunkFailsTheHashCheck()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk * 2, 5);
var start = await Uploads.StartAsync(uploader, data);
var id = start.UploadId!.Value;
var damaged = Uploads.Chunk(data, Chunk, 1).ToArray();
damaged[100] ^= 0xFF;
using (var a = await Uploads.PutChunkAsync(uploader, id, 0, Uploads.Chunk(data, Chunk, 0))) await a.ExpectAsync(HttpStatusCode.OK);
using (var b = await Uploads.PutChunkAsync(uploader, id, 1, damaged)) await b.ExpectAsync(HttpStatusCode.OK);
using var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null);
await complete.ExpectErrorAsync(HttpStatusCode.UnprocessableEntity, "hash_mismatch");
using (var state = await uploader.GetAsync($"/api/v1/uploads/{id}")) Assert.Equal("failed", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
Assert.False(server.Services.GetRequiredService<UploadStore>().Exists(id));
Assert.Empty(await server.Database.Frames.FindBySha256Async(Uploads.Sha(data)));
Assert.Empty(server.Services.GetRequiredService<FakeArchiveIngest>().Requests);
var retry = await Uploads.StartAsync(uploader, data);
Assert.NotEqual(id, retry.UploadId!.Value);
}
[Test]
public async Task ChunksAndCompletionsAreValidated()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk + 10, 6);
var id = (await Uploads.StartAsync(uploader, data)).UploadId!.Value;
using (var small = await Uploads.PutChunkAsync(uploader, id, 0, new byte[10])) await small.ExpectErrorAsync(HttpStatusCode.BadRequest, "chunk_size");
using (var range = await Uploads.PutChunkAsync(uploader, id, 2, new byte[10])) await range.ExpectErrorAsync(HttpStatusCode.BadRequest, "chunk_out_of_range");
using (var early = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null)) await early.ExpectErrorAsync(HttpStatusCode.Conflict, "incomplete");
using (var name = await uploader.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(null, "../../etc/passwd", 10, Uploads.Sha(data), null, null),
ProtocolJson.Default.StartUploadRequest))
await name.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_file_name");
using (var hash = await uploader.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(null, "a.jpg", 10, "abc", null, null), ProtocolJson.Default.StartUploadRequest))
await hash.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_hash");
var (other, _, _) = await server.UserAsync("intruder", "uploader");
using (var foreign = await Uploads.PutChunkAsync(other, id, 0, Uploads.Chunk(data, Chunk, 0))) await foreign.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
using (var cancel = await uploader.DeleteAsync($"/api/v1/uploads/{id}")) await cancel.ExpectAsync(HttpStatusCode.NoContent);
Assert.False(server.Services.GetRequiredService<UploadStore>().Exists(id));
using var late = await Uploads.PutChunkAsync(uploader, id, 0, Uploads.Chunk(data, Chunk, 0));
await late.ExpectErrorAsync(HttpStatusCode.Conflict, "upload_state");
}
[Test]
public async Task QuotaCountsCommittedAndPendingUploads()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader", quotaBytes: 100_000);
var first = Uploads.Photo(80_000, 7);
var pending = await Uploads.StartAsync(uploader, first);
using (var over = await uploader.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(null, "b.jpg", 40_000, Uploads.Sha(Uploads.Photo(40_000, 8)), null, null),
ProtocolJson.Default.StartUploadRequest))
await over.ExpectErrorAsync(HttpStatusCode.Forbidden, "quota_exceeded");
using (var cancel = await uploader.DeleteAsync($"/api/v1/uploads/{pending.UploadId}")) await cancel.ExpectAsync(HttpStatusCode.NoContent);
await Uploads.UploadAsync(uploader, first);
using var after = await uploader.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(null, "c.jpg", 30_000, Uploads.Sha(Uploads.Photo(30_000, 9)), null, null),
ProtocolJson.Default.StartUploadRequest);
await after.ExpectErrorAsync(HttpStatusCode.Forbidden, "quota_exceeded");
}
[Test]
public async Task FilesAboveTheLimitAreRefused()
{
await using var server = await TestServer.StartAsync(context, o => o.MaxUploadBytes = 1000);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
using var response = await uploader.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(null, "big.jpg", 1001, new string('0', 64), null, null),
ProtocolJson.Default.StartUploadRequest);
var error = await response.ExpectErrorAsync(HttpStatusCode.RequestEntityTooLarge, "too_large");
Assert.Equal("1000", error.Details);
}
[Test]
public async Task AbandonedUploadsExpireAndLoseTheirFiles()
{
var clock = new ManualTimeProvider();
await using var server = await TestServer.StartAsync(context, o => o.UploadRetentionHours = 48, s => s.AddSingleton<TimeProvider>(clock));
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var id = (await Uploads.StartAsync(uploader, Uploads.Photo(1000, 10))).UploadId!.Value;
var store = server.Services.GetRequiredService<UploadStore>();
Assert.True(store.Exists(id));
// Upload rows carry real timestamps; the sweep looks 48 h past the moved clock.
clock.Advance(TimeSpan.FromHours(49));
await server.Services.GetRequiredService<UploadMaintenance>().SweepAsync(cancellationToken: context.Cancellation);
Assert.False(store.Exists(id));
using var state = await uploader.GetAsync($"/api/v1/uploads/{id}");
Assert.Equal("cancelled", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
}
[Test]
public void ARenderOfferOfAnEarlierClientReadsWithoutItsProjectAndAWorkspaceOneKeepsIt()
{
// As a client of phase 5b sends it: no project, no version.
var earlier = System.Text.Json.JsonSerializer.Deserialize("""
{ "renderId": "0192a4c1-7b3e-7cc0-8a51-2f9d0e6b1a01", "fileName": "Roof.mp4", "size": 10, "sha256": "ab", "presetName": "4K HEVC",
"codec": "hevc", "width": 3840, "height": 2160, "fps": 30, "durationSeconds": 10 }
""", ProtocolJson.Default.RenderUploadRequest);
Assert.NotNull(earlier);
Assert.Equal(("Roof.mp4", (Guid?)null, (Guid?)null), (earlier.FileName, earlier.ProjectId, earlier.ProjectVersionId));
var omitted = System.Text.Json.JsonSerializer.Serialize(earlier, ProtocolJson.Default.RenderUploadRequest);
Assert.False(omitted.Contains("projectId", StringComparison.Ordinal), "an offer without a project names none");
var workspace = earlier with { ProjectId = Guid.CreateVersion7(), ProjectVersionId = Guid.CreateVersion7() };
var json = System.Text.Json.JsonSerializer.Serialize(workspace, ProtocolJson.Default.RenderUploadRequest);
Assert.Contains("\"projectVersionId\"", json);
Assert.Equal(workspace, System.Text.Json.JsonSerializer.Deserialize(json, ProtocolJson.Default.RenderUploadRequest));
}
/// <summary>Reports synchronously (Progress&lt;T&gt; posts to the thread pool and may arrive late).</summary>
private sealed class SyncProgress<T>(Action<T> report) : IProgress<T>
{
public void Report(T value) => report(value);
}
}