Files
Chronos/tests/Chronos.Server.Tests/Storage/UploadTests.cs
T
Alby96andClaude Opus 5.5 f2b58522b9
CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
refactor!: rename Titano to Chronos and move CI and releases to Gitea (#6)
The owner renamed the product and moved the repository to the self-hosted
Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control,
and asked for Actions and a first automatic release with the server package
and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md).

- Chronos everywhere: a byte-exact rename of contents and paths, then a
  per-language review of the forms the old name's ending had dictated.
- Gitea instead of GitHub: the update check, licence, issue and support
  links read ChronosReleasesUrl; the update check takes the highest stable
  version, offers a pre-release build its final release and reports a
  repository it cannot see as a failed check. EULA, security policy and
  Gitea issue forms live at the repository root.
- Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release,
  nightly image and performance gates in the .NET SDK container with git
  checkout and the system SQLite. The release cross-compiles the Windows
  programs, checks their icon, version and DPI manifest, packs installers
  and zips, publishes the Gitea release and pushes the server image
  (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows
  runner (WINDOWS_RUNNER).
- New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the
  docker-exec script, which Gitea's service containers do not allow; the
  throw-away server runs without durability) and pe-check.
- Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR,
  3 on the shared runner; the weekly performance gates keep theirs).

BREAKING CHANGE: programs, settings, data folders, file types, environment
variables and the Docker image use the new name; nothing is migrated from
Titano, which never had a release.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-10-23 12:45:18 +02:00

350 lines
21 KiB
C#

using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Chronos.Core.Model;
using Chronos.Protocol;
using Chronos.Protocol.Client;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Storage;
[Category("server")]
public sealed class UploadTests(TestContext context)
{
private const int Chunk = 64 * 1024;
[Test]
public async Task ChunksInAnyOrderCommitAVerifiedFrame()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk * 3 + 1234, 1);
var start = await Uploads.StartAsync(uploader, data, sessionKey: "roof-2026-09-28");
var id = Assert.NotNull(start.UploadId);
Assert.Equal(Chunk, start.ChunkSize);
Assert.Empty(start.Received);
foreach (var index in new[] { 3, 0, 2, 1, 0 })
{
using var put = await Uploads.PutChunkAsync(uploader, id, index, Uploads.Chunk(data, Chunk, index));
await put.ExpectAsync(HttpStatusCode.OK);
Assert.Equal(index, (await put.ReadAsync(ProtocolJson.Default.ChunkReceived)).Index);
}
using (var state = await uploader.GetAsync($"/api/v1/uploads/{id}"))
{
var resume = await state.ReadAsync(ProtocolJson.Default.UploadState);
Assert.SequenceEqual([0, 1, 2, 3], resume.Received);
Assert.Equal("receiving", resume.State);
}
using var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null);
await complete.ExpectAsync(HttpStatusCode.OK);
var frame = await complete.ReadAsync(ProtocolJson.Default.CompleteUploadResponse);
var ingest = server.Services.GetRequiredService<FakeArchiveIngest>();
var request = Assert.NotNull(ingest.Requests.LastOrDefault());
Assert.Equal("roof-2026-09-28", request.SessionKey);
Assert.Null(request.SequenceId);
Assert.Equal(Uploads.Sha(data), request.Sha256);
Assert.True(request.StagedPath.StartsWith(server.Options.StagingPath, StringComparison.Ordinal), request.StagedPath);
var stored = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
await Http.BytesEqualAsync(data, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, stored.RelativePath)));
Assert.False(server.Services.GetRequiredService<UploadStore>().Exists(id), "the partial file is gone");
Assert.Empty(Directory.EnumerateFiles(server.Options.StagingPath));
using (var me = await uploader.GetAsync("/api/v1/auth/me")) Assert.Equal(data.Length, (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes);
// A completion whose response was lost can be repeated.
using var again = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null);
await again.ExpectAsync(HttpStatusCode.OK);
Assert.Equal(frame.FrameId, (await again.ReadAsync(ProtocolJson.Default.CompleteUploadResponse)).FrameId);
}
[Test]
public async Task AnInterruptedUploadResumesWithItsReceivedChunks()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk * 4, 2);
var first = await Uploads.StartAsync(uploader, data);
foreach (var index in new[] { 0, 2 })
{
using var put = await Uploads.PutChunkAsync(uploader, first.UploadId!.Value, index, Uploads.Chunk(data, Chunk, index));
await put.ExpectAsync(HttpStatusCode.OK);
}
var resumed = await Uploads.StartAsync(uploader, data);
Assert.Equal(first.UploadId, resumed.UploadId);
Assert.SequenceEqual([0, 2], resumed.Received);
foreach (var index in new[] { 1, 3 })
{
using var put = await Uploads.PutChunkAsync(uploader, first.UploadId!.Value, index, Uploads.Chunk(data, Chunk, index));
await put.ExpectAsync(HttpStatusCode.OK);
}
using var complete = await uploader.PostAsync($"/api/v1/uploads/{first.UploadId}/complete", null);
await complete.ExpectAsync(HttpStatusCode.OK);
}
[Test]
public async Task AKnownHashShortCircuitsTheUpload()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(5000, 3);
var frame = await Uploads.UploadAsync(uploader, data);
var again = await Uploads.StartAsync(uploader, data, expected: HttpStatusCode.OK);
Assert.True(again.AlreadyExists);
Assert.Null(again.UploadId);
Assert.Equal(frame.FrameId, again.FrameId);
}
[Test]
public async Task ACopyFlaggedOrGoneIsNoShortcut()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
// A client may delete its file on "already exists": a damaged copy must ask for the bytes again.
var flagged = Uploads.Photo(5000, 11);
var first = await Uploads.UploadAsync(uploader, flagged);
await server.Database.Frames.UpdateFlagsAsync([first.FrameId], FrameFlags.Corrupt, FrameFlags.None);
Assert.False((await Uploads.StartAsync(uploader, flagged)).AlreadyExists);
var gone = Uploads.Photo(5000, 12);
var second = await Uploads.UploadAsync(uploader, gone);
File.Delete(Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(second.FrameId)).RelativePath));
Assert.False((await Uploads.StartAsync(uploader, gone)).AlreadyExists);
}
[Test]
public async Task ACommitLeftByACrashIsSettledOnlyByAnIntactFrame()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk + 100, 13);
var missing = await Uploads.UploadAsync(uploader, data);
await server.Database.Frames.UpdateFlagsAsync([missing.FrameId], FrameFlags.Missing, FrameFlags.None);
var start = await Uploads.StartAsync(uploader, data);
var id = Assert.NotNull(start.UploadId);
for (var i = 0; i < 2; i++)
{
using var put = await Uploads.PutChunkAsync(uploader, id, i, Uploads.Chunk(data, Chunk, i));
await put.ExpectAsync(HttpStatusCode.OK);
}
// The server stopped while committing it.
Assert.True(await server.Database.Uploads.TryTransitionAsync(id, Chronos.Data.Repositories.UploadState.Receiving,
Chronos.Data.Repositories.UploadState.Committing, cancellationToken: context.Cancellation));
await server.Services.GetRequiredService<UploadMaintenance>().RecoverAsync(context.Cancellation);
// The flagged frame does not stand for it: the client completes again with its part file intact.
using (var state = await uploader.GetAsync($"/api/v1/uploads/{id}")) Assert.Equal("receiving", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
Assert.True(server.Services.GetRequiredService<UploadStore>().Exists(id));
}
[Test]
public async Task AnUploadKeepsItsChunkSizeWhenTheSettingShrinks()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk * 2);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk * 3, 14);
var start = await Uploads.StartAsync(uploader, data);
var id = Assert.NotNull(start.UploadId);
Assert.Equal(Chunk * 2, start.ChunkSize);
// An administrator lowers the chunk size; the upload resumes with the size it started with.
server.Options.ChunkSizeBytes = Chunk;
var resumed = await Uploads.StartAsync(uploader, data);
Assert.Equal((id, Chunk * 2), (resumed.UploadId!.Value, resumed.ChunkSize));
for (var i = 0; i < 2; i++)
{
using var put = await Uploads.PutChunkAsync(uploader, id, i, Uploads.Chunk(data, Chunk * 2, i));
await put.ExpectAsync(HttpStatusCode.OK);
}
using var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null);
await complete.ExpectAsync(HttpStatusCode.OK);
// A new upload takes the new size, and a body above it is refused.
var next = Uploads.Photo(Chunk * 2, 15);
var small = await Uploads.StartAsync(uploader, next);
Assert.Equal(Chunk, small.ChunkSize);
using var oversized = await Uploads.PutChunkAsync(uploader, small.UploadId!.Value, 0, next);
await oversized.ExpectErrorAsync(HttpStatusCode.BadRequest, "chunk_size");
}
[Test]
public async Task AnswersThatLeaveALargeChunkUnreadReachTheClient()
{
// Chunks far above the 1 MiB other requests may send: more than the connection buffers while the server reads nothing.
const int large = 32 * 1024 * 1024;
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = large);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(24 * 1024 * 1024, 16);
var id = (await Uploads.StartAsync(uploader, data)).UploadId!.Value;
using (var first = await Uploads.PutChunkAsync(uploader, id, 0, data)) await first.ExpectAsync(HttpStatusCode.OK);
// The client lost the answer and sends the chunk again: the same answer, not a closed connection.
using (var again = await Uploads.PutChunkAsync(uploader, id, 0, data))
{
await again.ExpectAsync(HttpStatusCode.OK);
Assert.SequenceEqual([0], (await again.ReadAsync(ProtocolJson.Default.ChunkReceived)).Received);
}
using (var unknown = await Uploads.PutChunkAsync(uploader, Guid.CreateVersion7(), 0, data)) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
using (var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null)) await complete.ExpectAsync(HttpStatusCode.OK);
using (var late = await Uploads.PutChunkAsync(uploader, id, 0, data)) await late.ExpectErrorAsync(HttpStatusCode.Conflict, "upload_state");
// Authentication answers before the endpoint runs: a user who may not upload, and a session signed out meanwhile.
var (viewer, _, _) = await server.UserAsync("marco", "viewer");
using (var forbidden = await Uploads.PutChunkAsync(viewer, id, 0, data)) await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using (var logout = await uploader.PostAsync("/api/v1/auth/logout", null)) await logout.ExpectAsync(HttpStatusCode.NoContent);
using var revoked = await Uploads.PutChunkAsync(uploader, id, 0, data);
await revoked.ExpectErrorAsync(HttpStatusCode.Unauthorized, "unauthorized");
}
[Test]
public async Task RenderUploadsNeedTheEditPermission()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var frame = await Uploads.UploadAsync(uploader, Uploads.Photo(5000, 16), sessionKey: "roof");
var output = Uploads.Photo(3000, 17);
var request = new RenderUploadRequest(Guid.CreateVersion7(), "Roof.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null);
// Render outputs count against no quota: an upload-only account cannot add them to any sequence.
using (var refused = await uploader.PostAsJsonAsync($"/api/v1/sequences/{frame.SequenceId}/renders", request, ProtocolJson.Default.RenderUploadRequest))
await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using (var complete = await uploader.PostAsync($"/api/v1/sequences/{frame.SequenceId}/renders/{Guid.CreateVersion7()}/complete", null))
await complete.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using var accepted = await editor.PostAsJsonAsync($"/api/v1/sequences/{frame.SequenceId}/renders", request, ProtocolJson.Default.RenderUploadRequest);
await accepted.ExpectAsync(HttpStatusCode.Created);
}
[Test]
public async Task TheProtocolClientUploadsAndDownloadsLikeTheUploader()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (_, _, token) = await server.UserAsync("alberto", "editor");
var folder = context.CreateTempDirectory("source");
var path = Path.Combine(folder, "2026-09-28_18-02-11_0413.CR3");
var data = Uploads.Photo(Chunk * 2 + 77, 4);
await File.WriteAllBytesAsync(path, data);
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "Chronos Uploader (tests)") { Token = token };
var sequence = await client.CreateSequenceAsync(new CreateSequenceRequest("Harbour sunset", null));
var progress = new List<UploadProgress>();
var result = Assert.NotNull(await client.UploadFileAsync(path, sequence.Id, new DateTime(2026, 9, 28), progress: new SyncProgress<UploadProgress>(progress.Add)));
Assert.Equal(sequence.Id, result.SequenceId);
Assert.Equal(0, result.Index);
Assert.Equal(data.Length, progress[^1].BytesSent);
var duplicate = Assert.NotNull(await client.UploadFileAsync(path, sequence.Id, null));
Assert.Equal((result.FrameId, -1), (duplicate.FrameId, duplicate.Index));
using var downloaded = new MemoryStream();
await client.DownloadFrameAsync(result.FrameId, "original", downloaded);
Assert.BytesEqual(data, downloaded.ToArray());
var manifest = await client.GetManifestAsync(sequence.Id);
Assert.Equal(Uploads.Sha(data), manifest.Frames.Single().Sha256);
}
[Test]
public async Task ADamagedChunkFailsTheHashCheck()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk * 2, 5);
var start = await Uploads.StartAsync(uploader, data);
var id = start.UploadId!.Value;
var damaged = Uploads.Chunk(data, Chunk, 1).ToArray();
damaged[100] ^= 0xFF;
using (var a = await Uploads.PutChunkAsync(uploader, id, 0, Uploads.Chunk(data, Chunk, 0))) await a.ExpectAsync(HttpStatusCode.OK);
using (var b = await Uploads.PutChunkAsync(uploader, id, 1, damaged)) await b.ExpectAsync(HttpStatusCode.OK);
using var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null);
await complete.ExpectErrorAsync(HttpStatusCode.UnprocessableEntity, "hash_mismatch");
using (var state = await uploader.GetAsync($"/api/v1/uploads/{id}")) Assert.Equal("failed", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
Assert.False(server.Services.GetRequiredService<UploadStore>().Exists(id));
Assert.Empty(await server.Database.Frames.FindBySha256Async(Uploads.Sha(data)));
Assert.Empty(server.Services.GetRequiredService<FakeArchiveIngest>().Requests);
var retry = await Uploads.StartAsync(uploader, data);
Assert.NotEqual(id, retry.UploadId!.Value);
}
[Test]
public async Task ChunksAndCompletionsAreValidated()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var data = Uploads.Photo(Chunk + 10, 6);
var id = (await Uploads.StartAsync(uploader, data)).UploadId!.Value;
using (var small = await Uploads.PutChunkAsync(uploader, id, 0, new byte[10])) await small.ExpectErrorAsync(HttpStatusCode.BadRequest, "chunk_size");
using (var range = await Uploads.PutChunkAsync(uploader, id, 2, new byte[10])) await range.ExpectErrorAsync(HttpStatusCode.BadRequest, "chunk_out_of_range");
using (var early = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null)) await early.ExpectErrorAsync(HttpStatusCode.Conflict, "incomplete");
using (var name = await uploader.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(null, "../../etc/passwd", 10, Uploads.Sha(data), null, null),
ProtocolJson.Default.StartUploadRequest))
await name.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_file_name");
using (var hash = await uploader.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(null, "a.jpg", 10, "abc", null, null), ProtocolJson.Default.StartUploadRequest))
await hash.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_hash");
var (other, _, _) = await server.UserAsync("intruder", "uploader");
using (var foreign = await Uploads.PutChunkAsync(other, id, 0, Uploads.Chunk(data, Chunk, 0))) await foreign.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
using (var cancel = await uploader.DeleteAsync($"/api/v1/uploads/{id}")) await cancel.ExpectAsync(HttpStatusCode.NoContent);
Assert.False(server.Services.GetRequiredService<UploadStore>().Exists(id));
using var late = await Uploads.PutChunkAsync(uploader, id, 0, Uploads.Chunk(data, Chunk, 0));
await late.ExpectErrorAsync(HttpStatusCode.Conflict, "upload_state");
}
[Test]
public async Task QuotaCountsCommittedAndPendingUploads()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader", quotaBytes: 100_000);
var first = Uploads.Photo(80_000, 7);
var pending = await Uploads.StartAsync(uploader, first);
using (var over = await uploader.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(null, "b.jpg", 40_000, Uploads.Sha(Uploads.Photo(40_000, 8)), null, null),
ProtocolJson.Default.StartUploadRequest))
await over.ExpectErrorAsync(HttpStatusCode.Forbidden, "quota_exceeded");
using (var cancel = await uploader.DeleteAsync($"/api/v1/uploads/{pending.UploadId}")) await cancel.ExpectAsync(HttpStatusCode.NoContent);
await Uploads.UploadAsync(uploader, first);
using var after = await uploader.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(null, "c.jpg", 30_000, Uploads.Sha(Uploads.Photo(30_000, 9)), null, null),
ProtocolJson.Default.StartUploadRequest);
await after.ExpectErrorAsync(HttpStatusCode.Forbidden, "quota_exceeded");
}
[Test]
public async Task FilesAboveTheLimitAreRefused()
{
await using var server = await TestServer.StartAsync(context, o => o.MaxUploadBytes = 1000);
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
using var response = await uploader.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(null, "big.jpg", 1001, new string('0', 64), null, null),
ProtocolJson.Default.StartUploadRequest);
var error = await response.ExpectErrorAsync(HttpStatusCode.RequestEntityTooLarge, "too_large");
Assert.Equal("1000", error.Details);
}
[Test]
public async Task AbandonedUploadsExpireAndLoseTheirFiles()
{
var clock = new ManualTimeProvider();
await using var server = await TestServer.StartAsync(context, o => o.UploadRetentionHours = 48, s => s.AddSingleton<TimeProvider>(clock));
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var id = (await Uploads.StartAsync(uploader, Uploads.Photo(1000, 10))).UploadId!.Value;
var store = server.Services.GetRequiredService<UploadStore>();
Assert.True(store.Exists(id));
// Upload rows carry real timestamps; the sweep looks 48 h past the moved clock.
clock.Advance(TimeSpan.FromHours(49));
await server.Services.GetRequiredService<UploadMaintenance>().SweepAsync(context.Cancellation);
Assert.False(store.Exists(id));
using var state = await uploader.GetAsync($"/api/v1/uploads/{id}");
Assert.Equal("cancelled", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
}
/// <summary>Reports synchronously (Progress&lt;T&gt; posts to the thread pool and may arrive late).</summary>
private sealed class SyncProgress<T>(Action<T> report) : IProgress<T>
{
public void Report(T value) => report(value);
}
}