Files
Chronos/src/Chronos.Server/Storage/UploadService.cs
T
Alby96andClaude Opus 5.5 f2b58522b9
CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
refactor!: rename Titano to Chronos and move CI and releases to Gitea (#6)
The owner renamed the product and moved the repository to the self-hosted
Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control,
and asked for Actions and a first automatic release with the server package
and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md).

- Chronos everywhere: a byte-exact rename of contents and paths, then a
  per-language review of the forms the old name's ending had dictated.
- Gitea instead of GitHub: the update check, licence, issue and support
  links read ChronosReleasesUrl; the update check takes the highest stable
  version, offers a pre-release build its final release and reports a
  repository it cannot see as a failed check. EULA, security policy and
  Gitea issue forms live at the repository root.
- Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release,
  nightly image and performance gates in the .NET SDK container with git
  checkout and the system SQLite. The release cross-compiles the Windows
  programs, checks their icon, version and DPI manifest, packs installers
  and zips, publishes the Gitea release and pushes the server image
  (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows
  runner (WINDOWS_RUNNER).
- New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the
  docker-exec script, which Gitea's service containers do not allow; the
  throw-away server runs without durability) and pe-check.
- Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR,
  3 on the shared runner; the weekly performance gates keep theirs).

BREAKING CHANGE: programs, settings, data folders, file types, environment
variables and the Docker image use the new name; nothing is migrated from
Titano, which never had a release.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-10-23 12:45:18 +02:00

304 lines
17 KiB
C#

using System.Collections.Concurrent;
using System.Globalization;
using Microsoft.AspNetCore.Http.Features;
using Chronos.Data;
using Chronos.Data.Archiving;
using Chronos.Data.Repositories;
using Chronos.Protocol;
using Chronos.Server.Api;
using Chronos.Server.Configuration;
using Chronos.Server.Events;
using Chronos.Server.Jobs;
using Chronos.Server.Security;
using UploadState = Chronos.Data.Repositories.UploadState;
using UploadStateDto = Chronos.Protocol.UploadState;
namespace Chronos.Server.Storage;
/// <summary>
/// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256 held by an
/// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the
/// file on the archive volume, verifies its hash and hands it to <see cref="IArchiveIngest"/>, which repairs a missing or
/// damaged frame of that content (no quota is charged for a repair: the archive already counts those bytes).
/// </summary>
public sealed class UploadService(
ChronosDatabase db,
ServerOptions options,
ServerArchive archive,
UploadStore store,
IArchiveIngest ingest,
SequenceJobs sequenceJobs,
EventHub events,
IHostApplicationLifetime lifetime,
TimeProvider time,
ILogger<UploadService> logger)
{
/// <summary>Space kept free on the upload volume beyond the announced size.</summary>
private const long DiskReserve = 64L * 1024 * 1024;
/// <summary>Card events one account may publish per minute: progress once a second leaves room for every start and end.</summary>
private const int CardEventsPerMinute = 120;
private static readonly TimeSpan CardProgressInterval = TimeSpan.FromSeconds(1);
private static readonly TimeSpan CardEventWindow = TimeSpan.FromMinutes(1);
private readonly ConcurrentDictionary<Guid, int> _chunksInFlight = new();
private readonly ConcurrentDictionary<Guid, CardEvents> _cardEvents = new();
public async Task<Outcome<StartUploadResponse>> StartAsync(Caller caller, StartUploadRequest request, CancellationToken cancellationToken)
{
if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Fail<StartUploadResponse>(400, "invalid_file_name");
if (!UploadRules.IsSha256(request.Sha256)) return Fail<StartUploadResponse>(400, "invalid_hash");
if (request.Size <= 0) return Fail<StartUploadResponse>(400, "bad_request", "size");
if (request.Size > options.MaxUploadBytes) return Fail<StartUploadResponse>(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture));
if (request.SessionKey is { Length: > 200 }) return Fail<StartUploadResponse>(400, "bad_request", "sessionKey");
if (request.SequenceId is { } requested && await db.Sequences.GetAsync(requested, cancellationToken) is null)
return Fail<StartUploadResponse>(404, "not_found", "sequence");
var sha = request.Sha256.ToLowerInvariant();
var sessionKey = string.IsNullOrWhiteSpace(request.SessionKey) ? null : request.SessionKey.Trim();
var copies = await db.Frames.FindBySha256Async(sha, cancellationToken);
var target = copies.Count == 0 ? null : await UploadRules.TargetAsync(db, archive.Id, request.SequenceId, sessionKey, caller.Id, cancellationToken);
// Only a copy that is still whole stands for the upload: a client may delete its file on this answer.
if (UploadRules.StandIn(archive.Root, copies, target) is { } known)
return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, known.Id);
var resumable = await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken);
if (resumable is not null && resumable.Size == request.Size && resumable.SequenceId == request.SequenceId && store.Exists(resumable.Id))
return Started(resumable);
// A repair puts back bytes the archive already counts: no quota for it.
if (caller.User.QuotaBytes > 0 && !copies.Any(f => f.SequenceId == target))
{
var pending = await db.Uploads.PendingBytesAsync(caller.Id, cancellationToken);
if (caller.User.UsedBytes + pending + request.Size > caller.User.QuotaBytes)
return Fail<StartUploadResponse>(403, "quota_exceeded", caller.User.QuotaBytes.ToString(CultureInfo.InvariantCulture));
}
if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size + DiskReserve)
return Fail<StartUploadResponse>(507, "insufficient_storage");
var now = time.GetUtcNow();
var upload = await db.Uploads.CreateAsync(new Upload
{
Id = Guid.CreateVersion7(),
UserId = caller.Id,
SequenceId = request.SequenceId,
FileName = fileName,
Size = request.Size,
Sha256 = sha,
CapturedAt = request.CapturedAt,
ChunkSize = options.ChunkSizeBytes,
State = UploadState.Receiving,
Created = now,
UpdatedAt = now,
SessionKey = sessionKey,
}, cancellationToken);
store.Create(upload.Id, upload.Size);
return Started(upload);
}
/// <summary>Stores chunk <paramref name="index"/> of an upload the caller owns; repeating a received chunk changes nothing.</summary>
public async Task<Outcome<ChunkReceived>> ReceiveChunkAsync(Caller caller, Guid id, int index, HttpRequest request, CancellationToken cancellationToken)
{
var upload = await db.Uploads.GetAsync(id, cancellationToken);
// A body left unread by an answer below is drained after it: one of a chunk's size must not count as too large, or
// Kestrel would close the connection while the client still sends it and the client would never see the answer. The
// upload keeps the chunk size it started with: a smaller setting since a restart must not refuse its chunks (413).
if (request.HttpContext.Features.Get<IHttpMaxRequestBodySizeFeature>() is { IsReadOnly: false, MaxRequestBodySize: { } current } limit)
limit.MaxRequestBodySize = Math.Max(current, Math.Max(upload?.ChunkSize ?? 0, options.ChunkSizeBytes));
if (upload is null || upload.UserId != caller.Id) return Fail<ChunkReceived>(404, "not_found", "upload");
if (upload.State != UploadState.Receiving) return Fail<ChunkReceived>(409, "upload_state");
if (index < 0 || index >= upload.ChunkCount) return Fail<ChunkReceived>(400, "chunk_out_of_range");
var expected = Math.Min(upload.ChunkSize, upload.Size - (long)index * upload.ChunkSize);
if (request.ContentLength is { } announced && announced != expected) return Fail<ChunkReceived>(400, "chunk_size", expected.ToString(CultureInfo.InvariantCulture));
if (upload.HasChunk(index)) return new ChunkReceived(index, UploadRules.ReceivedChunks(upload));
if (_chunksInFlight.AddOrUpdate(caller.Id, 1, (_, n) => n + 1) > options.ConcurrentChunksPerUser)
{
Release(caller.Id);
return Fail<ChunkReceived>(429, "too_many_uploads");
}
try
{
if (!await store.WriteChunkAsync(id, (long)index * upload.ChunkSize, request.Body, expected, cancellationToken))
return Fail<ChunkReceived>(400, "chunk_size", expected.ToString(CultureInfo.InvariantCulture));
}
finally
{
Release(caller.Id);
}
var updated = await db.Uploads.MarkChunkReceivedAsync(id, index, cancellationToken);
return updated is null ? Fail<ChunkReceived>(404, "not_found", "upload") : new ChunkReceived(index, UploadRules.ReceivedChunks(updated));
}
public async Task<Outcome<UploadStateDto>> GetStateAsync(Caller caller, Guid id, CancellationToken cancellationToken)
{
var upload = await db.Uploads.GetAsync(id, cancellationToken);
if (upload is null || upload.UserId != caller.Id) return Fail<UploadStateDto>(404, "not_found", "upload");
return new UploadStateDto(upload.Id, upload.FileName, upload.Size, upload.Sha256, upload.ChunkSize, UploadRules.ReceivedChunks(upload),
upload.State.ToString().ToLowerInvariant());
}
/// <summary>Abandons an upload that is still receiving and deletes its partial file.</summary>
public async Task<Outcome<bool>> CancelAsync(Caller caller, Guid id, CancellationToken cancellationToken)
{
var upload = await db.Uploads.GetAsync(id, cancellationToken);
if (upload is null || upload.UserId != caller.Id) return Fail<bool>(404, "not_found", "upload");
if (!await db.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Cancelled, cancellationToken: cancellationToken))
return Fail<bool>(409, "upload_state");
store.Delete(id);
return true;
}
/// <summary>
/// Verifies and commits a frame upload. Completing an already committed upload answers its frame again, so a client
/// may retry a completion whose response it lost. Runs to the end even if the client disconnects.
/// </summary>
public async Task<Outcome<CompleteUploadResponse>> CompleteAsync(Caller caller, Guid id)
{
var stopping = lifetime.ApplicationStopping;
var upload = await db.Uploads.GetAsync(id, stopping);
if (upload is null || upload.UserId != caller.Id) return Fail<CompleteUploadResponse>(404, "not_found", "upload");
if (upload.JobId is not null || upload.RenderJson is not null) return Fail<CompleteUploadResponse>(409, "wrong_upload_kind");
if (upload is { State: UploadState.Committed, FrameId: { } committed } && await db.Frames.GetAsync(committed, stopping) is { } done)
return new CompleteUploadResponse(done.Id, done.SequenceId, done.Index);
if (upload.State != UploadState.Receiving) return Fail<CompleteUploadResponse>(409, "upload_state");
if (!upload.IsComplete)
return Fail<CompleteUploadResponse>(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture));
if (!await db.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: stopping))
return Fail<CompleteUploadResponse>(409, "upload_state");
var user = await db.Users.GetAsync(caller.Id, stopping);
if (user is { QuotaBytes: > 0 } && user.UsedBytes + upload.Size > user.QuotaBytes && !await HeldByTargetAsync(upload, stopping))
{
await FailAsync(upload);
return Fail<CompleteUploadResponse>(403, "quota_exceeded", user.QuotaBytes.ToString(CultureInfo.InvariantCulture));
}
var staged = await StageVerifiedAsync(upload, stopping);
if (staged is null) return Fail<CompleteUploadResponse>(422, "hash_mismatch");
IngestResult result;
try
{
result = await ingest.CommitAsync(new IngestRequest(
archive.Id,
upload.SequenceId,
NewSequenceTitle: null,
upload.SequenceId is null ? UploadRules.SessionKeyOf(upload.SessionKey, caller.Id) : null,
staged,
upload.FileName,
upload.Size,
upload.Sha256,
upload.CapturedAt,
caller.Id), stopping);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
// The part file is intact: the client may complete again.
logger.LogError(ex, "Archive ingest of upload {Upload} ({File}) failed", id, upload.FileName);
UploadStore.TryDelete(staged);
await db.Uploads.TryTransitionAsync(id, UploadState.Committing, UploadState.Receiving, cancellationToken: CancellationToken.None);
return Fail<CompleteUploadResponse>(503, "commit_failed");
}
await db.Uploads.TryTransitionAsync(id, UploadState.Committing, UploadState.Committed, result.FrameId, CancellationToken.None);
store.Delete(id);
if (File.Exists(staged)) UploadStore.TryDelete(staged);
if (!result.Duplicate && !result.Repaired) await db.Users.AddUsedBytesAsync(caller.Id, upload.Size, CancellationToken.None);
if (await db.Sequences.GetAsync(result.SequenceId, CancellationToken.None) is { } sequence)
{
await sequenceJobs.FramesAddedAsync(sequence, CancellationToken.None);
events.Publish(new SequenceChangedEvent(sequence.Id, "frames", sequence.FramesVersion, sequence.SettingsVersion));
}
return new CompleteUploadResponse(result.FrameId, result.SequenceId, result.Index);
}
/// <summary>
/// Tells every connected client about a card import (BRIEF §14.3 <c>cardEvent</c>), so one account cannot crowd their
/// event queues: its progress goes out at most once a second (a report sooner is dropped, the next one replaces it as
/// on the client), and at most 120 of its events a minute (429 beyond). False when the report was dropped.
/// </summary>
public Outcome<bool> PublishCard(Caller caller, CardEvent card)
{
var now = time.GetUtcNow();
var sent = _cardEvents.GetOrAdd(caller.Id, _ => new CardEvents());
lock (sent.Gate)
{
var progress = card.State == CardImportStates.Progress;
if (progress && now - sent.LastProgress < CardProgressInterval) return false;
while (sent.Recent.Count > 0 && now - sent.Recent.Peek() >= CardEventWindow) sent.Recent.Dequeue();
if (sent.Recent.Count >= CardEventsPerMinute)
{
var wait = Math.Max(1, (int)Math.Ceiling((CardEventWindow - (now - sent.Recent.Peek())).TotalSeconds));
return Fail<bool>(429, "rate_limited", wait.ToString(CultureInfo.InvariantCulture));
}
sent.Recent.Enqueue(now);
if (progress) sent.LastProgress = now;
}
events.Publish(card);
return true;
}
/// <summary>
/// Copies a fully received upload to the archive staging folder and checks its SHA-256. On a mismatch the upload
/// fails, its files are deleted and null is returned; the client starts over.
/// </summary>
public async Task<string?> StageVerifiedAsync(Upload upload, CancellationToken cancellationToken)
{
var staged = Path.Combine(options.StagingPath, upload.Id.ToString("N") + Path.GetExtension(upload.FileName).ToLowerInvariant());
string hash;
try
{
hash = await store.StageAsync(upload.Id, staged, cancellationToken);
}
catch
{
UploadStore.TryDelete(staged);
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Receiving, cancellationToken: CancellationToken.None);
throw;
}
if (hash == upload.Sha256) return staged;
logger.LogWarning("Upload {Upload} ({File}) arrived damaged: SHA-256 {Actual} instead of {Expected}", upload.Id, upload.FileName, hash, upload.Sha256);
UploadStore.TryDelete(staged);
await FailAsync(upload);
return null;
}
private async Task FailAsync(Upload upload)
{
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None);
store.Delete(upload.Id);
}
/// <summary>
/// The sequence the upload files into holds its content already: filing it adds no bytes (an intact copy makes it a
/// duplicate, a damaged one is repaired with it).
/// </summary>
private async Task<bool> HeldByTargetAsync(Upload upload, CancellationToken cancellationToken)
{
var copies = await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken);
if (copies.Count == 0) return false;
var target = await UploadRules.TargetAsync(db, archive.Id, upload.SequenceId, upload.SessionKey, upload.UserId, cancellationToken);
return copies.Any(f => f.SequenceId == target);
}
private StartUploadResponse Started(Upload upload) =>
new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null);
private void Release(Guid userId)
{
if (_chunksInFlight.AddOrUpdate(userId, 0, (_, n) => n - 1) <= 0) _chunksInFlight.TryRemove(new KeyValuePair<Guid, int>(userId, 0));
}
private static Outcome<T> Fail<T>(int status, string code, string? details = null) => Outcome<T>.Fail(status, code, details);
/// <summary>The card events one account published lately.</summary>
private sealed class CardEvents
{
public Lock Gate { get; } = new();
public Queue<DateTimeOffset> Recent { get; } = new();
public DateTimeOffset LastProgress { get; set; } = DateTimeOffset.MinValue;
}
}