Files
Chronos/src/Chronos.Server/Storage/UploadService.cs
T
Alby96andClaude Opus 5.5 d560b41b81 feat(server): show the Unraid array's free space and check the archive volume before filing uploads
On Unraid the archive is a user share (FUSE shfs), which reports the share's
primary storage, often the cache, while the mover sends the photos to the
array: the admin panel showed the cache's free space (owner review 2, R2).

- New option SpacePath (server.json) / CHRONOS_SPACE_PATH: the admin panel
  shows that folder's volume; empty, or a folder that cannot be measured, falls
  back to the archive's volume (one warning, at start, not per request).
- VolumeSpace measures folders (DriveInfo; the path root on Windows) and reads
  the filesystem type and mount from /proc/self/mountinfo on Linux (longest
  mount point on segment boundaries, octal escapes, optional fields, the top of
  stacked mounts), since DriveFormat calls an shfs share only "fuse". It is an
  injectable service, so tests simulate a full volume; the /data checks use it.
- ArchiveStatsDto keeps FreeBytes/CapacityBytes for the shown volume and gains
  optional trailing members: the write volume's figures (when a space path is
  in use and they differ), the filesystem type and UsesSpacePath.
- Admin panel (dashboard and archive): a tooltip on the storage figures names
  the volume and its filesystem and, on an shfs share without a space path,
  says they are the primary storage and to map the array view to /space; a
  "Write volume" line shows the archive's own volume when it differs.
- Completing an upload, a render upload or a render node's result checks the
  archive volume for its size plus the 64 MiB reserve before the staging copy
  and answers 507 insufficient_storage, the upload still receiving.
- Diagnostics list the archive and space volumes with filesystem and mount.
- Unraid template: optional read-only "Array view" /space (default
  /mnt/user0/Timelapse) and "Space path" CHRONOS_SPACE_PATH=/space; compose and
  the releases README describe the same as an option.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 21:58:27 +01:00

322 lines
18 KiB
C#

using System.Collections.Concurrent;
using System.Globalization;
using Microsoft.AspNetCore.Http.Features;
using Chronos.Data;
using Chronos.Data.Archiving;
using Chronos.Data.Repositories;
using Chronos.Protocol;
using Chronos.Server.Api;
using Chronos.Server.Configuration;
using Chronos.Server.Events;
using Chronos.Server.Jobs;
using Chronos.Server.Security;
using UploadState = Chronos.Data.Repositories.UploadState;
using UploadStateDto = Chronos.Protocol.UploadState;
namespace Chronos.Server.Storage;
/// <summary>
/// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256 held by an
/// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that checks the
/// archive volume's free space, stages the file there, verifies its hash and hands it to <see cref="IArchiveIngest"/>,
/// which repairs a missing or damaged frame of that content (no quota is charged for a repair: the archive already
/// counts those bytes).
/// </summary>
public sealed class UploadService(
ChronosDatabase db,
ServerOptions options,
ServerArchive archive,
UploadStore store,
IArchiveIngest ingest,
SequenceJobs sequenceJobs,
EventHub events,
VolumeSpace volumes,
IHostApplicationLifetime lifetime,
TimeProvider time,
ILogger<UploadService> logger)
{
/// <summary>Space kept free on the upload and archive volumes beyond an upload's size.</summary>
private const long DiskReserve = 64L * 1024 * 1024;
/// <summary>Card events one account may publish per minute: progress once a second leaves room for every start and end.</summary>
private const int CardEventsPerMinute = 120;
private static readonly TimeSpan CardProgressInterval = TimeSpan.FromSeconds(1);
private static readonly TimeSpan CardEventWindow = TimeSpan.FromMinutes(1);
private readonly ConcurrentDictionary<Guid, int> _chunksInFlight = new();
private readonly ConcurrentDictionary<Guid, CardEvents> _cardEvents = new();
public async Task<Outcome<StartUploadResponse>> StartAsync(Caller caller, StartUploadRequest request, CancellationToken cancellationToken)
{
if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Fail<StartUploadResponse>(400, "invalid_file_name");
if (!UploadRules.IsSha256(request.Sha256)) return Fail<StartUploadResponse>(400, "invalid_hash");
if (request.Size <= 0) return Fail<StartUploadResponse>(400, "bad_request", "size");
if (request.Size > options.MaxUploadBytes) return Fail<StartUploadResponse>(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture));
if (request.SessionKey is { Length: > 200 }) return Fail<StartUploadResponse>(400, "bad_request", "sessionKey");
if (request.SequenceId is { } requested && await db.Sequences.GetAsync(requested, cancellationToken) is null)
return Fail<StartUploadResponse>(404, "not_found", "sequence");
var sha = request.Sha256.ToLowerInvariant();
var sessionKey = string.IsNullOrWhiteSpace(request.SessionKey) ? null : request.SessionKey.Trim();
var copies = await db.Frames.FindBySha256Async(sha, cancellationToken);
var target = copies.Count == 0 ? null : await UploadRules.TargetAsync(db, archive.Id, request.SequenceId, sessionKey, caller.Id, cancellationToken);
// Only a copy that is still whole stands for the upload: a client may delete its file on this answer.
if (UploadRules.StandIn(archive.Root, copies, target) is { } known)
return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, known.Id);
var resumable = await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken);
if (resumable is not null && resumable.Size == request.Size && resumable.SequenceId == request.SequenceId && store.Exists(resumable.Id))
return Started(resumable);
// A repair puts back bytes the archive already counts: no quota for it.
if (caller.User.QuotaBytes > 0 && !copies.Any(f => f.SequenceId == target))
{
var pending = await db.Uploads.PendingBytesAsync(caller.Id, cancellationToken);
if (caller.User.UsedBytes + pending + request.Size > caller.User.QuotaBytes)
return Fail<StartUploadResponse>(403, "quota_exceeded", caller.User.QuotaBytes.ToString(CultureInfo.InvariantCulture));
}
if (volumes.Measure(options.UploadsPath) is { } uploadVolume && uploadVolume.FreeBytes < request.Size + DiskReserve)
return Fail<StartUploadResponse>(507, "insufficient_storage");
var now = time.GetUtcNow();
var upload = await db.Uploads.CreateAsync(new Upload
{
Id = Guid.CreateVersion7(),
UserId = caller.Id,
SequenceId = request.SequenceId,
FileName = fileName,
Size = request.Size,
Sha256 = sha,
CapturedAt = request.CapturedAt,
ChunkSize = options.ChunkSizeBytes,
State = UploadState.Receiving,
Created = now,
UpdatedAt = now,
SessionKey = sessionKey,
}, cancellationToken);
store.Create(upload.Id, upload.Size);
return Started(upload);
}
/// <summary>Stores chunk <paramref name="index"/> of an upload the caller owns; repeating a received chunk changes nothing.</summary>
public async Task<Outcome<ChunkReceived>> ReceiveChunkAsync(Caller caller, Guid id, int index, HttpRequest request, CancellationToken cancellationToken)
{
var upload = await db.Uploads.GetAsync(id, cancellationToken);
// A body left unread by an answer below is drained after it: one of a chunk's size must not count as too large, or
// Kestrel would close the connection while the client still sends it and the client would never see the answer. The
// upload keeps the chunk size it started with: a smaller setting since a restart must not refuse its chunks (413).
if (request.HttpContext.Features.Get<IHttpMaxRequestBodySizeFeature>() is { IsReadOnly: false, MaxRequestBodySize: { } current } limit)
limit.MaxRequestBodySize = Math.Max(current, Math.Max(upload?.ChunkSize ?? 0, options.ChunkSizeBytes));
if (upload is null || upload.UserId != caller.Id) return Fail<ChunkReceived>(404, "not_found", "upload");
if (upload.State != UploadState.Receiving) return Fail<ChunkReceived>(409, "upload_state");
if (index < 0 || index >= upload.ChunkCount) return Fail<ChunkReceived>(400, "chunk_out_of_range");
var expected = Math.Min(upload.ChunkSize, upload.Size - (long)index * upload.ChunkSize);
if (request.ContentLength is { } announced && announced != expected) return Fail<ChunkReceived>(400, "chunk_size", expected.ToString(CultureInfo.InvariantCulture));
if (upload.HasChunk(index)) return new ChunkReceived(index, UploadRules.ReceivedChunks(upload));
if (_chunksInFlight.AddOrUpdate(caller.Id, 1, (_, n) => n + 1) > options.ConcurrentChunksPerUser)
{
Release(caller.Id);
return Fail<ChunkReceived>(429, "too_many_uploads");
}
try
{
if (!await store.WriteChunkAsync(id, (long)index * upload.ChunkSize, request.Body, expected, cancellationToken))
return Fail<ChunkReceived>(400, "chunk_size", expected.ToString(CultureInfo.InvariantCulture));
}
finally
{
Release(caller.Id);
}
var updated = await db.Uploads.MarkChunkReceivedAsync(id, index, cancellationToken);
return updated is null ? Fail<ChunkReceived>(404, "not_found", "upload") : new ChunkReceived(index, UploadRules.ReceivedChunks(updated));
}
public async Task<Outcome<UploadStateDto>> GetStateAsync(Caller caller, Guid id, CancellationToken cancellationToken)
{
var upload = await db.Uploads.GetAsync(id, cancellationToken);
if (upload is null || upload.UserId != caller.Id) return Fail<UploadStateDto>(404, "not_found", "upload");
return new UploadStateDto(upload.Id, upload.FileName, upload.Size, upload.Sha256, upload.ChunkSize, UploadRules.ReceivedChunks(upload),
upload.State.ToString().ToLowerInvariant());
}
/// <summary>Abandons an upload that is still receiving and deletes its partial file.</summary>
public async Task<Outcome<bool>> CancelAsync(Caller caller, Guid id, CancellationToken cancellationToken)
{
var upload = await db.Uploads.GetAsync(id, cancellationToken);
if (upload is null || upload.UserId != caller.Id) return Fail<bool>(404, "not_found", "upload");
if (!await db.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Cancelled, cancellationToken: cancellationToken))
return Fail<bool>(409, "upload_state");
store.Delete(id);
return true;
}
/// <summary>
/// Verifies and commits a frame upload. Completing an already committed upload answers its frame again, so a client
/// may retry a completion whose response it lost. Runs to the end even if the client disconnects.
/// </summary>
public async Task<Outcome<CompleteUploadResponse>> CompleteAsync(Caller caller, Guid id)
{
var stopping = lifetime.ApplicationStopping;
var upload = await db.Uploads.GetAsync(id, stopping);
if (upload is null || upload.UserId != caller.Id) return Fail<CompleteUploadResponse>(404, "not_found", "upload");
if (upload.JobId is not null || upload.RenderJson is not null) return Fail<CompleteUploadResponse>(409, "wrong_upload_kind");
if (upload is { State: UploadState.Committed, FrameId: { } committed } && await db.Frames.GetAsync(committed, stopping) is { } done)
return new CompleteUploadResponse(done.Id, done.SequenceId, done.Index);
if (upload.State != UploadState.Receiving) return Fail<CompleteUploadResponse>(409, "upload_state");
if (!upload.IsComplete)
return Fail<CompleteUploadResponse>(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture));
if (!ArchiveHasRoom(upload)) return Fail<CompleteUploadResponse>(507, "insufficient_storage");
if (!await db.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: stopping))
return Fail<CompleteUploadResponse>(409, "upload_state");
var user = await db.Users.GetAsync(caller.Id, stopping);
if (user is { QuotaBytes: > 0 } && user.UsedBytes + upload.Size > user.QuotaBytes && !await HeldByTargetAsync(upload, stopping))
{
await FailAsync(upload);
return Fail<CompleteUploadResponse>(403, "quota_exceeded", user.QuotaBytes.ToString(CultureInfo.InvariantCulture));
}
var staged = await StageVerifiedAsync(upload, stopping);
if (staged is null) return Fail<CompleteUploadResponse>(422, "hash_mismatch");
IngestResult result;
try
{
result = await ingest.CommitAsync(new IngestRequest(
archive.Id,
upload.SequenceId,
NewSequenceTitle: null,
upload.SequenceId is null ? UploadRules.SessionKeyOf(upload.SessionKey, caller.Id) : null,
staged,
upload.FileName,
upload.Size,
upload.Sha256,
upload.CapturedAt,
caller.Id), stopping);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
// The part file is intact: the client may complete again.
logger.LogError(ex, "Archive ingest of upload {Upload} ({File}) failed", id, upload.FileName);
UploadStore.TryDelete(staged);
await db.Uploads.TryTransitionAsync(id, UploadState.Committing, UploadState.Receiving, cancellationToken: CancellationToken.None);
return Fail<CompleteUploadResponse>(503, "commit_failed");
}
await db.Uploads.TryTransitionAsync(id, UploadState.Committing, UploadState.Committed, result.FrameId, CancellationToken.None);
store.Delete(id);
if (File.Exists(staged)) UploadStore.TryDelete(staged);
if (!result.Duplicate && !result.Repaired) await db.Users.AddUsedBytesAsync(caller.Id, upload.Size, CancellationToken.None);
if (await db.Sequences.GetAsync(result.SequenceId, CancellationToken.None) is { } sequence)
{
await sequenceJobs.FramesAddedAsync(sequence, CancellationToken.None);
events.Publish(new SequenceChangedEvent(sequence.Id, "frames", sequence.FramesVersion, sequence.SettingsVersion));
}
return new CompleteUploadResponse(result.FrameId, result.SequenceId, result.Index);
}
/// <summary>
/// Tells every connected client about a card import (BRIEF §14.3 <c>cardEvent</c>), so one account cannot crowd their
/// event queues: its progress goes out at most once a second (a report sooner is dropped, the next one replaces it as
/// on the client), and at most 120 of its events a minute (429 beyond). False when the report was dropped.
/// </summary>
public Outcome<bool> PublishCard(Caller caller, CardEvent card)
{
var now = time.GetUtcNow();
var sent = _cardEvents.GetOrAdd(caller.Id, _ => new CardEvents());
lock (sent.Gate)
{
var progress = card.State == CardImportStates.Progress;
if (progress && now - sent.LastProgress < CardProgressInterval) return false;
while (sent.Recent.Count > 0 && now - sent.Recent.Peek() >= CardEventWindow) sent.Recent.Dequeue();
if (sent.Recent.Count >= CardEventsPerMinute)
{
var wait = Math.Max(1, (int)Math.Ceiling((CardEventWindow - (now - sent.Recent.Peek())).TotalSeconds));
return Fail<bool>(429, "rate_limited", wait.ToString(CultureInfo.InvariantCulture));
}
sent.Recent.Enqueue(now);
if (progress) sent.LastProgress = now;
}
events.Publish(card);
return true;
}
/// <summary>
/// The archive volume has room for <paramref name="upload"/> and the reserve: completing it copies the file into the
/// staging folder there (<see cref="StageVerifiedAsync"/>). Checked before the upload leaves the receiving state, so a
/// refused completion (507) can be repeated once there is room. A volume that cannot be measured refuses nothing.
/// </summary>
public bool ArchiveHasRoom(Upload upload)
{
ArgumentNullException.ThrowIfNull(upload);
if (volumes.Measure(archive.Root) is not { } volume || volume.FreeBytes >= upload.Size + DiskReserve) return true;
logger.LogWarning("The archive volume {Archive} has {Free} bytes free: upload {Upload} ({File}, {Size} bytes) waits for room",
archive.Root, volume.FreeBytes, upload.Id, upload.FileName, upload.Size);
return false;
}
/// <summary>
/// Copies a fully received upload to the archive staging folder and checks its SHA-256. On a mismatch the upload
/// fails, its files are deleted and null is returned; the client starts over. Callers check
/// <see cref="ArchiveHasRoom"/> first.
/// </summary>
public async Task<string?> StageVerifiedAsync(Upload upload, CancellationToken cancellationToken)
{
var staged = Path.Combine(options.StagingPath, upload.Id.ToString("N") + Path.GetExtension(upload.FileName).ToLowerInvariant());
string hash;
try
{
hash = await store.StageAsync(upload.Id, staged, cancellationToken);
}
catch
{
UploadStore.TryDelete(staged);
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Receiving, cancellationToken: CancellationToken.None);
throw;
}
if (hash == upload.Sha256) return staged;
logger.LogWarning("Upload {Upload} ({File}) arrived damaged: SHA-256 {Actual} instead of {Expected}", upload.Id, upload.FileName, hash, upload.Sha256);
UploadStore.TryDelete(staged);
await FailAsync(upload);
return null;
}
private async Task FailAsync(Upload upload)
{
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None);
store.Delete(upload.Id);
}
/// <summary>
/// The sequence the upload files into holds its content already: filing it adds no bytes (an intact copy makes it a
/// duplicate, a damaged one is repaired with it).
/// </summary>
private async Task<bool> HeldByTargetAsync(Upload upload, CancellationToken cancellationToken)
{
var copies = await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken);
if (copies.Count == 0) return false;
var target = await UploadRules.TargetAsync(db, archive.Id, upload.SequenceId, upload.SessionKey, upload.UserId, cancellationToken);
return copies.Any(f => f.SequenceId == target);
}
private StartUploadResponse Started(Upload upload) =>
new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null);
private void Release(Guid userId)
{
if (_chunksInFlight.AddOrUpdate(userId, 0, (_, n) => n - 1) <= 0) _chunksInFlight.TryRemove(new KeyValuePair<Guid, int>(userId, 0));
}
private static Outcome<T> Fail<T>(int status, string code, string? details = null) => Outcome<T>.Fail(status, code, details);
/// <summary>The card events one account published lately.</summary>
private sealed class CardEvents
{
public Lock Gate { get; } = new();
public Queue<DateTimeOffset> Recent { get; } = new();
public DateTimeOffset LastProgress { get; set; } = DateTimeOffset.MinValue;
}
}