CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
The owner renamed the product and moved the repository to the self-hosted Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control, and asked for Actions and a first automatic release with the server package and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md). - Chronos everywhere: a byte-exact rename of contents and paths, then a per-language review of the forms the old name's ending had dictated. - Gitea instead of GitHub: the update check, licence, issue and support links read ChronosReleasesUrl; the update check takes the highest stable version, offers a pre-release build its final release and reports a repository it cannot see as a failed check. EULA, security policy and Gitea issue forms live at the repository root. - Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release, nightly image and performance gates in the .NET SDK container with git checkout and the system SQLite. The release cross-compiles the Windows programs, checks their icon, version and DPI manifest, packs installers and zips, publishes the Gitea release and pushes the server image (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows runner (WINDOWS_RUNNER). - New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the docker-exec script, which Gitea's service containers do not allow; the throw-away server runs without durability) and pe-check. - Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR, 3 on the shared runner; the weekly performance gates keep theirs). BREAKING CHANGE: programs, settings, data folders, file types, environment variables and the Docker image use the new name; nothing is migrated from Titano, which never had a release. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
304 lines
17 KiB
C#
304 lines
17 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Globalization;
|
|
using Microsoft.AspNetCore.Http.Features;
|
|
using Chronos.Data;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Repositories;
|
|
using Chronos.Protocol;
|
|
using Chronos.Server.Api;
|
|
using Chronos.Server.Configuration;
|
|
using Chronos.Server.Events;
|
|
using Chronos.Server.Jobs;
|
|
using Chronos.Server.Security;
|
|
using UploadState = Chronos.Data.Repositories.UploadState;
|
|
using UploadStateDto = Chronos.Protocol.UploadState;
|
|
|
|
namespace Chronos.Server.Storage;
|
|
|
|
/// <summary>
|
|
/// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256 held by an
|
|
/// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the
|
|
/// file on the archive volume, verifies its hash and hands it to <see cref="IArchiveIngest"/>, which repairs a missing or
|
|
/// damaged frame of that content (no quota is charged for a repair: the archive already counts those bytes).
|
|
/// </summary>
|
|
public sealed class UploadService(
|
|
ChronosDatabase db,
|
|
ServerOptions options,
|
|
ServerArchive archive,
|
|
UploadStore store,
|
|
IArchiveIngest ingest,
|
|
SequenceJobs sequenceJobs,
|
|
EventHub events,
|
|
IHostApplicationLifetime lifetime,
|
|
TimeProvider time,
|
|
ILogger<UploadService> logger)
|
|
{
|
|
/// <summary>Space kept free on the upload volume beyond the announced size.</summary>
|
|
private const long DiskReserve = 64L * 1024 * 1024;
|
|
|
|
/// <summary>Card events one account may publish per minute: progress once a second leaves room for every start and end.</summary>
|
|
private const int CardEventsPerMinute = 120;
|
|
|
|
private static readonly TimeSpan CardProgressInterval = TimeSpan.FromSeconds(1);
|
|
private static readonly TimeSpan CardEventWindow = TimeSpan.FromMinutes(1);
|
|
|
|
private readonly ConcurrentDictionary<Guid, int> _chunksInFlight = new();
|
|
private readonly ConcurrentDictionary<Guid, CardEvents> _cardEvents = new();
|
|
|
|
public async Task<Outcome<StartUploadResponse>> StartAsync(Caller caller, StartUploadRequest request, CancellationToken cancellationToken)
|
|
{
|
|
if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Fail<StartUploadResponse>(400, "invalid_file_name");
|
|
if (!UploadRules.IsSha256(request.Sha256)) return Fail<StartUploadResponse>(400, "invalid_hash");
|
|
if (request.Size <= 0) return Fail<StartUploadResponse>(400, "bad_request", "size");
|
|
if (request.Size > options.MaxUploadBytes) return Fail<StartUploadResponse>(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture));
|
|
if (request.SessionKey is { Length: > 200 }) return Fail<StartUploadResponse>(400, "bad_request", "sessionKey");
|
|
if (request.SequenceId is { } requested && await db.Sequences.GetAsync(requested, cancellationToken) is null)
|
|
return Fail<StartUploadResponse>(404, "not_found", "sequence");
|
|
|
|
var sha = request.Sha256.ToLowerInvariant();
|
|
var sessionKey = string.IsNullOrWhiteSpace(request.SessionKey) ? null : request.SessionKey.Trim();
|
|
var copies = await db.Frames.FindBySha256Async(sha, cancellationToken);
|
|
var target = copies.Count == 0 ? null : await UploadRules.TargetAsync(db, archive.Id, request.SequenceId, sessionKey, caller.Id, cancellationToken);
|
|
// Only a copy that is still whole stands for the upload: a client may delete its file on this answer.
|
|
if (UploadRules.StandIn(archive.Root, copies, target) is { } known)
|
|
return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, known.Id);
|
|
|
|
var resumable = await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken);
|
|
if (resumable is not null && resumable.Size == request.Size && resumable.SequenceId == request.SequenceId && store.Exists(resumable.Id))
|
|
return Started(resumable);
|
|
|
|
// A repair puts back bytes the archive already counts: no quota for it.
|
|
if (caller.User.QuotaBytes > 0 && !copies.Any(f => f.SequenceId == target))
|
|
{
|
|
var pending = await db.Uploads.PendingBytesAsync(caller.Id, cancellationToken);
|
|
if (caller.User.UsedBytes + pending + request.Size > caller.User.QuotaBytes)
|
|
return Fail<StartUploadResponse>(403, "quota_exceeded", caller.User.QuotaBytes.ToString(CultureInfo.InvariantCulture));
|
|
}
|
|
if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size + DiskReserve)
|
|
return Fail<StartUploadResponse>(507, "insufficient_storage");
|
|
|
|
var now = time.GetUtcNow();
|
|
var upload = await db.Uploads.CreateAsync(new Upload
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
UserId = caller.Id,
|
|
SequenceId = request.SequenceId,
|
|
FileName = fileName,
|
|
Size = request.Size,
|
|
Sha256 = sha,
|
|
CapturedAt = request.CapturedAt,
|
|
ChunkSize = options.ChunkSizeBytes,
|
|
State = UploadState.Receiving,
|
|
Created = now,
|
|
UpdatedAt = now,
|
|
SessionKey = sessionKey,
|
|
}, cancellationToken);
|
|
store.Create(upload.Id, upload.Size);
|
|
return Started(upload);
|
|
}
|
|
|
|
/// <summary>Stores chunk <paramref name="index"/> of an upload the caller owns; repeating a received chunk changes nothing.</summary>
|
|
public async Task<Outcome<ChunkReceived>> ReceiveChunkAsync(Caller caller, Guid id, int index, HttpRequest request, CancellationToken cancellationToken)
|
|
{
|
|
var upload = await db.Uploads.GetAsync(id, cancellationToken);
|
|
// A body left unread by an answer below is drained after it: one of a chunk's size must not count as too large, or
|
|
// Kestrel would close the connection while the client still sends it and the client would never see the answer. The
|
|
// upload keeps the chunk size it started with: a smaller setting since a restart must not refuse its chunks (413).
|
|
if (request.HttpContext.Features.Get<IHttpMaxRequestBodySizeFeature>() is { IsReadOnly: false, MaxRequestBodySize: { } current } limit)
|
|
limit.MaxRequestBodySize = Math.Max(current, Math.Max(upload?.ChunkSize ?? 0, options.ChunkSizeBytes));
|
|
if (upload is null || upload.UserId != caller.Id) return Fail<ChunkReceived>(404, "not_found", "upload");
|
|
if (upload.State != UploadState.Receiving) return Fail<ChunkReceived>(409, "upload_state");
|
|
if (index < 0 || index >= upload.ChunkCount) return Fail<ChunkReceived>(400, "chunk_out_of_range");
|
|
var expected = Math.Min(upload.ChunkSize, upload.Size - (long)index * upload.ChunkSize);
|
|
if (request.ContentLength is { } announced && announced != expected) return Fail<ChunkReceived>(400, "chunk_size", expected.ToString(CultureInfo.InvariantCulture));
|
|
if (upload.HasChunk(index)) return new ChunkReceived(index, UploadRules.ReceivedChunks(upload));
|
|
|
|
if (_chunksInFlight.AddOrUpdate(caller.Id, 1, (_, n) => n + 1) > options.ConcurrentChunksPerUser)
|
|
{
|
|
Release(caller.Id);
|
|
return Fail<ChunkReceived>(429, "too_many_uploads");
|
|
}
|
|
try
|
|
{
|
|
if (!await store.WriteChunkAsync(id, (long)index * upload.ChunkSize, request.Body, expected, cancellationToken))
|
|
return Fail<ChunkReceived>(400, "chunk_size", expected.ToString(CultureInfo.InvariantCulture));
|
|
}
|
|
finally
|
|
{
|
|
Release(caller.Id);
|
|
}
|
|
var updated = await db.Uploads.MarkChunkReceivedAsync(id, index, cancellationToken);
|
|
return updated is null ? Fail<ChunkReceived>(404, "not_found", "upload") : new ChunkReceived(index, UploadRules.ReceivedChunks(updated));
|
|
}
|
|
|
|
public async Task<Outcome<UploadStateDto>> GetStateAsync(Caller caller, Guid id, CancellationToken cancellationToken)
|
|
{
|
|
var upload = await db.Uploads.GetAsync(id, cancellationToken);
|
|
if (upload is null || upload.UserId != caller.Id) return Fail<UploadStateDto>(404, "not_found", "upload");
|
|
return new UploadStateDto(upload.Id, upload.FileName, upload.Size, upload.Sha256, upload.ChunkSize, UploadRules.ReceivedChunks(upload),
|
|
upload.State.ToString().ToLowerInvariant());
|
|
}
|
|
|
|
/// <summary>Abandons an upload that is still receiving and deletes its partial file.</summary>
|
|
public async Task<Outcome<bool>> CancelAsync(Caller caller, Guid id, CancellationToken cancellationToken)
|
|
{
|
|
var upload = await db.Uploads.GetAsync(id, cancellationToken);
|
|
if (upload is null || upload.UserId != caller.Id) return Fail<bool>(404, "not_found", "upload");
|
|
if (!await db.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Cancelled, cancellationToken: cancellationToken))
|
|
return Fail<bool>(409, "upload_state");
|
|
store.Delete(id);
|
|
return true;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies and commits a frame upload. Completing an already committed upload answers its frame again, so a client
|
|
/// may retry a completion whose response it lost. Runs to the end even if the client disconnects.
|
|
/// </summary>
|
|
public async Task<Outcome<CompleteUploadResponse>> CompleteAsync(Caller caller, Guid id)
|
|
{
|
|
var stopping = lifetime.ApplicationStopping;
|
|
var upload = await db.Uploads.GetAsync(id, stopping);
|
|
if (upload is null || upload.UserId != caller.Id) return Fail<CompleteUploadResponse>(404, "not_found", "upload");
|
|
if (upload.JobId is not null || upload.RenderJson is not null) return Fail<CompleteUploadResponse>(409, "wrong_upload_kind");
|
|
if (upload is { State: UploadState.Committed, FrameId: { } committed } && await db.Frames.GetAsync(committed, stopping) is { } done)
|
|
return new CompleteUploadResponse(done.Id, done.SequenceId, done.Index);
|
|
if (upload.State != UploadState.Receiving) return Fail<CompleteUploadResponse>(409, "upload_state");
|
|
if (!upload.IsComplete)
|
|
return Fail<CompleteUploadResponse>(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture));
|
|
if (!await db.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: stopping))
|
|
return Fail<CompleteUploadResponse>(409, "upload_state");
|
|
|
|
var user = await db.Users.GetAsync(caller.Id, stopping);
|
|
if (user is { QuotaBytes: > 0 } && user.UsedBytes + upload.Size > user.QuotaBytes && !await HeldByTargetAsync(upload, stopping))
|
|
{
|
|
await FailAsync(upload);
|
|
return Fail<CompleteUploadResponse>(403, "quota_exceeded", user.QuotaBytes.ToString(CultureInfo.InvariantCulture));
|
|
}
|
|
|
|
var staged = await StageVerifiedAsync(upload, stopping);
|
|
if (staged is null) return Fail<CompleteUploadResponse>(422, "hash_mismatch");
|
|
|
|
IngestResult result;
|
|
try
|
|
{
|
|
result = await ingest.CommitAsync(new IngestRequest(
|
|
archive.Id,
|
|
upload.SequenceId,
|
|
NewSequenceTitle: null,
|
|
upload.SequenceId is null ? UploadRules.SessionKeyOf(upload.SessionKey, caller.Id) : null,
|
|
staged,
|
|
upload.FileName,
|
|
upload.Size,
|
|
upload.Sha256,
|
|
upload.CapturedAt,
|
|
caller.Id), stopping);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
// The part file is intact: the client may complete again.
|
|
logger.LogError(ex, "Archive ingest of upload {Upload} ({File}) failed", id, upload.FileName);
|
|
UploadStore.TryDelete(staged);
|
|
await db.Uploads.TryTransitionAsync(id, UploadState.Committing, UploadState.Receiving, cancellationToken: CancellationToken.None);
|
|
return Fail<CompleteUploadResponse>(503, "commit_failed");
|
|
}
|
|
|
|
await db.Uploads.TryTransitionAsync(id, UploadState.Committing, UploadState.Committed, result.FrameId, CancellationToken.None);
|
|
store.Delete(id);
|
|
if (File.Exists(staged)) UploadStore.TryDelete(staged);
|
|
if (!result.Duplicate && !result.Repaired) await db.Users.AddUsedBytesAsync(caller.Id, upload.Size, CancellationToken.None);
|
|
if (await db.Sequences.GetAsync(result.SequenceId, CancellationToken.None) is { } sequence)
|
|
{
|
|
await sequenceJobs.FramesAddedAsync(sequence, CancellationToken.None);
|
|
events.Publish(new SequenceChangedEvent(sequence.Id, "frames", sequence.FramesVersion, sequence.SettingsVersion));
|
|
}
|
|
return new CompleteUploadResponse(result.FrameId, result.SequenceId, result.Index);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tells every connected client about a card import (BRIEF §14.3 <c>cardEvent</c>), so one account cannot crowd their
|
|
/// event queues: its progress goes out at most once a second (a report sooner is dropped, the next one replaces it as
|
|
/// on the client), and at most 120 of its events a minute (429 beyond). False when the report was dropped.
|
|
/// </summary>
|
|
public Outcome<bool> PublishCard(Caller caller, CardEvent card)
|
|
{
|
|
var now = time.GetUtcNow();
|
|
var sent = _cardEvents.GetOrAdd(caller.Id, _ => new CardEvents());
|
|
lock (sent.Gate)
|
|
{
|
|
var progress = card.State == CardImportStates.Progress;
|
|
if (progress && now - sent.LastProgress < CardProgressInterval) return false;
|
|
while (sent.Recent.Count > 0 && now - sent.Recent.Peek() >= CardEventWindow) sent.Recent.Dequeue();
|
|
if (sent.Recent.Count >= CardEventsPerMinute)
|
|
{
|
|
var wait = Math.Max(1, (int)Math.Ceiling((CardEventWindow - (now - sent.Recent.Peek())).TotalSeconds));
|
|
return Fail<bool>(429, "rate_limited", wait.ToString(CultureInfo.InvariantCulture));
|
|
}
|
|
sent.Recent.Enqueue(now);
|
|
if (progress) sent.LastProgress = now;
|
|
}
|
|
events.Publish(card);
|
|
return true;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Copies a fully received upload to the archive staging folder and checks its SHA-256. On a mismatch the upload
|
|
/// fails, its files are deleted and null is returned; the client starts over.
|
|
/// </summary>
|
|
public async Task<string?> StageVerifiedAsync(Upload upload, CancellationToken cancellationToken)
|
|
{
|
|
var staged = Path.Combine(options.StagingPath, upload.Id.ToString("N") + Path.GetExtension(upload.FileName).ToLowerInvariant());
|
|
string hash;
|
|
try
|
|
{
|
|
hash = await store.StageAsync(upload.Id, staged, cancellationToken);
|
|
}
|
|
catch
|
|
{
|
|
UploadStore.TryDelete(staged);
|
|
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Receiving, cancellationToken: CancellationToken.None);
|
|
throw;
|
|
}
|
|
if (hash == upload.Sha256) return staged;
|
|
logger.LogWarning("Upload {Upload} ({File}) arrived damaged: SHA-256 {Actual} instead of {Expected}", upload.Id, upload.FileName, hash, upload.Sha256);
|
|
UploadStore.TryDelete(staged);
|
|
await FailAsync(upload);
|
|
return null;
|
|
}
|
|
|
|
private async Task FailAsync(Upload upload)
|
|
{
|
|
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None);
|
|
store.Delete(upload.Id);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The sequence the upload files into holds its content already: filing it adds no bytes (an intact copy makes it a
|
|
/// duplicate, a damaged one is repaired with it).
|
|
/// </summary>
|
|
private async Task<bool> HeldByTargetAsync(Upload upload, CancellationToken cancellationToken)
|
|
{
|
|
var copies = await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken);
|
|
if (copies.Count == 0) return false;
|
|
var target = await UploadRules.TargetAsync(db, archive.Id, upload.SequenceId, upload.SessionKey, upload.UserId, cancellationToken);
|
|
return copies.Any(f => f.SequenceId == target);
|
|
}
|
|
|
|
private StartUploadResponse Started(Upload upload) =>
|
|
new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null);
|
|
|
|
private void Release(Guid userId)
|
|
{
|
|
if (_chunksInFlight.AddOrUpdate(userId, 0, (_, n) => n - 1) <= 0) _chunksInFlight.TryRemove(new KeyValuePair<Guid, int>(userId, 0));
|
|
}
|
|
|
|
private static Outcome<T> Fail<T>(int status, string code, string? details = null) => Outcome<T>.Fail(status, code, details);
|
|
|
|
/// <summary>The card events one account published lately.</summary>
|
|
private sealed class CardEvents
|
|
{
|
|
public Lock Gate { get; } = new();
|
|
public Queue<DateTimeOffset> Recent { get; } = new();
|
|
public DateTimeOffset LastProgress { get; set; } = DateTimeOffset.MinValue;
|
|
}
|
|
}
|