namespace Chronos.Data.Repositories; /// Life cycle of a resumable upload (BRIEF §14.4). public enum UploadState { /// Chunks are arriving. Receiving, /// All chunks are in; the server is verifying the hash and moving the file into the archive. Committing, /// The file is a frame of the archive (). Committed, Failed, Cancelled, } /// A chunked upload and its received-chunk bitmap (bit n of byte n/8, least significant first). public sealed record Upload { public required Guid Id { get; init; } public required Guid UserId { get; init; } public Guid? SequenceId { get; init; } public required string FileName { get; init; } public long Size { get; init; } /// Lower-case hex SHA-256 announced by the client. public required string Sha256 { get; init; } public DateTime? CapturedAt { get; init; } public required int ChunkSize { get; init; } public byte[] Received { get; init; } = []; public int ReceivedCount { get; init; } public UploadState State { get; init; } public Guid? FrameId { get; init; } public DateTimeOffset Created { get; init; } public DateTimeOffset UpdatedAt { get; init; } /// Automatic grouping key of a frame upload without a target sequence (BRIEF §14.4). public string? SessionKey { get; init; } /// The render job whose output this upload carries; null for frame uploads. public Guid? JobId { get; init; } /// Metadata of the render a render-result upload becomes (JSON). public string? RenderJson { get; init; } public int ChunkCount => (int)((Size + ChunkSize - 1) / ChunkSize); public bool IsComplete => ReceivedCount >= ChunkCount; public bool HasChunk(int index) => index >= 0 && index / 8 < Received.Length && (Received[index / 8] & (1 << (index % 8))) != 0; } /// Committed uploads of one UTC day. public sealed record UploadDay(DateOnly Day, long Count, long Bytes); /// Resumable uploads: creation, per-chunk bookkeeping, state changes and cleanup. public sealed class UploadRepository { private const string Columns = "id, user_id, sequence_id, file_name, size, sha256, captured_at, chunk_size, received, received_count, state, frame_id, created, updated_at, " + "session_key, job_id, render_json"; private readonly DbAccess _db; internal UploadRepository(DbAccess db) => _db = db; /// Stores a new upload with an empty bitmap sized for its chunk count. public async Task CreateAsync(Upload upload, CancellationToken cancellationToken = default) { ArgumentOutOfRangeException.ThrowIfLessThan(upload.ChunkSize, 1); ArgumentOutOfRangeException.ThrowIfNegative(upload.Size); var stored = upload with { Received = new byte[(upload.ChunkCount + 7) / 8], ReceivedCount = 0, Sha256 = upload.Sha256.ToLowerInvariant() }; await _db.ExecuteAsync( $"INSERT INTO uploads ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)", [ stored.Id, stored.UserId, stored.SequenceId, stored.FileName, stored.Size, stored.Sha256, SequenceRepository.Wall(stored.CapturedAt), stored.ChunkSize, stored.Received, 0, (int)stored.State, stored.FrameId, stored.Created, stored.UpdatedAt, stored.SessionKey, stored.JobId, DbParam.Json(stored.RenderJson), ], cancellationToken).ConfigureAwait(false); return stored; } public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => _db.SingleAsync($"SELECT {Columns} FROM uploads WHERE id = $1", [id], Map, cancellationToken); /// A frame upload of the same content by the same user that is still receiving, so a client can resume it. public Task FindReceivingAsync(Guid userId, string sha256, CancellationToken cancellationToken = default) => _db.SingleAsync( $"SELECT {Columns} FROM uploads WHERE user_id = $1 AND sha256 = $2 AND state = {(int)UploadState.Receiving} AND job_id IS NULL " + "ORDER BY created DESC LIMIT 1", [userId, sha256.ToLowerInvariant()], Map, cancellationToken); /// /// Marks chunk as received (idempotent; concurrent chunks of one upload serialise on /// its row) and returns the updated upload, or null when it does not exist. /// public Task MarkChunkReceivedAsync(Guid id, int index, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { var upload = await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM uploads WHERE id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken) .ConfigureAwait(false); if (upload is null) return null; ArgumentOutOfRangeException.ThrowIfNegative(index); ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, upload.ChunkCount); if (upload.HasChunk(index)) return upload; var bitmap = (byte[])upload.Received.Clone(); bitmap[index / 8] |= (byte)(1 << (index % 8)); var updated = upload with { Received = bitmap, ReceivedCount = upload.ReceivedCount + 1, UpdatedAt = DateTimeOffset.UtcNow }; await session.ExecuteAsync("UPDATE uploads SET received = $2, received_count = $3, updated_at = $4 WHERE id = $1", [id, bitmap, updated.ReceivedCount, updated.UpdatedAt], cancellationToken).ConfigureAwait(false); return updated; }, cancellationToken); /// Moves an upload to , recording the frame it became when committed. public async Task SetStateAsync(Guid id, UploadState state, Guid? frameId = null, CancellationToken cancellationToken = default) => await _db.ExecuteAsync("UPDATE uploads SET state = $2, frame_id = COALESCE($3, frame_id), updated_at = $4 WHERE id = $1", [id, (int)state, frameId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; public Task> ListByUserAsync(Guid userId, CancellationToken cancellationToken = default) => _db.ListAsync($"SELECT {Columns} FROM uploads WHERE user_id = $1 ORDER BY created DESC", [userId], Map, cancellationToken); /// Uploads still receiving but untouched since (abandoned; their part files can go). public Task> ListStaleAsync(DateTimeOffset updatedBefore, CancellationToken cancellationToken = default) => _db.ListAsync($"SELECT {Columns} FROM uploads WHERE state = {(int)UploadState.Receiving} AND updated_at < $1 ORDER BY updated_at", [updatedBefore], Map, cancellationToken); public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => await _db.ExecuteAsync("DELETE FROM uploads WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; /// /// Moves an upload from to only if it is still in , /// so two concurrent completions cannot both commit it. Returns false when the state had changed. /// public async Task TryTransitionAsync(Guid id, UploadState from, UploadState to, Guid? frameId = null, CancellationToken cancellationToken = default) => await _db.ExecuteAsync("UPDATE uploads SET state = $3, frame_id = COALESCE($4, frame_id), updated_at = $5 WHERE id = $1 AND state = $2", [id, (int)from, (int)to, frameId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; /// The newest upload carrying the output of render job . public Task FindByJobAsync(Guid jobId, CancellationToken cancellationToken = default) => _db.SingleAsync($"SELECT {Columns} FROM uploads WHERE job_id = $1 ORDER BY created DESC, id DESC LIMIT 1", [jobId], Map, cancellationToken); /// Every upload in , oldest change first. public Task> ListByStateAsync(UploadState state, CancellationToken cancellationToken = default) => _db.ListAsync($"SELECT {Columns} FROM uploads WHERE state = $1 ORDER BY updated_at, id", [(int)state], Map, cancellationToken); /// Bytes a user has announced in frame uploads that are not committed yet (reserved against the quota). public async Task PendingBytesAsync(Guid userId, CancellationToken cancellationToken = default) => await _db.ScalarAsync( $"SELECT COALESCE(SUM(size), 0) FROM uploads WHERE user_id = $1 AND job_id IS NULL AND state IN ({(int)UploadState.Receiving}, {(int)UploadState.Committing})", [userId], cancellationToken).ConfigureAwait(false); /// Committed frame uploads per UTC day since , oldest day first. public Task> CommittedPerDayAsync(DateTimeOffset since, CancellationToken cancellationToken = default) { var day = _db.Sql.IsPostgres ? "to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD')" : "substr(updated_at, 1, 10)"; return _db.ListAsync( $"SELECT {day}, count(*), COALESCE(SUM(size), 0) FROM uploads WHERE state = {(int)UploadState.Committed} AND job_id IS NULL AND updated_at >= $1 " + "GROUP BY 1 ORDER BY 1", [since], r => new UploadDay(DateOnly.ParseExact(r.GetString(0), "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture), r.GetInt64(1), r.GetInt64(2)), cancellationToken); } /// Committed frame uploads per user since . public async Task> CommittedCountByUserAsync(DateTimeOffset since, CancellationToken cancellationToken = default) { var rows = await _db.ListAsync( $"SELECT user_id, count(*) FROM uploads WHERE state = {(int)UploadState.Committed} AND job_id IS NULL AND updated_at >= $1 GROUP BY user_id", [since], r => KeyValuePair.Create(r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false); return new Dictionary(rows); } /// Deletes committed, failed and cancelled uploads last changed before ; returns how many. public Task PurgeFinishedAsync(DateTimeOffset updatedBefore, CancellationToken cancellationToken = default) => _db.ExecuteAsync( $"DELETE FROM uploads WHERE state IN ({(int)UploadState.Committed}, {(int)UploadState.Failed}, {(int)UploadState.Cancelled}) AND updated_at < $1", [updatedBefore], cancellationToken); private static Upload Map(DbRow r) => new() { Id = r.GetGuid(0), UserId = r.GetGuid(1), SequenceId = r.GetGuidOrNull(2), FileName = r.GetString(3), Size = r.GetInt64(4), Sha256 = r.GetString(5), CapturedAt = r.GetDateTimeOrNull(6), ChunkSize = r.GetInt32(7), Received = r.GetBytes(8), ReceivedCount = r.GetInt32(9), State = (UploadState)r.GetInt32(10), FrameId = r.GetGuidOrNull(11), Created = r.GetDateTimeOffset(12), UpdatedAt = r.GetDateTimeOffset(13), SessionKey = r.GetStringOrNull(14), JobId = r.GetGuidOrNull(15), RenderJson = r.GetStringOrNull(16), }; }