namespace Chronos.Data.Repositories;
/// Life cycle of a resumable upload (BRIEF §14.4).
public enum UploadState
{
/// Chunks are arriving.
Receiving,
/// All chunks are in; the server is verifying the hash and moving the file into the archive.
Committing,
/// The file is a frame of the archive ().
Committed,
Failed,
Cancelled,
}
/// A chunked upload and its received-chunk bitmap (bit n of byte n/8, least significant first).
public sealed record Upload
{
public required Guid Id { get; init; }
public required Guid UserId { get; init; }
public Guid? SequenceId { get; init; }
public required string FileName { get; init; }
public long Size { get; init; }
/// Lower-case hex SHA-256 announced by the client.
public required string Sha256 { get; init; }
public DateTime? CapturedAt { get; init; }
public required int ChunkSize { get; init; }
public byte[] Received { get; init; } = [];
public int ReceivedCount { get; init; }
public UploadState State { get; init; }
public Guid? FrameId { get; init; }
public DateTimeOffset Created { get; init; }
public DateTimeOffset UpdatedAt { get; init; }
/// Automatic grouping key of a frame upload without a target sequence (BRIEF §14.4).
public string? SessionKey { get; init; }
/// The render job whose output this upload carries; null for frame uploads.
public Guid? JobId { get; init; }
/// Metadata of the render a render-result upload becomes (JSON).
public string? RenderJson { get; init; }
public int ChunkCount => (int)((Size + ChunkSize - 1) / ChunkSize);
public bool IsComplete => ReceivedCount >= ChunkCount;
public bool HasChunk(int index) =>
index >= 0 && index / 8 < Received.Length && (Received[index / 8] & (1 << (index % 8))) != 0;
}
/// Committed uploads of one UTC day.
public sealed record UploadDay(DateOnly Day, long Count, long Bytes);
/// Resumable uploads: creation, per-chunk bookkeeping, state changes and cleanup.
public sealed class UploadRepository
{
private const string Columns =
"id, user_id, sequence_id, file_name, size, sha256, captured_at, chunk_size, received, received_count, state, frame_id, created, updated_at, " +
"session_key, job_id, render_json";
private readonly DbAccess _db;
internal UploadRepository(DbAccess db) => _db = db;
/// Stores a new upload with an empty bitmap sized for its chunk count.
public async Task CreateAsync(Upload upload, CancellationToken cancellationToken = default)
{
ArgumentOutOfRangeException.ThrowIfLessThan(upload.ChunkSize, 1);
ArgumentOutOfRangeException.ThrowIfNegative(upload.Size);
var stored = upload with { Received = new byte[(upload.ChunkCount + 7) / 8], ReceivedCount = 0, Sha256 = upload.Sha256.ToLowerInvariant() };
await _db.ExecuteAsync(
$"INSERT INTO uploads ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)",
[
stored.Id, stored.UserId, stored.SequenceId, stored.FileName, stored.Size, stored.Sha256, SequenceRepository.Wall(stored.CapturedAt),
stored.ChunkSize, stored.Received, 0, (int)stored.State, stored.FrameId, stored.Created, stored.UpdatedAt,
stored.SessionKey, stored.JobId, DbParam.Json(stored.RenderJson),
], cancellationToken).ConfigureAwait(false);
return stored;
}
public Task GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM uploads WHERE id = $1", [id], Map, cancellationToken);
/// A frame upload of the same content by the same user that is still receiving, so a client can resume it.
public Task FindReceivingAsync(Guid userId, string sha256, CancellationToken cancellationToken = default) =>
_db.SingleAsync(
$"SELECT {Columns} FROM uploads WHERE user_id = $1 AND sha256 = $2 AND state = {(int)UploadState.Receiving} AND job_id IS NULL " +
"ORDER BY created DESC LIMIT 1",
[userId, sha256.ToLowerInvariant()], Map, cancellationToken);
///
/// Marks chunk as received (idempotent; concurrent chunks of one upload serialise on
/// its row) and returns the updated upload, or null when it does not exist.
///
public Task MarkChunkReceivedAsync(Guid id, int index, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var upload = await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM uploads WHERE id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (upload is null) return null;
ArgumentOutOfRangeException.ThrowIfNegative(index);
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, upload.ChunkCount);
if (upload.HasChunk(index)) return upload;
var bitmap = (byte[])upload.Received.Clone();
bitmap[index / 8] |= (byte)(1 << (index % 8));
var updated = upload with { Received = bitmap, ReceivedCount = upload.ReceivedCount + 1, UpdatedAt = DateTimeOffset.UtcNow };
await session.ExecuteAsync("UPDATE uploads SET received = $2, received_count = $3, updated_at = $4 WHERE id = $1",
[id, bitmap, updated.ReceivedCount, updated.UpdatedAt], cancellationToken).ConfigureAwait(false);
return updated;
}, cancellationToken);
/// Moves an upload to , recording the frame it became when committed.
public async Task SetStateAsync(Guid id, UploadState state, Guid? frameId = null, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE uploads SET state = $2, frame_id = COALESCE($3, frame_id), updated_at = $4 WHERE id = $1",
[id, (int)state, frameId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
public Task> ListByUserAsync(Guid userId, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM uploads WHERE user_id = $1 ORDER BY created DESC", [userId], Map, cancellationToken);
/// Uploads still receiving but untouched since (abandoned; their part files can go).
public Task> ListStaleAsync(DateTimeOffset updatedBefore, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM uploads WHERE state = {(int)UploadState.Receiving} AND updated_at < $1 ORDER BY updated_at",
[updatedBefore], Map, cancellationToken);
public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM uploads WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
///
/// Moves an upload from to only if it is still in ,
/// so two concurrent completions cannot both commit it. Returns false when the state had changed.
///
public async Task TryTransitionAsync(Guid id, UploadState from, UploadState to, Guid? frameId = null, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE uploads SET state = $3, frame_id = COALESCE($4, frame_id), updated_at = $5 WHERE id = $1 AND state = $2",
[id, (int)from, (int)to, frameId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
/// The newest upload carrying the output of render job .
public Task FindByJobAsync(Guid jobId, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM uploads WHERE job_id = $1 ORDER BY created DESC, id DESC LIMIT 1", [jobId], Map, cancellationToken);
/// Every upload in , oldest change first.
public Task> ListByStateAsync(UploadState state, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM uploads WHERE state = $1 ORDER BY updated_at, id", [(int)state], Map, cancellationToken);
/// Bytes a user has announced in frame uploads that are not committed yet (reserved against the quota).
public async Task PendingBytesAsync(Guid userId, CancellationToken cancellationToken = default) =>
await _db.ScalarAsync(
$"SELECT COALESCE(SUM(size), 0) FROM uploads WHERE user_id = $1 AND job_id IS NULL AND state IN ({(int)UploadState.Receiving}, {(int)UploadState.Committing})",
[userId], cancellationToken).ConfigureAwait(false);
/// Committed frame uploads per UTC day since , oldest day first.
public Task> CommittedPerDayAsync(DateTimeOffset since, CancellationToken cancellationToken = default)
{
var day = _db.Sql.IsPostgres ? "to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD')" : "substr(updated_at, 1, 10)";
return _db.ListAsync(
$"SELECT {day}, count(*), COALESCE(SUM(size), 0) FROM uploads WHERE state = {(int)UploadState.Committed} AND job_id IS NULL AND updated_at >= $1 " +
"GROUP BY 1 ORDER BY 1",
[since], r => new UploadDay(DateOnly.ParseExact(r.GetString(0), "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture), r.GetInt64(1), r.GetInt64(2)),
cancellationToken);
}
/// Committed frame uploads per user since .
public async Task> CommittedCountByUserAsync(DateTimeOffset since, CancellationToken cancellationToken = default)
{
var rows = await _db.ListAsync(
$"SELECT user_id, count(*) FROM uploads WHERE state = {(int)UploadState.Committed} AND job_id IS NULL AND updated_at >= $1 GROUP BY user_id",
[since], r => KeyValuePair.Create(r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false);
return new Dictionary(rows);
}
/// Deletes committed, failed and cancelled uploads last changed before ; returns how many.
public Task PurgeFinishedAsync(DateTimeOffset updatedBefore, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync(
$"DELETE FROM uploads WHERE state IN ({(int)UploadState.Committed}, {(int)UploadState.Failed}, {(int)UploadState.Cancelled}) AND updated_at < $1",
[updatedBefore], cancellationToken);
private static Upload Map(DbRow r) => new()
{
Id = r.GetGuid(0),
UserId = r.GetGuid(1),
SequenceId = r.GetGuidOrNull(2),
FileName = r.GetString(3),
Size = r.GetInt64(4),
Sha256 = r.GetString(5),
CapturedAt = r.GetDateTimeOrNull(6),
ChunkSize = r.GetInt32(7),
Received = r.GetBytes(8),
ReceivedCount = r.GetInt32(9),
State = (UploadState)r.GetInt32(10),
FrameId = r.GetGuidOrNull(11),
Created = r.GetDateTimeOffset(12),
UpdatedAt = r.GetDateTimeOffset(13),
SessionKey = r.GetStringOrNull(14),
JobId = r.GetGuidOrNull(15),
RenderJson = r.GetStringOrNull(16),
};
}