feat(data): server queries for uploads, jobs, render nodes and sequence edits
Additive repository methods the server needs: upload state transitions, pending bytes, per-day and per-user upload counts, render-result uploads; job claim by id, node progress and completion, priority, queue counts and requeue of numbered workers; sequence edits with settings_version optimistic concurrency; audit filter by action; latest token per user. Migration 0005 adds session_key, job_id and render_json to uploads. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
9 files changed
+282
-6
No files matched your search
@@ -0,0 +1,9 @@
|
||||
-- Titano Server additions (PostgreSQL): automatic grouping key of frame uploads, render-result uploads of
|
||||
-- render-node jobs, and the sign-in log lookup. Mirrors sqlite/0005_server_uploads.sql.
|
||||
|
||||
ALTER TABLE uploads ADD COLUMN session_key text;
|
||||
ALTER TABLE uploads ADD COLUMN job_id uuid;
|
||||
ALTER TABLE uploads ADD COLUMN render_json jsonb;
|
||||
CREATE INDEX ix_uploads_job ON uploads (job_id);
|
||||
|
||||
CREATE INDEX ix_audit_log_action ON audit_log (action, at);
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Titano Server additions (SQLite): automatic grouping key of frame uploads, render-result uploads of render-node
|
||||
-- jobs, and the sign-in log lookup. Mirrors postgres/0005_server_uploads.sql.
|
||||
|
||||
ALTER TABLE uploads ADD COLUMN session_key TEXT;
|
||||
ALTER TABLE uploads ADD COLUMN job_id TEXT;
|
||||
ALTER TABLE uploads ADD COLUMN render_json TEXT;
|
||||
CREATE INDEX ix_uploads_job ON uploads (job_id);
|
||||
|
||||
CREATE INDEX ix_audit_log_action ON audit_log (action, at);
|
||||
@@ -17,11 +17,16 @@ public sealed class AuditRepository
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
/// <summary>Entries newest first, optionally of one user.</summary>
|
||||
public Task<Page<AuditEntry>> ListAsync(Guid? userId = null, int limit = 100, string? cursor = null, CancellationToken cancellationToken = default)
|
||||
public Task<Page<AuditEntry>> ListAsync(Guid? userId = null, int limit = 100, string? cursor = null, CancellationToken cancellationToken = default) =>
|
||||
ListAsync(userId, action: null, limit, cursor, cancellationToken);
|
||||
|
||||
/// <summary>Entries newest first, optionally of one user and one action (e.g. the sign-in log).</summary>
|
||||
public Task<Page<AuditEntry>> ListAsync(Guid? userId, string? action, int limit = 100, string? cursor = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var p = new SqlParameters();
|
||||
var where = new List<string>();
|
||||
if (userId is { } user) where.Add($"user_id = {p.Add(user)}");
|
||||
if (action is not null) where.Add($"action = {p.Add(action)}");
|
||||
if (cursor is not null)
|
||||
{
|
||||
var (value, id) = Cursor.Decode("audit", cursor);
|
||||
|
||||
@@ -7,6 +7,9 @@ namespace Titano.Data.Repositories;
|
||||
/// <summary>Light statistics of one frame, written by the analysis job.</summary>
|
||||
public readonly record struct FrameLightStats(Guid FrameId, double? MeanLogLuminance, double? ClipLow, double? ClipHigh);
|
||||
|
||||
/// <summary>Per-format totals and flagged-frame counts of one sequence.</summary>
|
||||
public sealed record SequenceFrameStats(IReadOnlyList<FormatBucket> ByFormat, long Missing, long Corrupt, long Duplicate, long Excluded);
|
||||
|
||||
/// <summary>Frames of sequences: bulk import, paged listing, duplicate lookup and bulk updates.</summary>
|
||||
public sealed class FrameRepository
|
||||
{
|
||||
@@ -112,6 +115,23 @@ public sealed class FrameRepository
|
||||
return updated;
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Formats and flagged-frame counters of one sequence.</summary>
|
||||
public Task<SequenceFrameStats> GetSequenceStatsAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
|
||||
_db.RunAsync(async session =>
|
||||
{
|
||||
var byFormat = await session.QueryListAsync(
|
||||
"SELECT format, count(*), COALESCE(SUM(size), 0) FROM frames WHERE sequence_id = $1 GROUP BY format ORDER BY 2 DESC, 1",
|
||||
[sequenceId], r => new FormatBucket((ImageFileFormat)r.GetInt32(0), r.GetInt64(1), r.GetInt64(2)), cancellationToken).ConfigureAwait(false);
|
||||
var flags = await session.QuerySingleAsync(
|
||||
"SELECT " +
|
||||
$"COALESCE(SUM(CASE WHEN (flags & {(int)FrameFlags.Missing}) <> 0 THEN 1 ELSE 0 END), 0), " +
|
||||
$"COALESCE(SUM(CASE WHEN (flags & {(int)FrameFlags.Corrupt}) <> 0 THEN 1 ELSE 0 END), 0), " +
|
||||
$"COALESCE(SUM(CASE WHEN (flags & {(int)FrameFlags.Duplicate}) <> 0 THEN 1 ELSE 0 END), 0), " +
|
||||
$"COALESCE(SUM(CASE WHEN (flags & {(int)FrameFlags.Excluded}) <> 0 THEN 1 ELSE 0 END), 0) " +
|
||||
"FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
|
||||
return new SequenceFrameStats(byFormat, flags.GetInt64(0), flags.GetInt64(1), flags.GetInt64(2), flags.GetInt64(3));
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Removes every frame of a sequence; returns how many.</summary>
|
||||
public Task<int> DeleteBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken);
|
||||
|
||||
@@ -3,6 +3,9 @@ using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Repositories;
|
||||
|
||||
/// <summary>Jobs per state; <see cref="DoneSince"/> and <see cref="FailedSince"/> count those finished since a given time.</summary>
|
||||
public sealed record JobCounts(int Queued, int Running, int Paused, int DoneSince, int FailedSince);
|
||||
|
||||
/// <summary>
|
||||
/// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically,
|
||||
/// write checkpoints while running, and on startup requeue what a crash left running.
|
||||
@@ -74,11 +77,16 @@ public sealed class JobRepository
|
||||
}
|
||||
|
||||
/// <summary>Jobs, newest first, optionally of one state.</summary>
|
||||
public Task<Page<Job>> ListAsync(JobState? state = null, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default)
|
||||
public Task<Page<Job>> ListAsync(JobState? state = null, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
|
||||
ListAsync(state, kind: null, limit, cursor, cancellationToken);
|
||||
|
||||
/// <summary>Jobs, newest first, optionally of one state and one kind.</summary>
|
||||
public Task<Page<Job>> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var p = new SqlParameters();
|
||||
var where = new List<string>();
|
||||
if (state is { } s) where.Add($"state = {p.Add((int)s)}");
|
||||
if (kind is { } k) where.Add($"kind = {p.Add((int)k)}");
|
||||
if (cursor is not null)
|
||||
{
|
||||
var (value, id) = Cursor.Decode("jobs", cursor);
|
||||
@@ -116,6 +124,84 @@ public sealed class JobRepository
|
||||
(nodeId is null ? string.Empty : " AND node_id = $1"),
|
||||
nodeId is null ? [] : [nodeId], cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Crash recovery of one job engine: requeues running jobs claimed by <paramref name="workerId"/> or by its numbered
|
||||
/// workers (<c>workerId#n</c>), whatever their number was in the previous run.
|
||||
/// </summary>
|
||||
public Task<int> RequeueRunningWorkersAsync(string workerId, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync(
|
||||
$"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE state = {(int)JobState.Running} " +
|
||||
"AND (node_id = $1 OR node_id LIKE $2 ESCAPE '\\')",
|
||||
[workerId, SqlDialect.ContainsPattern(workerId)[1..^1] + "#%"], cancellationToken);
|
||||
|
||||
public async Task<bool> SetPriorityAsync(Guid id, JobPriority priority, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("UPDATE jobs SET priority = $2 WHERE id = $1", [id, (int)priority], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>Queued jobs of one kind in claim order (highest priority, then oldest).</summary>
|
||||
public Task<IReadOnlyList<Job>> ListQueuedAsync(JobKind kind, int limit = 50, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
$"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind = $1 ORDER BY priority DESC, created, id LIMIT $2",
|
||||
[(int)kind, Cursor.ClampLimit(limit)], Map, cancellationToken);
|
||||
|
||||
/// <summary>Running jobs claimed by <paramref name="nodeId"/>.</summary>
|
||||
public Task<IReadOnlyList<Job>> ListRunningByNodeAsync(string nodeId, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync($"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Running} AND node_id = $1 ORDER BY started, id",
|
||||
[nodeId], Map, cancellationToken);
|
||||
|
||||
/// <summary>True when a job of <paramref name="kind"/> for exactly <paramref name="targetId"/> is waiting in the queue.</summary>
|
||||
public async Task<bool> IsQueuedAsync(JobKind kind, Guid targetId, CancellationToken cancellationToken = default) =>
|
||||
await _db.ScalarAsync<long>(
|
||||
$"SELECT count(*) FROM jobs WHERE state = {(int)JobState.Queued} AND kind = $1 AND target_ids = $2",
|
||||
[(int)kind, targetId.ToString()], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Claims one specific queued job for <paramref name="nodeId"/> (render nodes pick by capability); null when another
|
||||
/// claimer was faster or the job is no longer queued.
|
||||
/// </summary>
|
||||
public Task<Job?> TryClaimAsync(Guid id, string nodeId, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
var claimed = await session.ExecuteAsync(
|
||||
$"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $2, started = $3, finished = NULL, error = NULL, attempts = attempts + 1 " +
|
||||
$"WHERE id = $1 AND state = {(int)JobState.Queued}",
|
||||
[id, nodeId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false);
|
||||
if (claimed == 0) return null;
|
||||
return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false);
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Progress of a job a render node runs; false when the job is no longer running on that node.</summary>
|
||||
public async Task<bool> UpdateNodeProgressAsync(Guid id, string nodeId, long done, long total, double? itemsPerSecond, TimeSpan? eta,
|
||||
string? logTail, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync(
|
||||
"UPDATE jobs SET progress_done = $3, progress_total = $4, items_per_second = $5, eta_seconds = $6, log_tail = $7 " +
|
||||
$"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
|
||||
[id, nodeId, done, total, itemsPerSecond, eta?.TotalSeconds, logTail], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>Ends a job a render node runs (done or failed); false when it is no longer running on that node.</summary>
|
||||
public async Task<bool> FinishNodeJobAsync(Guid id, string nodeId, JobState state, string? error, string? logTail, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (state is not (JobState.Done or JobState.Failed)) throw new ArgumentOutOfRangeException(nameof(state), state, "Only done or failed end a node job.");
|
||||
return await _db.ExecuteAsync(
|
||||
"UPDATE jobs SET state = $3, error = $4, finished = $5, log_tail = COALESCE($6, log_tail) " +
|
||||
$"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
|
||||
[id, nodeId, (int)state, error, DateTimeOffset.UtcNow, logTail], cancellationToken).ConfigureAwait(false) > 0;
|
||||
}
|
||||
|
||||
/// <summary>Queue and history counters for dashboards; finished counts are those since <paramref name="finishedSince"/>.</summary>
|
||||
public Task<JobCounts> CountAsync(DateTimeOffset finishedSince, CancellationToken cancellationToken = default) =>
|
||||
_db.RunAsync(async session =>
|
||||
{
|
||||
var row = await session.QuerySingleAsync(
|
||||
"SELECT " +
|
||||
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Queued} THEN 1 ELSE 0 END), 0), " +
|
||||
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Running} THEN 1 ELSE 0 END), 0), " +
|
||||
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Paused} THEN 1 ELSE 0 END), 0), " +
|
||||
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Done} AND finished >= $1 THEN 1 ELSE 0 END), 0), " +
|
||||
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Failed} AND finished >= $1 THEN 1 ELSE 0 END), 0) " +
|
||||
"FROM jobs", [finishedSince], cancellationToken).ConfigureAwait(false);
|
||||
return new JobCounts(row.GetInt32(0), row.GetInt32(1), row.GetInt32(2), row.GetInt32(3), row.GetInt32(4));
|
||||
}, cancellationToken);
|
||||
|
||||
private static Job Map(DbRow r) => new()
|
||||
{
|
||||
Id = r.GetGuid(0),
|
||||
|
||||
@@ -37,6 +37,15 @@ public sealed class RenderNodeRepository
|
||||
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("DELETE FROM render_nodes WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>Changes the state only (a node that stopped heart-beating keeps its last heartbeat time).</summary>
|
||||
public async Task<bool> SetStateAsync(Guid id, string state, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("UPDATE render_nodes SET state = $2 WHERE id = $1", [id, state], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>Nodes not in <paramref name="exceptState"/> whose last heartbeat is older than <paramref name="before"/>.</summary>
|
||||
public Task<IReadOnlyList<RenderNode>> ListSilentAsync(DateTimeOffset before, string exceptState, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync($"SELECT {Columns} FROM render_nodes WHERE last_heartbeat < $1 AND state <> $2 ORDER BY last_heartbeat, id",
|
||||
[before, exceptState], Map, cancellationToken);
|
||||
|
||||
private static RenderNode Map(DbRow r) => new()
|
||||
{
|
||||
Id = r.GetGuid(0),
|
||||
|
||||
@@ -60,6 +60,28 @@ public sealed record ArchiveStats(
|
||||
IReadOnlyList<FormatBucket> ByFormat,
|
||||
ArchiveHealth Health);
|
||||
|
||||
/// <summary>Fields a client may change on a sequence; null leaves a field unchanged, an empty title or notes clears it.</summary>
|
||||
public sealed record SequenceEdit
|
||||
{
|
||||
public string? Title { get; init; }
|
||||
public IReadOnlyList<string>? Tags { get; init; }
|
||||
public int? Rating { get; init; }
|
||||
public string? Notes { get; init; }
|
||||
public string? SettingsJson { get; init; }
|
||||
}
|
||||
|
||||
public enum SequenceEditOutcome
|
||||
{
|
||||
Updated,
|
||||
NotFound,
|
||||
|
||||
/// <summary>The settings version moved since the client read it.</summary>
|
||||
Conflict,
|
||||
}
|
||||
|
||||
/// <summary>Result of <see cref="SequenceRepository.EditAsync"/>: the updated sequence, or the stored one on conflict.</summary>
|
||||
public sealed record SequenceEditResult(SequenceEditOutcome Outcome, Sequence? Sequence);
|
||||
|
||||
public sealed record YearBucket(int Year, int Sequences, long Photos, long Bytes);
|
||||
|
||||
/// <summary>One heatmap cell: sequences and photos whose first capture falls in the month.</summary>
|
||||
@@ -172,6 +194,38 @@ public sealed class SequenceRepository
|
||||
return updated;
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Applies a client edit atomically and bumps <c>settings_version</c>. With <paramref name="expectedSettingsVersion"/>
|
||||
/// the edit is refused (<see cref="SequenceEditOutcome.Conflict"/>, carrying the stored sequence) when someone else
|
||||
/// changed the sequence first (BRIEF §14.5).
|
||||
/// </summary>
|
||||
public Task<SequenceEditResult> EditAsync(Guid id, SequenceEdit edit, int? expectedSettingsVersion, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (current is null) return new SequenceEditResult(SequenceEditOutcome.NotFound, null);
|
||||
if (expectedSettingsVersion is { } expected && expected != current.SettingsVersion)
|
||||
return new SequenceEditResult(SequenceEditOutcome.Conflict, current);
|
||||
var updated = current with
|
||||
{
|
||||
Title = edit.Title is null ? current.Title : NullIfEmpty(edit.Title),
|
||||
Notes = edit.Notes is null ? current.Notes : NullIfEmpty(edit.Notes),
|
||||
Rating = edit.Rating ?? current.Rating,
|
||||
SettingsJson = edit.SettingsJson ?? current.SettingsJson,
|
||||
SettingsVersion = current.SettingsVersion + 1,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
await UpsertRowAsync(session, updated, cancellationToken).ConfigureAwait(false);
|
||||
if (edit.Tags is not null) await ReplaceTagsAsync(session, id, edit.Tags, cancellationToken).ConfigureAwait(false);
|
||||
var stored = await session.QuerySingleAsync($"{_select} WHERE s.id = $1", [id], cancellationToken).ConfigureAwait(false);
|
||||
return new SequenceEditResult(SequenceEditOutcome.Updated, Map(stored));
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Gives every sequence owned by <paramref name="fromUserId"/> to <paramref name="toUserId"/>; returns how many.</summary>
|
||||
public Task<int> ReassignOwnerAsync(Guid fromUserId, Guid toUserId, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync("UPDATE sequences SET owner_user_id = $2 WHERE owner_user_id = $1", [fromUserId, toUserId], cancellationToken);
|
||||
|
||||
/// <summary>One page of sequences matching <paramref name="query"/>.</summary>
|
||||
public Task<Page<Sequence>> ListAsync(SequenceQuery query, CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -351,6 +405,8 @@ public sealed class SequenceRepository
|
||||
}
|
||||
}
|
||||
|
||||
private static string? NullIfEmpty(string value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
|
||||
/// <summary>Camera wall-clock times are stored without zone, whatever kind the caller's value has.</summary>
|
||||
internal static DateTime? Wall(DateTime? value) =>
|
||||
value is { } v ? DateTime.SpecifyKind(v, DateTimeKind.Unspecified) : null;
|
||||
|
||||
@@ -37,6 +37,22 @@ public sealed class TokenRepository
|
||||
public Task<int> RevokeAllAsync(Guid userId, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync("UPDATE tokens SET revoked = TRUE WHERE user_id = $1 AND revoked = FALSE", [userId], cancellationToken);
|
||||
|
||||
/// <summary>Revokes every live token of a user except <paramref name="keepTokenHash"/> (password changed in that session).</summary>
|
||||
public Task<int> RevokeAllExceptAsync(Guid userId, string keepTokenHash, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync("UPDATE tokens SET revoked = TRUE WHERE user_id = $1 AND token_hash <> $2 AND revoked = FALSE",
|
||||
[userId, keepTokenHash], cancellationToken);
|
||||
|
||||
/// <summary>The most recently used token of every user (last seen time and client), keyed by user id.</summary>
|
||||
public async Task<IReadOnlyDictionary<Guid, SessionToken>> LatestByUserAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var tokens = await _db.ListAsync(
|
||||
$"SELECT {Columns} FROM tokens t WHERE last_used = (SELECT MAX(last_used) FROM tokens l WHERE l.user_id = t.user_id)",
|
||||
[], Map, cancellationToken).ConfigureAwait(false);
|
||||
var latest = new Dictionary<Guid, SessionToken>();
|
||||
foreach (var token in tokens) latest[token.UserId] = token;
|
||||
return latest;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<SessionToken>> ListByUserAsync(Guid userId, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync($"SELECT {Columns} FROM tokens WHERE user_id = $1 ORDER BY last_used DESC", [userId], Map, cancellationToken);
|
||||
|
||||
|
||||
@@ -36,6 +36,15 @@ public sealed record Upload
|
||||
public DateTimeOffset Created { get; init; }
|
||||
public DateTimeOffset UpdatedAt { get; init; }
|
||||
|
||||
/// <summary>Automatic grouping key of a frame upload without a target sequence (BRIEF §14.4).</summary>
|
||||
public string? SessionKey { get; init; }
|
||||
|
||||
/// <summary>The render job whose output this upload carries; null for frame uploads.</summary>
|
||||
public Guid? JobId { get; init; }
|
||||
|
||||
/// <summary>Metadata of the render a render-result upload becomes (JSON).</summary>
|
||||
public string? RenderJson { get; init; }
|
||||
|
||||
public int ChunkCount => (int)((Size + ChunkSize - 1) / ChunkSize);
|
||||
|
||||
public bool IsComplete => ReceivedCount >= ChunkCount;
|
||||
@@ -44,11 +53,15 @@ public sealed record Upload
|
||||
index >= 0 && index / 8 < Received.Length && (Received[index / 8] & (1 << (index % 8))) != 0;
|
||||
}
|
||||
|
||||
/// <summary>Committed uploads of one UTC day.</summary>
|
||||
public sealed record UploadDay(DateOnly Day, long Count, long Bytes);
|
||||
|
||||
/// <summary>Resumable uploads: creation, per-chunk bookkeeping, state changes and cleanup.</summary>
|
||||
public sealed class UploadRepository
|
||||
{
|
||||
private const string Columns =
|
||||
"id, user_id, sequence_id, file_name, size, sha256, captured_at, chunk_size, received, received_count, state, frame_id, created, updated_at";
|
||||
"id, user_id, sequence_id, file_name, size, sha256, captured_at, chunk_size, received, received_count, state, frame_id, created, updated_at, " +
|
||||
"session_key, job_id, render_json";
|
||||
|
||||
private readonly DbAccess _db;
|
||||
|
||||
@@ -61,10 +74,11 @@ public sealed class UploadRepository
|
||||
ArgumentOutOfRangeException.ThrowIfNegative(upload.Size);
|
||||
var stored = upload with { Received = new byte[(upload.ChunkCount + 7) / 8], ReceivedCount = 0, Sha256 = upload.Sha256.ToLowerInvariant() };
|
||||
await _db.ExecuteAsync(
|
||||
$"INSERT INTO uploads ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)",
|
||||
$"INSERT INTO uploads ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)",
|
||||
[
|
||||
stored.Id, stored.UserId, stored.SequenceId, stored.FileName, stored.Size, stored.Sha256, SequenceRepository.Wall(stored.CapturedAt),
|
||||
stored.ChunkSize, stored.Received, 0, (int)stored.State, stored.FrameId, stored.Created, stored.UpdatedAt,
|
||||
stored.SessionKey, stored.JobId, DbParam.Json(stored.RenderJson),
|
||||
], cancellationToken).ConfigureAwait(false);
|
||||
return stored;
|
||||
}
|
||||
@@ -72,10 +86,11 @@ public sealed class UploadRepository
|
||||
public Task<Upload?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
|
||||
_db.SingleAsync($"SELECT {Columns} FROM uploads WHERE id = $1", [id], Map, cancellationToken);
|
||||
|
||||
/// <summary>An upload of the same content by the same user that is still receiving, so a client can resume it.</summary>
|
||||
/// <summary>A frame upload of the same content by the same user that is still receiving, so a client can resume it.</summary>
|
||||
public Task<Upload?> FindReceivingAsync(Guid userId, string sha256, CancellationToken cancellationToken = default) =>
|
||||
_db.SingleAsync(
|
||||
$"SELECT {Columns} FROM uploads WHERE user_id = $1 AND sha256 = $2 AND state = {(int)UploadState.Receiving} ORDER BY created DESC LIMIT 1",
|
||||
$"SELECT {Columns} FROM uploads WHERE user_id = $1 AND sha256 = $2 AND state = {(int)UploadState.Receiving} AND job_id IS NULL " +
|
||||
"ORDER BY created DESC LIMIT 1",
|
||||
[userId, sha256.ToLowerInvariant()], Map, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
@@ -115,6 +130,54 @@ public sealed class UploadRepository
|
||||
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("DELETE FROM uploads WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Moves an upload from <paramref name="from"/> to <paramref name="to"/> only if it is still in <paramref name="from"/>,
|
||||
/// so two concurrent completions cannot both commit it. Returns false when the state had changed.
|
||||
/// </summary>
|
||||
public async Task<bool> TryTransitionAsync(Guid id, UploadState from, UploadState to, Guid? frameId = null, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("UPDATE uploads SET state = $3, frame_id = COALESCE($4, frame_id), updated_at = $5 WHERE id = $1 AND state = $2",
|
||||
[id, (int)from, (int)to, frameId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>The newest upload carrying the output of render job <paramref name="jobId"/>.</summary>
|
||||
public Task<Upload?> FindByJobAsync(Guid jobId, CancellationToken cancellationToken = default) =>
|
||||
_db.SingleAsync($"SELECT {Columns} FROM uploads WHERE job_id = $1 ORDER BY created DESC, id DESC LIMIT 1", [jobId], Map, cancellationToken);
|
||||
|
||||
/// <summary>Every upload in <paramref name="state"/>, oldest change first.</summary>
|
||||
public Task<IReadOnlyList<Upload>> ListByStateAsync(UploadState state, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync($"SELECT {Columns} FROM uploads WHERE state = $1 ORDER BY updated_at, id", [(int)state], Map, cancellationToken);
|
||||
|
||||
/// <summary>Bytes a user has announced in frame uploads that are not committed yet (reserved against the quota).</summary>
|
||||
public async Task<long> PendingBytesAsync(Guid userId, CancellationToken cancellationToken = default) =>
|
||||
await _db.ScalarAsync<long>(
|
||||
$"SELECT COALESCE(SUM(size), 0) FROM uploads WHERE user_id = $1 AND job_id IS NULL AND state IN ({(int)UploadState.Receiving}, {(int)UploadState.Committing})",
|
||||
[userId], cancellationToken).ConfigureAwait(false);
|
||||
|
||||
/// <summary>Committed frame uploads per UTC day since <paramref name="since"/>, oldest day first.</summary>
|
||||
public Task<IReadOnlyList<UploadDay>> CommittedPerDayAsync(DateTimeOffset since, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var day = _db.Sql.IsPostgres ? "to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD')" : "substr(updated_at, 1, 10)";
|
||||
return _db.ListAsync(
|
||||
$"SELECT {day}, count(*), COALESCE(SUM(size), 0) FROM uploads WHERE state = {(int)UploadState.Committed} AND job_id IS NULL AND updated_at >= $1 " +
|
||||
"GROUP BY 1 ORDER BY 1",
|
||||
[since], r => new UploadDay(DateOnly.ParseExact(r.GetString(0), "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture), r.GetInt64(1), r.GetInt64(2)),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Committed frame uploads per user since <paramref name="since"/>.</summary>
|
||||
public async Task<IReadOnlyDictionary<Guid, long>> CommittedCountByUserAsync(DateTimeOffset since, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var rows = await _db.ListAsync(
|
||||
$"SELECT user_id, count(*) FROM uploads WHERE state = {(int)UploadState.Committed} AND job_id IS NULL AND updated_at >= $1 GROUP BY user_id",
|
||||
[since], r => KeyValuePair.Create(r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false);
|
||||
return new Dictionary<Guid, long>(rows);
|
||||
}
|
||||
|
||||
/// <summary>Deletes committed, failed and cancelled uploads last changed before <paramref name="updatedBefore"/>; returns how many.</summary>
|
||||
public Task<int> PurgeFinishedAsync(DateTimeOffset updatedBefore, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync(
|
||||
$"DELETE FROM uploads WHERE state IN ({(int)UploadState.Committed}, {(int)UploadState.Failed}, {(int)UploadState.Cancelled}) AND updated_at < $1",
|
||||
[updatedBefore], cancellationToken);
|
||||
|
||||
private static Upload Map(DbRow r) => new()
|
||||
{
|
||||
Id = r.GetGuid(0),
|
||||
@@ -131,5 +194,8 @@ public sealed class UploadRepository
|
||||
FrameId = r.GetGuidOrNull(11),
|
||||
Created = r.GetDateTimeOffset(12),
|
||||
UpdatedAt = r.GetDateTimeOffset(13),
|
||||
SessionKey = r.GetStringOrNull(14),
|
||||
JobId = r.GetGuidOrNull(15),
|
||||
RenderJson = r.GetStringOrNull(16),
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user