feat(data): add migrations, repositories and TitanoDatabase facade

Embedded per-dialect migrations applied in one transaction and recorded
in schema_migrations (advisory lock on PostgreSQL, newer schemas
refused). Schema for every BRIEF section 5 entity plus import history and
uploads. Hand-written repositories for archives, sequences (filters,
keyset paging, Archive statistics), frames (bulk insert and updates),
renders, jobs (exclusive claiming, checkpoints, crash recovery), media
cards, users, tokens, render nodes, settings, tags, audit and uploads.
TitanoDatabase opens, migrates and exposes them, also bound to a
transaction.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 10:05:43 +00:00
1 parent 14567ecac3
commit 8d93abd65b
18 files changed
+2334

No files matched your search

@@ -0,0 +1,142 @@
using System.Globalization;
using Titano.Core.Model;
namespace Titano.Data.Repositories;
/// <summary>
/// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically,
/// write checkpoints while running, and on startup requeue what a crash left running.
/// </summary>
public sealed class JobRepository
{
private const string Columns =
"id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " +
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished";
private readonly DbAccess _db;
internal JobRepository(DbAccess db) => _db = db;
public async Task EnqueueAsync(Job job, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
$"INSERT INTO jobs ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)",
[
job.Id, (int)job.Kind, string.Join(',', job.TargetIds), job.TargetLabel, DbParam.Json(job.ParametersJson), (int)job.State,
(int)job.Priority, job.ProgressDone, job.ProgressTotal, job.ItemsPerSecond, job.Eta?.TotalSeconds, job.Checkpoint, job.LogTail,
job.CreatedBy, job.NodeId, job.Attempts, job.Error, job.Created, job.Started, job.Finished,
], cancellationToken).ConfigureAwait(false);
public Task<Job?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken);
/// <summary>
/// Marks the next queued job (highest priority, then oldest) of the given kinds as running for
/// <paramref name="workerId"/> and returns it, or null when none waits. Concurrent claimers never get the same
/// job: PostgreSQL skips rows locked by other claimers, SQLite holds its write lock across select and update.
/// </summary>
public Task<Job?> ClaimNextAsync(string workerId, IReadOnlyCollection<JobKind>? kinds = null, CancellationToken cancellationToken = default)
{
var kindFilter = kinds is { Count: > 0 }
? " AND kind IN (" + string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture))) + ")"
: string.Empty;
var pick = $"SELECT id FROM jobs WHERE state = {(int)JobState.Queued}{kindFilter} ORDER BY priority DESC, created, id LIMIT 1";
var claim = $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $1, started = $2, finished = NULL, error = NULL, attempts = attempts + 1";
DbParam[] args = [workerId, DateTimeOffset.UtcNow];
if (_db.Sql.IsPostgres)
return _db.SingleAsync($"{claim} WHERE id = ({pick} FOR UPDATE SKIP LOCKED) RETURNING {Columns}", args, Map, cancellationToken);
return _db.InTransactionAsync(async session =>
{
var id = await session.ScalarAsync<Guid?>(pick, [], cancellationToken).ConfigureAwait(false);
if (id is null) return null;
await session.ExecuteAsync($"{claim} WHERE id = $3", [.. args, id], cancellationToken).ConfigureAwait(false);
return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false);
}, cancellationToken);
}
/// <summary>
/// Checkpoint of a running job. Returns false when the job is no longer running (cancelled or paused meanwhile),
/// which tells the worker to stop.
/// </summary>
public async Task<bool> UpdateProgressAsync(Guid id, long done, long total, double? itemsPerSecond, TimeSpan? eta,
string? checkpoint, string? logTail, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
"UPDATE jobs SET progress_done = $2, progress_total = $3, items_per_second = $4, eta_seconds = $5, checkpoint = $6, log_tail = $7 " +
$"WHERE id = $1 AND state = {(int)JobState.Running}",
[id, done, total, itemsPerSecond, eta?.TotalSeconds, checkpoint, logTail], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Moves a job to <paramref name="state"/>; done/failed/cancelled stamp the finish time, queued clears it for a retry.</summary>
public async Task<bool> SetStateAsync(Guid id, JobState state, string? error = null, CancellationToken cancellationToken = default)
{
var terminal = state is JobState.Done or JobState.Failed or JobState.Cancelled;
return await _db.ExecuteAsync("UPDATE jobs SET state = $2, error = $3, finished = $4 WHERE id = $1",
[id, (int)state, error, terminal ? DateTimeOffset.UtcNow : (DateTimeOffset?)null], cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>Jobs, newest first, optionally of one state.</summary>
public Task<Page<Job>> ListAsync(JobState? state = null, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default)
{
var p = new SqlParameters();
var where = new List<string>();
if (state is { } s) where.Add($"state = {p.Add((int)s)}");
if (cursor is not null)
{
var (value, id) = Cursor.Decode("jobs", cursor);
where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add));
}
var take = Cursor.ClampLimit(limit);
var sql = $"SELECT {Columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
$"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}";
return _db.RunAsync(async session =>
{
var jobs = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false);
string? next = null;
if (jobs.Count > take)
{
jobs.RemoveAt(take);
next = Cursor.Encode("jobs", jobs[^1].Created, jobs[^1].Id);
}
return new Page<Job>(jobs, next);
}, cancellationToken);
}
/// <summary>Deletes finished jobs (done, failed, cancelled) that finished before <paramref name="finishedBefore"/>.</summary>
public Task<int> PurgeHistoryAsync(DateTimeOffset finishedBefore, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync(
$"DELETE FROM jobs WHERE state IN ({(int)JobState.Done}, {(int)JobState.Failed}, {(int)JobState.Cancelled}) AND finished < $1",
[finishedBefore], cancellationToken);
/// <summary>
/// Crash recovery: puts running jobs back in the queue, keeping their checkpoints. With <paramref name="nodeId"/>
/// only that worker's jobs are requeued (other server instances keep theirs).
/// </summary>
public Task<int> RequeueRunningAsync(string? nodeId = null, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync(
$"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE state = {(int)JobState.Running}" +
(nodeId is null ? string.Empty : " AND node_id = $1"),
nodeId is null ? [] : [nodeId], cancellationToken);
private static Job Map(DbRow r) => new()
{
Id = r.GetGuid(0),
Kind = (JobKind)r.GetInt32(1),
TargetIds = r.GetString(2) is { Length: > 0 } ids ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [],
TargetLabel = r.GetStringOrNull(3),
ParametersJson = r.GetStringOrNull(4),
State = (JobState)r.GetInt32(5),
Priority = (JobPriority)r.GetInt32(6),
ProgressDone = r.GetInt64(7),
ProgressTotal = r.GetInt64(8),
ItemsPerSecond = r.GetDoubleOrNull(9),
Eta = r.GetDoubleOrNull(10) is { } seconds ? TimeSpan.FromSeconds(seconds) : null,
Checkpoint = r.GetStringOrNull(11),
LogTail = r.GetStringOrNull(12),
CreatedBy = r.GetGuidOrNull(13),
NodeId = r.GetStringOrNull(14),
Attempts = r.GetInt32(15),
Error = r.GetStringOrNull(16),
Created = r.GetDateTimeOffset(17),
Started = r.GetDateTimeOffsetOrNull(18),
Finished = r.GetDateTimeOffsetOrNull(19),
};
}