using System.Globalization;
using Titano.Core.Model;
namespace Titano.Data.Repositories;
/// Jobs per state; and count those finished since a given time.
public sealed record JobCounts(int Queued, int Running, int Paused, int DoneSince, int FailedSince);
///
/// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically,
/// write checkpoints while running, and on startup requeue what a crash left running.
///
public sealed class JobRepository
{
private const string Columns =
"id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " +
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished";
private readonly DbAccess _db;
internal JobRepository(DbAccess db) => _db = db;
public async Task EnqueueAsync(Job job, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
$"INSERT INTO jobs ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)",
[
job.Id, (int)job.Kind, string.Join(',', job.TargetIds), job.TargetLabel, DbParam.Json(job.ParametersJson), (int)job.State,
(int)job.Priority, job.ProgressDone, job.ProgressTotal, job.ItemsPerSecond, job.Eta?.TotalSeconds, job.Checkpoint, job.LogTail,
job.CreatedBy, job.NodeId, job.Attempts, job.Error, job.Created, job.Started, job.Finished,
], cancellationToken).ConfigureAwait(false);
public Task GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken);
///
/// Marks the next queued job (highest priority, then oldest) of the given kinds as running for
/// and returns it, or null when none waits. Concurrent claimers never get the same
/// job: PostgreSQL skips rows locked by other claimers, SQLite holds its write lock across select and update.
///
public Task ClaimNextAsync(string workerId, IReadOnlyCollection? kinds = null, CancellationToken cancellationToken = default)
{
var kindFilter = kinds is { Count: > 0 } ? $" AND kind IN ({KindList(kinds)})" : string.Empty;
var pick = $"SELECT id FROM jobs WHERE state = {(int)JobState.Queued}{kindFilter} ORDER BY priority DESC, created, id LIMIT 1";
var claim = $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $1, started = $2, finished = NULL, error = NULL, attempts = attempts + 1";
DbParam[] args = [workerId, DateTimeOffset.UtcNow];
if (_db.Sql.IsPostgres)
return _db.SingleAsync($"{claim} WHERE id = ({pick}{_db.Sql.ForUpdateSkipLocked}) RETURNING {Columns}", args, Map, cancellationToken);
return _db.InTransactionAsync(async session =>
{
var id = await session.ScalarAsync(pick, [], cancellationToken).ConfigureAwait(false);
if (id is null) return null;
await session.ExecuteAsync($"{claim} WHERE id = $3", [.. args, id], cancellationToken).ConfigureAwait(false);
return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false);
}, cancellationToken);
}
///
/// Checkpoint of a running job. Returns false when the job is no longer running (cancelled or paused meanwhile),
/// which tells the worker to stop.
///
public async Task UpdateProgressAsync(Guid id, long done, long total, double? itemsPerSecond, TimeSpan? eta,
string? checkpoint, string? logTail, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
"UPDATE jobs SET progress_done = $2, progress_total = $3, items_per_second = $4, eta_seconds = $5, checkpoint = $6, log_tail = $7 " +
$"WHERE id = $1 AND state = {(int)JobState.Running}",
[id, done, total, itemsPerSecond, eta?.TotalSeconds, checkpoint, logTail], cancellationToken).ConfigureAwait(false) > 0;
/// Moves a job to ; done/failed/cancelled stamp the finish time, queued clears it for a retry.
public async Task SetStateAsync(Guid id, JobState state, string? error = null, CancellationToken cancellationToken = default)
{
var terminal = state is JobState.Done or JobState.Failed or JobState.Cancelled;
return await _db.ExecuteAsync("UPDATE jobs SET state = $2, error = $3, finished = $4 WHERE id = $1",
[id, (int)state, error, terminal ? DateTimeOffset.UtcNow : (DateTimeOffset?)null], cancellationToken).ConfigureAwait(false) > 0;
}
/// Jobs, newest first, optionally of one state.
public Task> ListAsync(JobState? state = null, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
ListAsync(state, kind: null, limit, cursor, cancellationToken);
/// Jobs, newest first, optionally of one state and one kind.
public Task> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default)
{
var p = new SqlParameters();
var where = new List();
if (state is { } s) where.Add($"state = {p.Add((int)s)}");
if (kind is { } k) where.Add($"kind = {p.Add((int)k)}");
if (cursor is not null)
{
var (value, id) = Cursor.Decode("jobs", cursor);
where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add));
}
var take = Cursor.ClampLimit(limit);
var sql = $"SELECT {Columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
$"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}";
return _db.RunAsync(async session =>
{
var jobs = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false);
string? next = null;
if (jobs.Count > take)
{
jobs.RemoveAt(take);
next = Cursor.Encode("jobs", jobs[^1].Created, jobs[^1].Id);
}
return new Page(jobs, next);
}, cancellationToken);
}
/// Deletes finished jobs (done, failed, cancelled) that finished before .
public Task PurgeHistoryAsync(DateTimeOffset finishedBefore, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync(
$"DELETE FROM jobs WHERE state IN ({(int)JobState.Done}, {(int)JobState.Failed}, {(int)JobState.Cancelled}) AND finished < $1",
[finishedBefore], cancellationToken);
///
/// Crash recovery: puts running jobs back in the queue, keeping their checkpoints. With
/// only that node's jobs are requeued, including those of its numbered workers (node#0, node#1, …);
/// other server instances keep theirs.
///
public Task RequeueRunningAsync(string? nodeId = null, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync(
$"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE state = {(int)JobState.Running}" +
(nodeId is null ? string.Empty : " AND (node_id = $1 OR substr(node_id, 1, length($2)) = $2)"),
nodeId is null ? [] : [nodeId, nodeId + "#"], cancellationToken);
public async Task SetPriorityAsync(Guid id, JobPriority priority, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE jobs SET priority = $2 WHERE id = $1", [id, (int)priority], cancellationToken).ConfigureAwait(false) > 0;
/// Queued jobs of one kind in claim order (highest priority, then oldest).
public Task> ListQueuedAsync(JobKind kind, int limit = 50, CancellationToken cancellationToken = default) =>
ListQueuedAsync([kind], limit, cancellationToken);
/// Queued jobs of the given kinds in claim order (highest priority, then oldest).
public Task> ListQueuedAsync(IReadOnlyCollection kinds, int limit = 50, CancellationToken cancellationToken = default) =>
kinds.Count == 0 ? Task.FromResult>([]) : _db.ListAsync(
$"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY priority DESC, created, id LIMIT $1",
[Cursor.ClampLimit(limit)], Map, cancellationToken);
/// Running jobs claimed by .
public Task> ListRunningByNodeAsync(string nodeId, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Running} AND node_id = $1 ORDER BY started, id",
[nodeId], Map, cancellationToken);
/// True when a job of for exactly is waiting in the queue.
public async Task IsQueuedAsync(JobKind kind, Guid targetId, CancellationToken cancellationToken = default) =>
await _db.ScalarAsync(
$"SELECT count(*) FROM jobs WHERE state = {(int)JobState.Queued} AND kind = $1 AND target_ids = $2",
[(int)kind, targetId.ToString()], cancellationToken).ConfigureAwait(false) > 0;
///
/// Claims one specific queued job for (render nodes pick by capability); null when another
/// claimer was faster or the job is no longer queued.
///
public Task TryClaimAsync(Guid id, string nodeId, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var claimed = await session.ExecuteAsync(
$"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $2, started = $3, finished = NULL, error = NULL, attempts = attempts + 1 " +
$"WHERE id = $1 AND state = {(int)JobState.Queued}",
[id, nodeId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false);
if (claimed == 0) return null;
return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false);
}, cancellationToken);
/// Progress of a job a render node runs; false when the job is no longer running on that node.
public async Task UpdateNodeProgressAsync(Guid id, string nodeId, long done, long total, double? itemsPerSecond, TimeSpan? eta,
string? logTail, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
"UPDATE jobs SET progress_done = $3, progress_total = $4, items_per_second = $5, eta_seconds = $6, log_tail = $7 " +
$"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
[id, nodeId, done, total, itemsPerSecond, eta?.TotalSeconds, logTail], cancellationToken).ConfigureAwait(false) > 0;
/// Ends a job a render node runs (done or failed); false when it is no longer running on that node.
public async Task FinishNodeJobAsync(Guid id, string nodeId, JobState state, string? error, string? logTail, CancellationToken cancellationToken = default)
{
if (state is not (JobState.Done or JobState.Failed)) throw new ArgumentOutOfRangeException(nameof(state), state, "Only done or failed end a node job.");
return await _db.ExecuteAsync(
"UPDATE jobs SET state = $3, error = $4, finished = $5, log_tail = COALESCE($6, log_tail) " +
$"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
[id, nodeId, (int)state, error, DateTimeOffset.UtcNow, logTail], cancellationToken).ConfigureAwait(false) > 0;
}
/// Queue and history counters for dashboards; finished counts are those since .
public Task CountAsync(DateTimeOffset finishedSince, CancellationToken cancellationToken = default) =>
_db.RunAsync(async session =>
{
var row = await session.QuerySingleAsync(
"SELECT " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Queued} THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Running} THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Paused} THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Done} AND finished >= $1 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Failed} AND finished >= $1 THEN 1 ELSE 0 END), 0) " +
"FROM jobs", [finishedSince], cancellationToken).ConfigureAwait(false);
return new JobCounts(row.GetInt32(0), row.GetInt32(1), row.GetInt32(2), row.GetInt32(3), row.GetInt32(4));
}, cancellationToken);
private static string KindList(IEnumerable kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture)));
private static Job Map(DbRow r) => new()
{
Id = r.GetGuid(0),
Kind = (JobKind)r.GetInt32(1),
TargetIds = r.GetString(2) is { Length: > 0 } ids ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [],
TargetLabel = r.GetStringOrNull(3),
ParametersJson = r.GetStringOrNull(4),
State = (JobState)r.GetInt32(5),
Priority = (JobPriority)r.GetInt32(6),
ProgressDone = r.GetInt64(7),
ProgressTotal = r.GetInt64(8),
ItemsPerSecond = r.GetDoubleOrNull(9),
Eta = r.GetDoubleOrNull(10) is { } seconds ? TimeSpan.FromSeconds(seconds) : null,
Checkpoint = r.GetStringOrNull(11),
LogTail = r.GetStringOrNull(12),
CreatedBy = r.GetGuidOrNull(13),
NodeId = r.GetStringOrNull(14),
Attempts = r.GetInt32(15),
Error = r.GetStringOrNull(16),
Created = r.GetDateTimeOffset(17),
Started = r.GetDateTimeOffsetOrNull(18),
Finished = r.GetDateTimeOffsetOrNull(19),
};
}