- JobEngine: ISelectiveJobHandler lets a host decline jobs of a kind it runs; declined jobs stay queued for other claimers without holding up the rest. JobContext.EnqueueFollowUpAsync queues follow-up work on the same engine. - Server renders the jobs whose every output a managed encoder produces (ManagedRenderJobHandler); MP4 jobs stay for render nodes. Program.cs registers ProcessingModule next to ArchiveModule. Server renders ignore a client's export folder. - Light analysis is queued after imports and after scans that indexed new sequences, so the advisor's Auto values exist without user action. - Analysis jobs log a failing sequence and go on; the job then fails and a retry analyses only the failed ones. The advised frame rate follows the user's fit-to duration. - ProcessingJobs.RenderEach splits a batch into one job per preset for servers; render nodes say which batch presets they did not render, fetch each content once and stop on a changed certificate. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
224 lines
14 KiB
C#
224 lines
14 KiB
C#
using System.Globalization;
|
|
using Titano.Core.Model;
|
|
|
|
namespace Titano.Data.Repositories;
|
|
|
|
/// <summary>Jobs per state; <see cref="DoneSince"/> and <see cref="FailedSince"/> count those finished since a given time.</summary>
|
|
public sealed record JobCounts(int Queued, int Running, int Paused, int DoneSince, int FailedSince);
|
|
|
|
/// <summary>
|
|
/// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically,
|
|
/// write checkpoints while running, and on startup requeue what a crash left running.
|
|
/// </summary>
|
|
public sealed class JobRepository
|
|
{
|
|
private const string Columns =
|
|
"id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " +
|
|
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished";
|
|
|
|
private readonly DbAccess _db;
|
|
|
|
internal JobRepository(DbAccess db) => _db = db;
|
|
|
|
public async Task EnqueueAsync(Job job, CancellationToken cancellationToken = default) =>
|
|
await _db.ExecuteAsync(
|
|
$"INSERT INTO jobs ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)",
|
|
[
|
|
job.Id, (int)job.Kind, string.Join(',', job.TargetIds), job.TargetLabel, DbParam.Json(job.ParametersJson), (int)job.State,
|
|
(int)job.Priority, job.ProgressDone, job.ProgressTotal, job.ItemsPerSecond, job.Eta?.TotalSeconds, job.Checkpoint, job.LogTail,
|
|
job.CreatedBy, job.NodeId, job.Attempts, job.Error, job.Created, job.Started, job.Finished,
|
|
], cancellationToken).ConfigureAwait(false);
|
|
|
|
public Task<Job?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
|
|
_db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken);
|
|
|
|
/// <summary>
|
|
/// Marks the next queued job (highest priority, then oldest) of the given kinds as running for
|
|
/// <paramref name="workerId"/> and returns it, or null when none waits. Concurrent claimers never get the same
|
|
/// job: PostgreSQL skips rows locked by other claimers, SQLite holds its write lock across select and update.
|
|
/// </summary>
|
|
public Task<Job?> ClaimNextAsync(string workerId, IReadOnlyCollection<JobKind>? kinds = null, CancellationToken cancellationToken = default)
|
|
{
|
|
var kindFilter = kinds is { Count: > 0 } ? $" AND kind IN ({KindList(kinds)})" : string.Empty;
|
|
var pick = $"SELECT id FROM jobs WHERE state = {(int)JobState.Queued}{kindFilter} ORDER BY priority DESC, created, id LIMIT 1";
|
|
var claim = $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $1, started = $2, finished = NULL, error = NULL, attempts = attempts + 1";
|
|
DbParam[] args = [workerId, DateTimeOffset.UtcNow];
|
|
if (_db.Sql.IsPostgres)
|
|
return _db.SingleAsync($"{claim} WHERE id = ({pick}{_db.Sql.ForUpdateSkipLocked}) RETURNING {Columns}", args, Map, cancellationToken);
|
|
|
|
return _db.InTransactionAsync(async session =>
|
|
{
|
|
var id = await session.ScalarAsync<Guid?>(pick, [], cancellationToken).ConfigureAwait(false);
|
|
if (id is null) return null;
|
|
await session.ExecuteAsync($"{claim} WHERE id = $3", [.. args, id], cancellationToken).ConfigureAwait(false);
|
|
return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false);
|
|
}, cancellationToken);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Checkpoint of a running job. Returns false when the job is no longer running (cancelled or paused meanwhile),
|
|
/// which tells the worker to stop.
|
|
/// </summary>
|
|
public async Task<bool> UpdateProgressAsync(Guid id, long done, long total, double? itemsPerSecond, TimeSpan? eta,
|
|
string? checkpoint, string? logTail, CancellationToken cancellationToken = default) =>
|
|
await _db.ExecuteAsync(
|
|
"UPDATE jobs SET progress_done = $2, progress_total = $3, items_per_second = $4, eta_seconds = $5, checkpoint = $6, log_tail = $7 " +
|
|
$"WHERE id = $1 AND state = {(int)JobState.Running}",
|
|
[id, done, total, itemsPerSecond, eta?.TotalSeconds, checkpoint, logTail], cancellationToken).ConfigureAwait(false) > 0;
|
|
|
|
/// <summary>Moves a job to <paramref name="state"/>; done/failed/cancelled stamp the finish time, queued clears it for a retry.</summary>
|
|
public async Task<bool> SetStateAsync(Guid id, JobState state, string? error = null, CancellationToken cancellationToken = default)
|
|
{
|
|
var terminal = state is JobState.Done or JobState.Failed or JobState.Cancelled;
|
|
return await _db.ExecuteAsync("UPDATE jobs SET state = $2, error = $3, finished = $4 WHERE id = $1",
|
|
[id, (int)state, error, terminal ? DateTimeOffset.UtcNow : (DateTimeOffset?)null], cancellationToken).ConfigureAwait(false) > 0;
|
|
}
|
|
|
|
/// <summary>Jobs, newest first, optionally of one state.</summary>
|
|
public Task<Page<Job>> ListAsync(JobState? state = null, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
|
|
ListAsync(state, kind: null, limit, cursor, cancellationToken);
|
|
|
|
/// <summary>Jobs, newest first, optionally of one state and one kind.</summary>
|
|
public Task<Page<Job>> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default)
|
|
{
|
|
var p = new SqlParameters();
|
|
var where = new List<string>();
|
|
if (state is { } s) where.Add($"state = {p.Add((int)s)}");
|
|
if (kind is { } k) where.Add($"kind = {p.Add((int)k)}");
|
|
if (cursor is not null)
|
|
{
|
|
var (value, id) = Cursor.Decode("jobs", cursor);
|
|
where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add));
|
|
}
|
|
var take = Cursor.ClampLimit(limit);
|
|
var sql = $"SELECT {Columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
|
|
$"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}";
|
|
return _db.RunAsync(async session =>
|
|
{
|
|
var jobs = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false);
|
|
string? next = null;
|
|
if (jobs.Count > take)
|
|
{
|
|
jobs.RemoveAt(take);
|
|
next = Cursor.Encode("jobs", jobs[^1].Created, jobs[^1].Id);
|
|
}
|
|
return new Page<Job>(jobs, next);
|
|
}, cancellationToken);
|
|
}
|
|
|
|
/// <summary>Deletes finished jobs (done, failed, cancelled) that finished before <paramref name="finishedBefore"/>.</summary>
|
|
public Task<int> PurgeHistoryAsync(DateTimeOffset finishedBefore, CancellationToken cancellationToken = default) =>
|
|
_db.ExecuteAsync(
|
|
$"DELETE FROM jobs WHERE state IN ({(int)JobState.Done}, {(int)JobState.Failed}, {(int)JobState.Cancelled}) AND finished < $1",
|
|
[finishedBefore], cancellationToken);
|
|
|
|
/// <summary>
|
|
/// Crash recovery: puts running jobs back in the queue, keeping their checkpoints. With <paramref name="nodeId"/>
|
|
/// only that node's jobs are requeued, including those of its numbered workers (<c>node#0</c>, <c>node#1</c>, …);
|
|
/// other server instances keep theirs.
|
|
/// </summary>
|
|
public Task<int> RequeueRunningAsync(string? nodeId = null, CancellationToken cancellationToken = default) =>
|
|
_db.ExecuteAsync(
|
|
$"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE state = {(int)JobState.Running}" +
|
|
(nodeId is null ? string.Empty : " AND (node_id = $1 OR substr(node_id, 1, length($2)) = $2)"),
|
|
nodeId is null ? [] : [nodeId, nodeId + "#"], cancellationToken);
|
|
|
|
public async Task<bool> SetPriorityAsync(Guid id, JobPriority priority, CancellationToken cancellationToken = default) =>
|
|
await _db.ExecuteAsync("UPDATE jobs SET priority = $2 WHERE id = $1", [id, (int)priority], cancellationToken).ConfigureAwait(false) > 0;
|
|
|
|
/// <summary>Queued jobs of one kind in claim order (highest priority, then oldest).</summary>
|
|
public Task<IReadOnlyList<Job>> ListQueuedAsync(JobKind kind, int limit = 50, CancellationToken cancellationToken = default) =>
|
|
ListQueuedAsync([kind], limit, cancellationToken);
|
|
|
|
/// <summary>Queued jobs of the given kinds in claim order (highest priority, then oldest).</summary>
|
|
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit = 50, CancellationToken cancellationToken = default) =>
|
|
kinds.Count == 0 ? Task.FromResult<IReadOnlyList<Job>>([]) : _db.ListAsync(
|
|
$"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY priority DESC, created, id LIMIT $1",
|
|
[Cursor.ClampLimit(limit)], Map, cancellationToken);
|
|
|
|
/// <summary>Running jobs claimed by <paramref name="nodeId"/>.</summary>
|
|
public Task<IReadOnlyList<Job>> ListRunningByNodeAsync(string nodeId, CancellationToken cancellationToken = default) =>
|
|
_db.ListAsync($"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Running} AND node_id = $1 ORDER BY started, id",
|
|
[nodeId], Map, cancellationToken);
|
|
|
|
/// <summary>True when a job of <paramref name="kind"/> for exactly <paramref name="targetId"/> is waiting in the queue.</summary>
|
|
public async Task<bool> IsQueuedAsync(JobKind kind, Guid targetId, CancellationToken cancellationToken = default) =>
|
|
await _db.ScalarAsync<long>(
|
|
$"SELECT count(*) FROM jobs WHERE state = {(int)JobState.Queued} AND kind = $1 AND target_ids = $2",
|
|
[(int)kind, targetId.ToString()], cancellationToken).ConfigureAwait(false) > 0;
|
|
|
|
/// <summary>
|
|
/// Claims one specific queued job for <paramref name="nodeId"/> (render nodes pick by capability); null when another
|
|
/// claimer was faster or the job is no longer queued.
|
|
/// </summary>
|
|
public Task<Job?> TryClaimAsync(Guid id, string nodeId, CancellationToken cancellationToken = default) =>
|
|
_db.InTransactionAsync(async session =>
|
|
{
|
|
var claimed = await session.ExecuteAsync(
|
|
$"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $2, started = $3, finished = NULL, error = NULL, attempts = attempts + 1 " +
|
|
$"WHERE id = $1 AND state = {(int)JobState.Queued}",
|
|
[id, nodeId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false);
|
|
if (claimed == 0) return null;
|
|
return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false);
|
|
}, cancellationToken);
|
|
|
|
/// <summary>Progress of a job a render node runs; false when the job is no longer running on that node.</summary>
|
|
public async Task<bool> UpdateNodeProgressAsync(Guid id, string nodeId, long done, long total, double? itemsPerSecond, TimeSpan? eta,
|
|
string? logTail, CancellationToken cancellationToken = default) =>
|
|
await _db.ExecuteAsync(
|
|
"UPDATE jobs SET progress_done = $3, progress_total = $4, items_per_second = $5, eta_seconds = $6, log_tail = $7 " +
|
|
$"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
|
|
[id, nodeId, done, total, itemsPerSecond, eta?.TotalSeconds, logTail], cancellationToken).ConfigureAwait(false) > 0;
|
|
|
|
/// <summary>Ends a job a render node runs (done or failed); false when it is no longer running on that node.</summary>
|
|
public async Task<bool> FinishNodeJobAsync(Guid id, string nodeId, JobState state, string? error, string? logTail, CancellationToken cancellationToken = default)
|
|
{
|
|
if (state is not (JobState.Done or JobState.Failed)) throw new ArgumentOutOfRangeException(nameof(state), state, "Only done or failed end a node job.");
|
|
return await _db.ExecuteAsync(
|
|
"UPDATE jobs SET state = $3, error = $4, finished = $5, log_tail = COALESCE($6, log_tail) " +
|
|
$"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
|
|
[id, nodeId, (int)state, error, DateTimeOffset.UtcNow, logTail], cancellationToken).ConfigureAwait(false) > 0;
|
|
}
|
|
|
|
/// <summary>Queue and history counters for dashboards; finished counts are those since <paramref name="finishedSince"/>.</summary>
|
|
public Task<JobCounts> CountAsync(DateTimeOffset finishedSince, CancellationToken cancellationToken = default) =>
|
|
_db.RunAsync(async session =>
|
|
{
|
|
var row = await session.QuerySingleAsync(
|
|
"SELECT " +
|
|
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Queued} THEN 1 ELSE 0 END), 0), " +
|
|
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Running} THEN 1 ELSE 0 END), 0), " +
|
|
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Paused} THEN 1 ELSE 0 END), 0), " +
|
|
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Done} AND finished >= $1 THEN 1 ELSE 0 END), 0), " +
|
|
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Failed} AND finished >= $1 THEN 1 ELSE 0 END), 0) " +
|
|
"FROM jobs", [finishedSince], cancellationToken).ConfigureAwait(false);
|
|
return new JobCounts(row.GetInt32(0), row.GetInt32(1), row.GetInt32(2), row.GetInt32(3), row.GetInt32(4));
|
|
}, cancellationToken);
|
|
|
|
private static string KindList(IEnumerable<JobKind> kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture)));
|
|
|
|
private static Job Map(DbRow r) => new()
|
|
{
|
|
Id = r.GetGuid(0),
|
|
Kind = (JobKind)r.GetInt32(1),
|
|
TargetIds = r.GetString(2) is { Length: > 0 } ids ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [],
|
|
TargetLabel = r.GetStringOrNull(3),
|
|
ParametersJson = r.GetStringOrNull(4),
|
|
State = (JobState)r.GetInt32(5),
|
|
Priority = (JobPriority)r.GetInt32(6),
|
|
ProgressDone = r.GetInt64(7),
|
|
ProgressTotal = r.GetInt64(8),
|
|
ItemsPerSecond = r.GetDoubleOrNull(9),
|
|
Eta = r.GetDoubleOrNull(10) is { } seconds ? TimeSpan.FromSeconds(seconds) : null,
|
|
Checkpoint = r.GetStringOrNull(11),
|
|
LogTail = r.GetStringOrNull(12),
|
|
CreatedBy = r.GetGuidOrNull(13),
|
|
NodeId = r.GetStringOrNull(14),
|
|
Attempts = r.GetInt32(15),
|
|
Error = r.GetStringOrNull(16),
|
|
Created = r.GetDateTimeOffset(17),
|
|
Started = r.GetDateTimeOffsetOrNull(18),
|
|
Finished = r.GetDateTimeOffsetOrNull(19),
|
|
};
|
|
}
|