using System.Globalization; using Chronos.Core.Model; using Chronos.Data.Jobs; namespace Chronos.Data.Repositories; /// Jobs per state; and count those finished since a given time. public sealed record JobCounts(int Queued, int Running, int Paused, int DoneSince, int FailedSince); /// /// What a running or paused job works on (): its targets and, for a verify, its /// parameters, which name the sequences it holds; null for every other kind, whose parameters are not read. /// public sealed record JobHold(Guid Id, JobKind Kind, IReadOnlyList TargetIds, string? ParametersJson); /// /// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically, /// write checkpoints while running, and on startup requeue what a crash left running. /// public sealed class JobRepository { private const string Columns = "id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " + "eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id"; /// /// as lists read them (): the parameters left out, null. A conversion's or /// an import's name every sequence or file they work on; the job reads them, and so do the screens that act on that one job. /// private const string BriefColumns = "id, kind, target_ids, target_label, CAST(NULL AS TEXT), state, priority, progress_done, progress_total, items_per_second, " + "eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id"; /// The states of a job that has not ended: queued, running, paused. private static readonly string PendingStates = string.Join(", ", new[] { JobState.Queued, JobState.Running, JobState.Paused }.Select(s => ((int)s).ToString(CultureInfo.InvariantCulture))); private readonly DbAccess _db; internal JobRepository(DbAccess db) => _db = db; public async Task EnqueueAsync(Job job, CancellationToken cancellationToken = default) => await _db.ExecuteAsync( $"INSERT INTO jobs ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)", [ job.Id, (int)job.Kind, string.Join(',', job.TargetIds), job.TargetLabel, DbParam.Json(job.ParametersJson), (int)job.State, (int)job.Priority, job.ProgressDone, job.ProgressTotal, job.ItemsPerSecond, job.Eta?.TotalSeconds, job.Checkpoint, job.LogTail, job.CreatedBy, job.NodeId, job.Attempts, job.Error, job.Created, job.Started, job.Finished, job.ScheduleId, ], cancellationToken).ConfigureAwait(false); public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => _db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken); /// When the job was queued; null when there is no such job. Reads that column alone. public async Task GetCreatedAsync(Guid id, CancellationToken cancellationToken = default) => (await _db.ListAsync("SELECT created FROM jobs WHERE id = $1", [id], r => r.GetDateTimeOffset(0), cancellationToken).ConfigureAwait(false)) is [var created] ? created : null; /// /// Marks the next queued job (highest priority, then oldest) of the given kinds as running for /// and returns it, or null when none waits. Concurrent claimers never get the same /// job: PostgreSQL skips rows locked by other claimers, SQLite holds its write lock across select and update. /// public Task ClaimNextAsync(string workerId, IReadOnlyCollection? kinds = null, CancellationToken cancellationToken = default) => ClaimNextAsync(workerId, kinds, JobPriorityQueue.Off, cancellationToken); /// /// in the order of /// : while it is on, the kinds of the highest level first (). /// public Task ClaimNextAsync(string workerId, IReadOnlyCollection? kinds, JobPriorityQueue queue, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(queue); var kindFilter = kinds is { Count: > 0 } ? $" AND kind IN ({KindList(kinds)})" : string.Empty; var pick = $"SELECT id FROM jobs WHERE state = {(int)JobState.Queued}{kindFilter} ORDER BY {queue.OrderSql(kinds is { Count: > 0 } ? kinds : Enum.GetValues())}" + "priority DESC, created, id LIMIT 1"; var claim = $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $1, started = $2, finished = NULL, error = NULL, attempts = attempts + 1"; DbParam[] args = [workerId, DateTimeOffset.UtcNow]; if (_db.Sql.IsPostgres) return _db.SingleAsync($"{claim} WHERE id = ({pick}{_db.Sql.ForUpdateSkipLocked}) RETURNING {Columns}", args, Map, cancellationToken); return _db.InTransactionAsync(async session => { var id = await session.ScalarAsync(pick, [], cancellationToken).ConfigureAwait(false); if (id is null) return null; await session.ExecuteAsync($"{claim} WHERE id = $3", [.. args, id], cancellationToken).ConfigureAwait(false); return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false); }, cancellationToken); } /// /// Checkpoint of a running job. Returns false when the job is no longer running (cancelled or paused meanwhile), /// which tells the worker to stop. /// public async Task UpdateProgressAsync(Guid id, long done, long total, double? itemsPerSecond, TimeSpan? eta, string? checkpoint, string? logTail, CancellationToken cancellationToken = default) => await _db.ExecuteAsync( "UPDATE jobs SET progress_done = $2, progress_total = $3, items_per_second = $4, eta_seconds = $5, checkpoint = $6, log_tail = $7 " + $"WHERE id = $1 AND state = {(int)JobState.Running}", [id, done, total, itemsPerSecond, eta?.TotalSeconds, checkpoint, logTail], cancellationToken).ConfigureAwait(false) > 0; /// Moves a job to ; done/failed/cancelled stamp the finish time, queued clears it for a retry. public async Task SetStateAsync(Guid id, JobState state, string? error = null, CancellationToken cancellationToken = default) { var terminal = state is JobState.Done or JobState.Failed or JobState.Cancelled; return await _db.ExecuteAsync("UPDATE jobs SET state = $2, error = $3, finished = $4 WHERE id = $1", [id, (int)state, error, terminal ? DateTimeOffset.UtcNow : (DateTimeOffset?)null], cancellationToken).ConfigureAwait(false) > 0; } /// Jobs, newest first, optionally of one state. public Task> ListAsync(JobState? state = null, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) => ListAsync(state, kind: null, limit, cursor, cancellationToken); /// Jobs, newest first, optionally of one state and one kind. public Task> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) => PageAsync(Columns, state, kind, limit, cursor, cancellationToken); /// /// as lists show jobs: every field but /// , which stays null ( reads a job whole). /// public Task> ListBriefAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) => PageAsync(BriefColumns, state, kind, limit, cursor, cancellationToken); private Task> PageAsync(string columns, JobState? state, JobKind? kind, int limit, string? cursor, CancellationToken cancellationToken) { var p = new SqlParameters(); var where = new List(); if (state is { } s) where.Add($"state = {p.Add((int)s)}"); if (kind is { } k) where.Add($"kind = {p.Add((int)k)}"); if (cursor is not null) { var (value, id) = Cursor.Decode("jobs", cursor); where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add)); } var take = Cursor.ClampLimit(limit); var sql = $"SELECT {columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " + $"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}"; return _db.RunAsync(async session => { var jobs = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false); string? next = null; if (jobs.Count > take) { jobs.RemoveAt(take); next = Cursor.Encode("jobs", jobs[^1].Created, jobs[^1].Id); } return new Page(jobs, next); }, cancellationToken); } /// Deletes finished jobs (done, failed, cancelled) that finished before . public Task PurgeHistoryAsync(DateTimeOffset finishedBefore, CancellationToken cancellationToken = default) => _db.ExecuteAsync( $"DELETE FROM jobs WHERE state IN ({(int)JobState.Done}, {(int)JobState.Failed}, {(int)JobState.Cancelled}) AND finished < $1", [finishedBefore], cancellationToken); /// /// Crash recovery: puts running jobs back in the queue, keeping their checkpoints. With /// only that node's jobs are requeued, including those of its numbered workers (node#0, node#1, …); /// other server instances keep theirs. /// public Task RequeueRunningAsync(string? nodeId = null, CancellationToken cancellationToken = default) => _db.ExecuteAsync( $"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE state = {(int)JobState.Running}" + (nodeId is null ? string.Empty : " AND (node_id = $1 OR substr(node_id, 1, length($2)) = $2)"), nodeId is null ? [] : [nodeId, nodeId + "#"], cancellationToken); /// /// Puts the job runs back in the queue, keeping its checkpoint (a job that yielded to a higher /// one in the priority queue); false when it no longer runs there: paused or cancelled meanwhile, that state stays. /// public async Task RequeueAsync(Guid id, string nodeId, CancellationToken cancellationToken = default) => await _db.ExecuteAsync( $"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}", [id, nodeId], cancellationToken).ConfigureAwait(false) > 0; public async Task SetPriorityAsync(Guid id, JobPriority priority, CancellationToken cancellationToken = default) => await _db.ExecuteAsync("UPDATE jobs SET priority = $2 WHERE id = $1", [id, (int)priority], cancellationToken).ConfigureAwait(false) > 0; /// Queued jobs of one kind in claim order (highest priority, then oldest). public Task> ListQueuedAsync(JobKind kind, int limit = 50, CancellationToken cancellationToken = default) => ListQueuedAsync([kind], limit, cancellationToken); /// Queued jobs of the given kinds in claim order (highest priority, then oldest). public Task> ListQueuedAsync(IReadOnlyCollection kinds, int limit = 50, CancellationToken cancellationToken = default) => ListQueuedAsync(kinds, limit, JobPriorityQueue.Off, cancellationToken); /// Queued jobs of the given kinds in the claim order of . public Task> ListQueuedAsync(IReadOnlyCollection kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) => QueuedAsync(Columns, kinds, limit, queue, cancellationToken); /// /// without the /// parameters (null), for a look at what would be claimed next: claiming a job reads it whole. /// public Task> ListQueuedBriefAsync(IReadOnlyCollection kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) => QueuedAsync(BriefColumns, kinds, limit, queue, cancellationToken); private Task> QueuedAsync(string columns, IReadOnlyCollection kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(kinds); ArgumentNullException.ThrowIfNull(queue); return kinds.Count == 0 ? Task.FromResult>([]) : _db.ListAsync( $"SELECT {columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1", [Cursor.ClampLimit(limit)], Map, cancellationToken); } /// Running jobs claimed by . public Task> ListRunningByNodeAsync(string nodeId, CancellationToken cancellationToken = default) => _db.ListAsync($"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Running} AND node_id = $1 ORDER BY started, id", [nodeId], Map, cancellationToken); /// True when a job of for exactly is waiting in the queue. public async Task IsQueuedAsync(JobKind kind, Guid targetId, CancellationToken cancellationToken = default) => await _db.ScalarAsync( $"SELECT count(*) FROM jobs WHERE state = {(int)JobState.Queued} AND kind = $1 AND target_ids = $2", [(int)kind, targetId.ToString()], cancellationToken).ConfigureAwait(false) > 0; /// /// True when a job of one of is queued, running or paused; with , /// only such a job that targets at least one of them (an archive, sequences) counts. A schedule does not queue its /// action again while one waits (owner review 2, R10). /// public async Task HasPendingAsync(IReadOnlyCollection kinds, IReadOnlyCollection? targetIds = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(kinds); if (kinds.Count == 0 || targetIds is { Count: 0 }) return false; var p = new SqlParameters(); var sql = $"SELECT count(*) FROM jobs WHERE state IN ({PendingStates}) AND kind IN ({KindList(kinds)})"; // Target ids are stored comma-joined as lower-case text, which never holds a LIKE wildcard. if (targetIds is not null) sql += $" AND ({string.Join(" OR ", targetIds.Distinct().Select(id => $"target_ids LIKE {p.Add(SqlDialect.ContainsPattern(id.ToString()))} ESCAPE '\\'"))})"; return await _db.ScalarAsync(sql, p.ToArray(), cancellationToken).ConfigureAwait(false) > 0; } /// /// What the running and paused jobs hold, but the job (the one asking): each one's id, kind and /// targets, and only a verify's parameters (). Neither the other parameters nor the targets of /// are read: a conversion asks at every sequence it comes to, and its own row names them all. /// public Task> ListHoldsAsync(Guid? except = null, CancellationToken cancellationToken = default) => _db.ListAsync( $"SELECT id, kind, target_ids, CASE WHEN kind = {(int)JobKind.Verify} THEN parameters_json END FROM jobs " + $"WHERE state IN ({(int)JobState.Running}, {(int)JobState.Paused}){(except is null ? string.Empty : " AND id <> $1")} ORDER BY created, id", except is { } self ? [self] : [], r => new JobHold(r.GetGuid(0), (JobKind)r.GetInt32(1), Targets(r.GetString(2)), r.GetStringOrNull(3)), cancellationToken); /// The targets of the queued, running and paused jobs of (the newest 5000 such jobs). public Task> ListPendingTargetsAsync(JobKind kind, CancellationToken cancellationToken = default) => PendingTargetsAsync(kind, null, cancellationToken); /// /// The targets of the queued, running and paused jobs of created before /// (the newest 5000 such jobs): those that came first. /// public Task> ListPendingTargetsAsync(JobKind kind, DateTimeOffset createdBefore, CancellationToken cancellationToken = default) => PendingTargetsAsync(kind, createdBefore, cancellationToken); private Task> PendingTargetsAsync(JobKind kind, DateTimeOffset? createdBefore, CancellationToken cancellationToken) => _db.RunAsync(async session => { var before = createdBefore is null ? string.Empty : " AND created < $2"; DbParam[] parameters = createdBefore is { } created ? [(int)kind, created] : [(int)kind]; var rows = await session.QueryListAsync( $"SELECT target_ids FROM jobs WHERE kind = $1 AND state IN ({PendingStates}){before} ORDER BY created DESC, id DESC LIMIT 5000", parameters, r => r.GetString(0), cancellationToken).ConfigureAwait(false); var ids = new HashSet(); foreach (var text in rows) { if (text.Length == 0) continue; foreach (var id in text.Split(',')) ids.Add(Guid.Parse(id)); } return ids; }, cancellationToken); /// The jobs the schedule queued, newest first. public Task> ListByScheduleAsync(Guid scheduleId, int limit = 20, CancellationToken cancellationToken = default) => _db.ListAsync($"SELECT {Columns} FROM jobs WHERE schedule_id = $1 ORDER BY created DESC, id DESC LIMIT $2", [scheduleId, Cursor.ClampLimit(limit)], Map, cancellationToken); /// /// The jobs the schedule queued, newest first, a page at a time (the runs a schedule's /// row lists in the admin panel); continues after the previous page. /// public Task> PageByScheduleAsync(Guid scheduleId, int limit = 20, string? cursor = null, CancellationToken cancellationToken = default) { var p = new SqlParameters(); var where = $"schedule_id = {p.Add(scheduleId)}"; if (cursor is not null) { var (value, id) = Cursor.Decode("schedule-runs", cursor); where += " AND " + Cursor.After("created", "id", descending: true, value, id, p.Add); } var take = Cursor.ClampLimit(limit); var sql = $"SELECT {Columns} FROM jobs WHERE {where} ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}"; return _db.RunAsync(async session => { var jobs = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false); string? next = null; if (jobs.Count > take) { jobs.RemoveAt(take); next = Cursor.Encode("schedule-runs", jobs[^1].Created, jobs[^1].Id); } return new Page(jobs, next); }, cancellationToken); } /// /// Claims one specific queued job for (render nodes pick by capability); null when another /// claimer was faster or the job is no longer queued. /// public Task TryClaimAsync(Guid id, string nodeId, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { var claimed = await session.ExecuteAsync( $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $2, started = $3, finished = NULL, error = NULL, attempts = attempts + 1 " + $"WHERE id = $1 AND state = {(int)JobState.Queued}", [id, nodeId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false); if (claimed == 0) return null; return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false); }, cancellationToken); /// Progress of a job a render node runs; false when the job is no longer running on that node. public async Task UpdateNodeProgressAsync(Guid id, string nodeId, long done, long total, double? itemsPerSecond, TimeSpan? eta, string? logTail, CancellationToken cancellationToken = default) => await _db.ExecuteAsync( "UPDATE jobs SET progress_done = $3, progress_total = $4, items_per_second = $5, eta_seconds = $6, log_tail = $7 " + $"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}", [id, nodeId, done, total, itemsPerSecond, eta?.TotalSeconds, logTail], cancellationToken).ConfigureAwait(false) > 0; /// Ends a job a render node runs (done or failed); false when it is no longer running on that node. public async Task FinishNodeJobAsync(Guid id, string nodeId, JobState state, string? error, string? logTail, CancellationToken cancellationToken = default) { if (state is not (JobState.Done or JobState.Failed)) throw new ArgumentOutOfRangeException(nameof(state), state, "Only done or failed end a node job."); return await _db.ExecuteAsync( "UPDATE jobs SET state = $3, error = $4, finished = $5, log_tail = COALESCE($6, log_tail) " + $"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}", [id, nodeId, (int)state, error, DateTimeOffset.UtcNow, logTail], cancellationToken).ConfigureAwait(false) > 0; } /// Queue and history counters for dashboards; finished counts are those since . public Task CountAsync(DateTimeOffset finishedSince, CancellationToken cancellationToken = default) => _db.RunAsync(async session => { var row = await session.QuerySingleAsync( "SELECT " + $"COALESCE(SUM(CASE WHEN state = {(int)JobState.Queued} THEN 1 ELSE 0 END), 0), " + $"COALESCE(SUM(CASE WHEN state = {(int)JobState.Running} THEN 1 ELSE 0 END), 0), " + $"COALESCE(SUM(CASE WHEN state = {(int)JobState.Paused} THEN 1 ELSE 0 END), 0), " + $"COALESCE(SUM(CASE WHEN state = {(int)JobState.Done} AND finished >= $1 THEN 1 ELSE 0 END), 0), " + $"COALESCE(SUM(CASE WHEN state = {(int)JobState.Failed} AND finished >= $1 THEN 1 ELSE 0 END), 0) " + "FROM jobs", [finishedSince], cancellationToken).ConfigureAwait(false); return new JobCounts(row.GetInt32(0), row.GetInt32(1), row.GetInt32(2), row.GetInt32(3), row.GetInt32(4)); }, cancellationToken); private static string KindList(IEnumerable kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture))); /// Target ids as the table stores them: comma-joined, empty for none. private static Guid[] Targets(string ids) => ids.Length > 0 ? Array.ConvertAll(ids.Split(','), Guid.Parse) : []; private static Job Map(DbRow r) => new() { Id = r.GetGuid(0), Kind = (JobKind)r.GetInt32(1), TargetIds = Targets(r.GetString(2)), TargetLabel = r.GetStringOrNull(3), ParametersJson = r.GetStringOrNull(4), State = (JobState)r.GetInt32(5), Priority = (JobPriority)r.GetInt32(6), ProgressDone = r.GetInt64(7), ProgressTotal = r.GetInt64(8), ItemsPerSecond = r.GetDoubleOrNull(9), Eta = JobEta.FromSeconds(r.GetDoubleOrNull(10)), Checkpoint = r.GetStringOrNull(11), LogTail = r.GetStringOrNull(12), CreatedBy = r.GetGuidOrNull(13), NodeId = r.GetStringOrNull(14), Attempts = r.GetInt32(15), Error = r.GetStringOrNull(16), Created = r.GetDateTimeOffset(17), Started = r.GetDateTimeOffsetOrNull(18), Finished = r.GetDateTimeOffsetOrNull(19), ScheduleId = r.GetGuidOrNull(20), }; }