Files
Chronos/src/Chronos.Data/Repositories/JobRepository.cs
T
Alby96 c669d0504d fix(convert): the job converts exactly the photos its preview listed, once
A conversion job could still do something its confirmed dry run did not show,
or refuse what it should have converted:

- An upload landing between the confirmation and the sequence's turn made the
  sequence Changed at every retry. Each ConvertSequence now names the photos
  the plan converts (Photos, their frame ids); the job takes those from its
  re-plan, checks their content against the confirmed digest and leaves photos
  that came in since as they are. A photo the plan did not list is never
  converted, whatever changed on disk or in the index since the dry run.
- A file another program held during the dry run was planned as Missing and
  then converted or not depending on the re-plan. It is now its own reason
  (inaccessible, "held open or cannot be opened"); a truncated file reads as
  undecodable. At job time a confirmed photo that cannot be opened leaves the
  sequence Blocked for the retry; a held photo the job does not convert does
  not hide a real change.
- The same sequences could be converted twice (two dialogs or tabs, a lost
  answer, no dry run at all). MassConverter.QueueAsync checks the queued,
  running and paused conversions and queues under one lock, in the caller's
  context (the desktop's enqueue shows a toast on the UI thread); a job's own
  re-plan treats sequences of conversions queued before it as busy.
- The per-photo space check reserved a full batch of photos even when fewer
  were left, so a job whose preview had the space could stop at every retry.
  It now reserves the photo and those after it that may convert at once.
- New sequence now says in the dry run, the panel's totals and confirmation,
  and a new desktop confirmation, how many photos each new sequence lacks;
  only sequences that are copied use those words.
- On the server, a confirmation takes only the plan kept for the index as it
  is now (503 plan_dropped otherwise, never a plan made inside the request),
  and busy sequences and free space are as they are at every request. The
  panel's Apply asks the dry run again first, with Close still available,
  and confirms only the same fingerprint.
- Maintenance keeps another host's conversion staging as long as its journal
  and purges both together; the unstaged-copy failure and blocked sequences
  are logged in catalog sentences.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit fefd9c34ebfeed1f9e84161610ef286026d38355)
2026-11-02 10:33:34 +01:00

334 lines
21 KiB
C#

using System.Globalization;
using Chronos.Core.Model;
using Chronos.Data.Jobs;
namespace Chronos.Data.Repositories;
/// <summary>Jobs per state; <see cref="DoneSince"/> and <see cref="FailedSince"/> count those finished since a given time.</summary>
public sealed record JobCounts(int Queued, int Running, int Paused, int DoneSince, int FailedSince);
/// <summary>
/// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically,
/// write checkpoints while running, and on startup requeue what a crash left running.
/// </summary>
public sealed class JobRepository
{
private const string Columns =
"id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " +
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id";
/// <summary>The states of a job that has not ended: queued, running, paused.</summary>
private static readonly string PendingStates =
string.Join(", ", new[] { JobState.Queued, JobState.Running, JobState.Paused }.Select(s => ((int)s).ToString(CultureInfo.InvariantCulture)));
private readonly DbAccess _db;
internal JobRepository(DbAccess db) => _db = db;
public async Task EnqueueAsync(Job job, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
$"INSERT INTO jobs ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)",
[
job.Id, (int)job.Kind, string.Join(',', job.TargetIds), job.TargetLabel, DbParam.Json(job.ParametersJson), (int)job.State,
(int)job.Priority, job.ProgressDone, job.ProgressTotal, job.ItemsPerSecond, job.Eta?.TotalSeconds, job.Checkpoint, job.LogTail,
job.CreatedBy, job.NodeId, job.Attempts, job.Error, job.Created, job.Started, job.Finished, job.ScheduleId,
], cancellationToken).ConfigureAwait(false);
public Task<Job?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken);
/// <summary>
/// Marks the next queued job (highest priority, then oldest) of the given kinds as running for
/// <paramref name="workerId"/> and returns it, or null when none waits. Concurrent claimers never get the same
/// job: PostgreSQL skips rows locked by other claimers, SQLite holds its write lock across select and update.
/// </summary>
public Task<Job?> ClaimNextAsync(string workerId, IReadOnlyCollection<JobKind>? kinds = null, CancellationToken cancellationToken = default) =>
ClaimNextAsync(workerId, kinds, JobPriorityQueue.Off, cancellationToken);
/// <summary>
/// <see cref="ClaimNextAsync(string, IReadOnlyCollection{JobKind}?, CancellationToken)"/> in the order of
/// <paramref name="queue"/>: while it is on, the kinds of the highest level first (<see cref="JobPriorityQueue.ClaimsBefore"/>).
/// </summary>
public Task<Job?> ClaimNextAsync(string workerId, IReadOnlyCollection<JobKind>? kinds, JobPriorityQueue queue, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(queue);
var kindFilter = kinds is { Count: > 0 } ? $" AND kind IN ({KindList(kinds)})" : string.Empty;
var pick = $"SELECT id FROM jobs WHERE state = {(int)JobState.Queued}{kindFilter} ORDER BY {queue.OrderSql(kinds is { Count: > 0 } ? kinds : Enum.GetValues<JobKind>())}" +
"priority DESC, created, id LIMIT 1";
var claim = $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $1, started = $2, finished = NULL, error = NULL, attempts = attempts + 1";
DbParam[] args = [workerId, DateTimeOffset.UtcNow];
if (_db.Sql.IsPostgres)
return _db.SingleAsync($"{claim} WHERE id = ({pick}{_db.Sql.ForUpdateSkipLocked}) RETURNING {Columns}", args, Map, cancellationToken);
return _db.InTransactionAsync(async session =>
{
var id = await session.ScalarAsync<Guid?>(pick, [], cancellationToken).ConfigureAwait(false);
if (id is null) return null;
await session.ExecuteAsync($"{claim} WHERE id = $3", [.. args, id], cancellationToken).ConfigureAwait(false);
return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false);
}, cancellationToken);
}
/// <summary>
/// Checkpoint of a running job. Returns false when the job is no longer running (cancelled or paused meanwhile),
/// which tells the worker to stop.
/// </summary>
public async Task<bool> UpdateProgressAsync(Guid id, long done, long total, double? itemsPerSecond, TimeSpan? eta,
string? checkpoint, string? logTail, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
"UPDATE jobs SET progress_done = $2, progress_total = $3, items_per_second = $4, eta_seconds = $5, checkpoint = $6, log_tail = $7 " +
$"WHERE id = $1 AND state = {(int)JobState.Running}",
[id, done, total, itemsPerSecond, eta?.TotalSeconds, checkpoint, logTail], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Moves a job to <paramref name="state"/>; done/failed/cancelled stamp the finish time, queued clears it for a retry.</summary>
public async Task<bool> SetStateAsync(Guid id, JobState state, string? error = null, CancellationToken cancellationToken = default)
{
var terminal = state is JobState.Done or JobState.Failed or JobState.Cancelled;
return await _db.ExecuteAsync("UPDATE jobs SET state = $2, error = $3, finished = $4 WHERE id = $1",
[id, (int)state, error, terminal ? DateTimeOffset.UtcNow : (DateTimeOffset?)null], cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>Jobs, newest first, optionally of one state.</summary>
public Task<Page<Job>> ListAsync(JobState? state = null, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
ListAsync(state, kind: null, limit, cursor, cancellationToken);
/// <summary>Jobs, newest first, optionally of one state and one kind.</summary>
public Task<Page<Job>> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default)
{
var p = new SqlParameters();
var where = new List<string>();
if (state is { } s) where.Add($"state = {p.Add((int)s)}");
if (kind is { } k) where.Add($"kind = {p.Add((int)k)}");
if (cursor is not null)
{
var (value, id) = Cursor.Decode("jobs", cursor);
where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add));
}
var take = Cursor.ClampLimit(limit);
var sql = $"SELECT {Columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
$"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}";
return _db.RunAsync(async session =>
{
var jobs = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false);
string? next = null;
if (jobs.Count > take)
{
jobs.RemoveAt(take);
next = Cursor.Encode("jobs", jobs[^1].Created, jobs[^1].Id);
}
return new Page<Job>(jobs, next);
}, cancellationToken);
}
/// <summary>Deletes finished jobs (done, failed, cancelled) that finished before <paramref name="finishedBefore"/>.</summary>
public Task<int> PurgeHistoryAsync(DateTimeOffset finishedBefore, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync(
$"DELETE FROM jobs WHERE state IN ({(int)JobState.Done}, {(int)JobState.Failed}, {(int)JobState.Cancelled}) AND finished < $1",
[finishedBefore], cancellationToken);
/// <summary>
/// Crash recovery: puts running jobs back in the queue, keeping their checkpoints. With <paramref name="nodeId"/>
/// only that node's jobs are requeued, including those of its numbered workers (<c>node#0</c>, <c>node#1</c>, …);
/// other server instances keep theirs.
/// </summary>
public Task<int> RequeueRunningAsync(string? nodeId = null, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync(
$"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE state = {(int)JobState.Running}" +
(nodeId is null ? string.Empty : " AND (node_id = $1 OR substr(node_id, 1, length($2)) = $2)"),
nodeId is null ? [] : [nodeId, nodeId + "#"], cancellationToken);
/// <summary>
/// Puts the job <paramref name="nodeId"/> runs back in the queue, keeping its checkpoint (a job that yielded to a higher
/// one in the priority queue); false when it no longer runs there: paused or cancelled meanwhile, that state stays.
/// </summary>
public async Task<bool> RequeueAsync(Guid id, string nodeId, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
$"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
[id, nodeId], cancellationToken).ConfigureAwait(false) > 0;
public async Task<bool> SetPriorityAsync(Guid id, JobPriority priority, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE jobs SET priority = $2 WHERE id = $1", [id, (int)priority], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Queued jobs of one kind in claim order (highest priority, then oldest).</summary>
public Task<IReadOnlyList<Job>> ListQueuedAsync(JobKind kind, int limit = 50, CancellationToken cancellationToken = default) =>
ListQueuedAsync([kind], limit, cancellationToken);
/// <summary>Queued jobs of the given kinds in claim order (highest priority, then oldest).</summary>
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit = 50, CancellationToken cancellationToken = default) =>
ListQueuedAsync(kinds, limit, JobPriorityQueue.Off, cancellationToken);
/// <summary>Queued jobs of the given kinds in the claim order of <paramref name="queue"/>.</summary>
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(queue);
return kinds.Count == 0 ? Task.FromResult<IReadOnlyList<Job>>([]) : _db.ListAsync(
$"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1",
[Cursor.ClampLimit(limit)], Map, cancellationToken);
}
/// <summary>Running jobs claimed by <paramref name="nodeId"/>.</summary>
public Task<IReadOnlyList<Job>> ListRunningByNodeAsync(string nodeId, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Running} AND node_id = $1 ORDER BY started, id",
[nodeId], Map, cancellationToken);
/// <summary>True when a job of <paramref name="kind"/> for exactly <paramref name="targetId"/> is waiting in the queue.</summary>
public async Task<bool> IsQueuedAsync(JobKind kind, Guid targetId, CancellationToken cancellationToken = default) =>
await _db.ScalarAsync<long>(
$"SELECT count(*) FROM jobs WHERE state = {(int)JobState.Queued} AND kind = $1 AND target_ids = $2",
[(int)kind, targetId.ToString()], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// True when a job of one of <paramref name="kinds"/> is queued, running or paused; with <paramref name="targetIds"/>,
/// only such a job that targets at least one of them (an archive, sequences) counts. A schedule does not queue its
/// action again while one waits (owner review 2, R10).
/// </summary>
public async Task<bool> HasPendingAsync(IReadOnlyCollection<JobKind> kinds, IReadOnlyCollection<Guid>? targetIds = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(kinds);
if (kinds.Count == 0 || targetIds is { Count: 0 }) return false;
var p = new SqlParameters();
var sql = $"SELECT count(*) FROM jobs WHERE state IN ({PendingStates}) AND kind IN ({KindList(kinds)})";
// Target ids are stored comma-joined as lower-case text, which never holds a LIKE wildcard.
if (targetIds is not null)
sql += $" AND ({string.Join(" OR ", targetIds.Distinct().Select(id => $"target_ids LIKE {p.Add(SqlDialect.ContainsPattern(id.ToString()))} ESCAPE '\\'"))})";
return await _db.ScalarAsync<long>(sql, p.ToArray(), cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>The targets of the queued, running and paused jobs of <paramref name="kind"/> (the newest 5000 such jobs).</summary>
public Task<HashSet<Guid>> ListPendingTargetsAsync(JobKind kind, CancellationToken cancellationToken = default) =>
PendingTargetsAsync(kind, null, cancellationToken);
/// <summary>
/// The targets of the queued, running and paused jobs of <paramref name="kind"/> created before <paramref name="createdBefore"/>
/// (the newest 5000 such jobs): those that came first.
/// </summary>
public Task<HashSet<Guid>> ListPendingTargetsAsync(JobKind kind, DateTimeOffset createdBefore, CancellationToken cancellationToken = default) =>
PendingTargetsAsync(kind, createdBefore, cancellationToken);
private Task<HashSet<Guid>> PendingTargetsAsync(JobKind kind, DateTimeOffset? createdBefore, CancellationToken cancellationToken) =>
_db.RunAsync(async session =>
{
var before = createdBefore is null ? string.Empty : " AND created < $2";
DbParam[] parameters = createdBefore is { } created ? [(int)kind, created] : [(int)kind];
var rows = await session.QueryListAsync(
$"SELECT target_ids FROM jobs WHERE kind = $1 AND state IN ({PendingStates}){before} ORDER BY created DESC, id DESC LIMIT 5000",
parameters, r => r.GetString(0), cancellationToken).ConfigureAwait(false);
var ids = new HashSet<Guid>();
foreach (var text in rows)
{
if (text.Length == 0) continue;
foreach (var id in text.Split(',')) ids.Add(Guid.Parse(id));
}
return ids;
}, cancellationToken);
/// <summary>The jobs the schedule <paramref name="scheduleId"/> queued, newest first.</summary>
public Task<IReadOnlyList<Job>> ListByScheduleAsync(Guid scheduleId, int limit = 20, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM jobs WHERE schedule_id = $1 ORDER BY created DESC, id DESC LIMIT $2",
[scheduleId, Cursor.ClampLimit(limit)], Map, cancellationToken);
/// <summary>
/// The jobs the schedule <paramref name="scheduleId"/> queued, newest first, a page at a time (the runs a schedule's
/// row lists in the admin panel); <paramref name="cursor"/> continues after the previous page.
/// </summary>
public Task<Page<Job>> PageByScheduleAsync(Guid scheduleId, int limit = 20, string? cursor = null, CancellationToken cancellationToken = default)
{
var p = new SqlParameters();
var where = $"schedule_id = {p.Add(scheduleId)}";
if (cursor is not null)
{
var (value, id) = Cursor.Decode("schedule-runs", cursor);
where += " AND " + Cursor.After("created", "id", descending: true, value, id, p.Add);
}
var take = Cursor.ClampLimit(limit);
var sql = $"SELECT {Columns} FROM jobs WHERE {where} ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}";
return _db.RunAsync(async session =>
{
var jobs = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false);
string? next = null;
if (jobs.Count > take)
{
jobs.RemoveAt(take);
next = Cursor.Encode("schedule-runs", jobs[^1].Created, jobs[^1].Id);
}
return new Page<Job>(jobs, next);
}, cancellationToken);
}
/// <summary>
/// Claims one specific queued job for <paramref name="nodeId"/> (render nodes pick by capability); null when another
/// claimer was faster or the job is no longer queued.
/// </summary>
public Task<Job?> TryClaimAsync(Guid id, string nodeId, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var claimed = await session.ExecuteAsync(
$"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $2, started = $3, finished = NULL, error = NULL, attempts = attempts + 1 " +
$"WHERE id = $1 AND state = {(int)JobState.Queued}",
[id, nodeId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false);
if (claimed == 0) return null;
return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false);
}, cancellationToken);
/// <summary>Progress of a job a render node runs; false when the job is no longer running on that node.</summary>
public async Task<bool> UpdateNodeProgressAsync(Guid id, string nodeId, long done, long total, double? itemsPerSecond, TimeSpan? eta,
string? logTail, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
"UPDATE jobs SET progress_done = $3, progress_total = $4, items_per_second = $5, eta_seconds = $6, log_tail = $7 " +
$"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
[id, nodeId, done, total, itemsPerSecond, eta?.TotalSeconds, logTail], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Ends a job a render node runs (done or failed); false when it is no longer running on that node.</summary>
public async Task<bool> FinishNodeJobAsync(Guid id, string nodeId, JobState state, string? error, string? logTail, CancellationToken cancellationToken = default)
{
if (state is not (JobState.Done or JobState.Failed)) throw new ArgumentOutOfRangeException(nameof(state), state, "Only done or failed end a node job.");
return await _db.ExecuteAsync(
"UPDATE jobs SET state = $3, error = $4, finished = $5, log_tail = COALESCE($6, log_tail) " +
$"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
[id, nodeId, (int)state, error, DateTimeOffset.UtcNow, logTail], cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>Queue and history counters for dashboards; finished counts are those since <paramref name="finishedSince"/>.</summary>
public Task<JobCounts> CountAsync(DateTimeOffset finishedSince, CancellationToken cancellationToken = default) =>
_db.RunAsync(async session =>
{
var row = await session.QuerySingleAsync(
"SELECT " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Queued} THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Running} THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Paused} THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Done} AND finished >= $1 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Failed} AND finished >= $1 THEN 1 ELSE 0 END), 0) " +
"FROM jobs", [finishedSince], cancellationToken).ConfigureAwait(false);
return new JobCounts(row.GetInt32(0), row.GetInt32(1), row.GetInt32(2), row.GetInt32(3), row.GetInt32(4));
}, cancellationToken);
private static string KindList(IEnumerable<JobKind> kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture)));
private static Job Map(DbRow r) => new()
{
Id = r.GetGuid(0),
Kind = (JobKind)r.GetInt32(1),
TargetIds = r.GetString(2) is { Length: > 0 } ids ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [],
TargetLabel = r.GetStringOrNull(3),
ParametersJson = r.GetStringOrNull(4),
State = (JobState)r.GetInt32(5),
Priority = (JobPriority)r.GetInt32(6),
ProgressDone = r.GetInt64(7),
ProgressTotal = r.GetInt64(8),
ItemsPerSecond = r.GetDoubleOrNull(9),
Eta = JobEta.FromSeconds(r.GetDoubleOrNull(10)),
Checkpoint = r.GetStringOrNull(11),
LogTail = r.GetStringOrNull(12),
CreatedBy = r.GetGuidOrNull(13),
NodeId = r.GetStringOrNull(14),
Attempts = r.GetInt32(15),
Error = r.GetStringOrNull(16),
Created = r.GetDateTimeOffset(17),
Started = r.GetDateTimeOffsetOrNull(18),
Finished = r.GetDateTimeOffsetOrNull(19),
ScheduleId = r.GetGuidOrNull(20),
};
}