Stage A's space guard reserved, before each photo, only that photo and the ones after it, so at the end of a sequence it counted a single photo while up to twice the parallelism were still converting: a volume filling meanwhile could go into the reserve. It now reserves the photo and every photo started whose converted file has not landed yet (a file lands whole, by a rename, before it is verified), so nothing in flight is missed and nothing landed is counted twice, at the end of a sequence as anywhere. The confirmed photo ids (about 21 MB for a million photos) lived in the job's parameters, read by every dry run's busy check, by each sequence turn of the conversion and by the job lists. They are now written whole, under a job id chosen before the job is queued, into convert-<job>.photos beside the journal, and go with it; the row keeps each sequence's digest, so the job still converts exactly the photos confirmed (without the file, the whole plan made again must match). The busy check reads only ids, kinds and targets (and a verify's parameters), never the asking job's row; a conversion's turn reads only its creation time; the server's job list and dashboard and the engine's queue peeks read jobs without parameters. A retried New sequence conversion could make a second copy. The journal now names the copy as soon as its photos are filed; a retry finishes that copy whatever changed in the source since, never remakes one deleted since, and needs no space again for what it holds; and a source another conversion copied in the same format after this one was confirmed is done with, with no copy made (new outcome Superseded, with its job log line). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
99 lines
5.8 KiB
C#
99 lines
5.8 KiB
C#
using System.Text.Json;
|
|
using Chronos.Core.Diagnostics;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Jobs;
|
|
|
|
namespace Chronos.Data.Archiving.Jobs;
|
|
|
|
/// <summary>
|
|
/// <see cref="JobKind.Convert"/>: a mass conversion (owner review 2, R7; BRIEF §12.5) of the sequences of a confirmed plan
|
|
/// (<see cref="ConvertJobParameters"/>) through <see cref="MassConverter"/>, which settles what a crash left half-way, refuses
|
|
/// when the volume lacks the space (measured again before each sequence and each photo), and converts one sequence at a
|
|
/// time, each all or nothing, the photos confirmed alone and only while each holds the content confirmed (photos that came
|
|
/// in since stay as they are); its journal makes every stage resumable. Progress counts photos; the checkpoint only marks a
|
|
/// resume, the journal says where. Each sequence whose frames were replaced is told as <c>converted</c>, each new sequence
|
|
/// as <c>created</c> (<see cref="ArchiveServiceOptions.SequenceChanged"/>), and the passes it needs (light analysis, a deep
|
|
/// one when the source had it, thumbnails and proxies) are queued as follow-ups, as soon as it is done. Sequences left as
|
|
/// they were (changed after the plan, busy, a photo another program holds, a photo that failed) are logged and fail the
|
|
/// job at the end, after the others are converted; a retry tries again those busy, held or failed, with what they staged,
|
|
/// and leaves one that changed after the plan as it is (its log line says to plan again). A retry of a New sequence
|
|
/// conversion finishes the copies it made and makes none of a source another conversion copied in the same format since
|
|
/// this job was queued (its creation time: when the plan was confirmed).
|
|
/// </summary>
|
|
internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter converter, Func<Guid, string, CancellationToken, Task>? sequenceChanged) : IJobHandler
|
|
{
|
|
/// <summary>The change told about a sequence whose frames a conversion replaced.</summary>
|
|
public const string Converted = "converted";
|
|
|
|
/// <summary>The change told about a sequence a conversion made.</summary>
|
|
public const string Created = "created";
|
|
|
|
private const string Checkpoint = "convert";
|
|
|
|
public JobKind Kind => JobKind.Convert;
|
|
|
|
public async Task RunAsync(JobContext context)
|
|
{
|
|
var parameters = (context.Job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ConvertJobParameters) : null)
|
|
?? throw new InvalidOperationException($"Convert job {context.Job.Id} has no plan.");
|
|
if (parameters.Fingerprint.Length == 0) throw new InvalidOperationException($"Convert job {context.Job.Id} has no confirmed plan.");
|
|
var archive = await db.Archives.GetAsync(parameters.ArchiveId, context.Cancellation).ConfigureAwait(false)
|
|
?? throw new InvalidOperationException($"Convert job {context.Job.Id} does not name a known archive.");
|
|
var name = context.Job.CreatedBy is { } user ? (await db.Users.GetAsync(user, context.Cancellation).ConfigureAwait(false))?.Username : null;
|
|
var by = new ConvertRequester(name, context.Job.CreatedBy, context.Job.Created);
|
|
var result = await converter.RunAsync(archive, parameters, context.Job.Id, by, new ConvertCallbacks(
|
|
(done, total) => context.ReportAsync(done, total),
|
|
(done, total) => context.CheckpointAsync(Checkpoint, done, total),
|
|
context.Log, TellAsync, requests => QueueAsync(context, requests)), context.Cancellation).ConfigureAwait(false);
|
|
context.Log(Loc.L("Archive.Log.ConvertDone", result.Sequences.Count(s => s.Outcome == ConvertOutcome.Converted),
|
|
result.Sequences.Count(s => s.Outcome == ConvertOutcome.Created), result.Frames));
|
|
var kept = result.Kept.Count();
|
|
if (kept > 0) throw new InvalidOperationException(Loc.Plural("Archive.Error.ConvertKept", kept));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Queues what a sequence converted or made needs as soon as its outcome is journaled, each kind once per sequence; a
|
|
/// failure is logged, never passed on.
|
|
/// </summary>
|
|
private async ValueTask QueueAsync(JobContext context, IReadOnlyList<ArchiveJobRequest> requests)
|
|
{
|
|
foreach (var request in requests)
|
|
{
|
|
try
|
|
{
|
|
if (request.Targets.Count > 0 && await db.Jobs.IsQueuedAsync(request.Kind, request.Targets[0], CancellationToken.None).ConfigureAwait(false)) continue;
|
|
await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false);
|
|
}
|
|
catch (DatabaseException ex)
|
|
{
|
|
Log.Warning("archive", $"{request.Label}: the passes its converted frames need were not queued; the next scan queues them", ex);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tells a sequence whose frames were replaced, or the one made, as soon as its outcome is journaled, while the rest of the
|
|
/// job runs (an editor open on it reads it again); a failure is logged, never passed on to the job.
|
|
/// </summary>
|
|
private async ValueTask TellAsync(ConvertSequenceResult sequence)
|
|
{
|
|
if (sequenceChanged is null) return;
|
|
var (id, change) = sequence.Outcome switch
|
|
{
|
|
ConvertOutcome.Converted => (sequence.SequenceId, Converted),
|
|
ConvertOutcome.Created when sequence.NewSequenceId is { } made => (made, Created),
|
|
_ => (Guid.Empty, string.Empty),
|
|
};
|
|
if (id == Guid.Empty) return;
|
|
try
|
|
{
|
|
await sequenceChanged(id, change, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warning("archive", $"The change of sequence {id} was not told", ex);
|
|
}
|
|
}
|
|
}
|