fix(convert): reserve the photos still writing, keep photo ids out of the job row, never copy a source twice

Stage A's space guard reserved, before each photo, only that photo and the
ones after it, so at the end of a sequence it counted a single photo while
up to twice the parallelism were still converting: a volume filling
meanwhile could go into the reserve. It now reserves the photo and every
photo started whose converted file has not landed yet (a file lands whole,
by a rename, before it is verified), so nothing in flight is missed and
nothing landed is counted twice, at the end of a sequence as anywhere.

The confirmed photo ids (about 21 MB for a million photos) lived in the
job's parameters, read by every dry run's busy check, by each sequence turn
of the conversion and by the job lists. They are now written whole, under a
job id chosen before the job is queued, into convert-<job>.photos beside
the journal, and go with it; the row keeps each sequence's digest, so the
job still converts exactly the photos confirmed (without the file, the
whole plan made again must match). The busy check reads only ids, kinds and
targets (and a verify's parameters), never the asking job's row; a
conversion's turn reads only its creation time; the server's job list and
dashboard and the engine's queue peeks read jobs without parameters.

A retried New sequence conversion could make a second copy. The journal now
names the copy as soon as its photos are filed; a retry finishes that copy
whatever changed in the source since, never remakes one deleted since, and
needs no space again for what it holds; and a source another conversion
copied in the same format after this one was confirmed is done with, with
no copy made (new outcome Superseded, with its job log line).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alby96andClaude Opus 5.5 committed 2026-11-03 09:36:00 +01:00
1 parent b471a33bfc
commit 8ea25228eb
14 files changed
+966 -201

No files matched your search

+17 -11
View File
@@ -697,29 +697,35 @@ public static class ArchiveJobs
/// <summary>
/// <see cref="JobKind.Convert"/>: applies the conversion <paramref name="plan"/> (<see cref="ArchiveService.PlanConvertAsync"/>)
/// as confirmed (owner review 2, R7): the sequences it converts, in its order with their frames versions, the photos each
/// converts and their digest (<see cref="ConvertSequencePlan.Digest"/>), are the job's targets, so jobs that would touch
/// them wait. The job converts those photos alone, and leaves as it is and reports a sequence where one of them no longer
/// holds the content confirmed when the job comes to it. Queue it through <see cref="MassConverter.QueueAsync"/>, which
/// refuses a second conversion of the same sequences.
/// as confirmed (owner review 2, R7): the sequences it converts, in its order with their frames versions and the digest
/// of the photos each converts (<see cref="ConvertSequencePlan.Digest"/>), are the job's targets, so jobs that would touch
/// them wait. The request names the job's id, under which <see cref="MassConverter.QueueAsync"/> (the way to queue it,
/// which refuses a second conversion of the same sequences) writes which photos those are beside the job's journal
/// (<see cref="ConfirmedPhotos"/>). The job converts those photos alone, and leaves as it is and reports a sequence where
/// one of them no longer holds the content confirmed when the job comes to it.
/// </summary>
/// <exception cref="ArgumentException">The plan converts nothing.</exception>
public static ArchiveJobRequest Convert(Archive archive, ConvertPlan plan)
{
ArgumentNullException.ThrowIfNull(archive);
ArgumentNullException.ThrowIfNull(plan);
var sequences = plan.Converted
.Select(s => new ConvertSequence(s.SequenceId, s.FramesVersion, s.Digest(), ConvertSequence.EncodePhotos(s.Frames.Select(f => f.FrameId))))
.ToList();
var sequences = plan.Converted.Select(s => new ConvertSequence(s.SequenceId, s.FramesVersion, s.Digest())).ToList();
if (sequences.Count == 0) throw new ArgumentException("The plan converts nothing.", nameof(plan));
var label = sequences.Count == 1 ? plan.Converted.First().Title : archive.Name;
return new(JobKind.Convert, [.. sequences.Select(s => s.Id)], label,
Json(new ConvertJobParameters(archive.Id, plan.Options.Target, plan.Options.Quality, plan.Mode, sequences, plan.Fingerprint()),
ArchiveJobJson.Default.ConvertJobParameters), JobPriority.User);
ArchiveJobJson.Default.ConvertJobParameters), JobPriority.User, Guid.CreateVersion7());
}
public static Task<Job> EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) =>
engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
/// <summary>Queues <paramref name="request"/>, under its own id when it names one (<see cref="ArchiveJobRequest.Id"/>).</summary>
public static Task<Job> EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(engine);
ArgumentNullException.ThrowIfNull(request);
return request.Id is { } id
? engine.EnqueueAsync(id, request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken)
: engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
}
private static string Json<T>(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo) => JsonSerializer.Serialize(value, typeInfo);
}
@@ -56,55 +56,29 @@ public sealed record MoveSequence(Guid Id);
/// <see cref="JobKind.Convert"/> (owner review 2, R7): targets = the sequences converted, so the jobs that would touch them
/// wait; the archive, the target format and JPEG quality, the mode chosen for this run, the sequences of the confirmed plan
/// in its order, each with what it converts, and the plan's fingerprint (<see cref="ConvertPlan.Fingerprint"/>). A sequence
/// whose photos to convert are no longer those confirmed is left as it is and reported.
/// whose photos to convert are no longer those confirmed is left as it is and reported. Which photos those are is not
/// here but in the job's own file beside its journal (<see cref="ConfirmedPhotos"/>), so that the row stays as small as the
/// number of sequences, whatever the number of photos, for everything that reads jobs.
/// </summary>
public sealed record ConvertJobParameters(Guid ArchiveId, ConvertTarget Target, int Quality, ConvertMode Mode, IReadOnlyList<ConvertSequence> Sequences,
string Fingerprint);
/// <summary>
/// A sequence of a <see cref="ConvertJobParameters"/>: the frames version it was planned at, the digest of the photos it
/// converts, each with its content (<see cref="ConvertSequencePlan.Digest"/>), and which photos those are (<see cref="Photos"/>).
/// Under the sequence's lease, before it converts anything, the job takes those photos from its plan made again and goes
/// on only while they have the digest confirmed: photos that came in since (an upload while the job waited its turn) stay
/// as they are, and a photo the plan did not list (one another program held while it was made) is never converted. Null
/// in a job an earlier build queued, which checks the frames version only (no digest), or the whole plan's digest (no photos).
/// A sequence of a <see cref="ConvertJobParameters"/>: the frames version it was planned at, and the digest of the photos it
/// converts, each with its content (<see cref="ConvertSequencePlan.Digest"/>). Under the sequence's lease, before it
/// converts anything, the job takes the photos confirmed (<see cref="ConfirmedPhotos"/>) from its plan made again and goes
/// on only while they have that digest: photos that came in since (an upload while the job waited its turn) stay as they
/// are, and a photo the plan did not list (one another program held while it was made) is never converted. Without the
/// confirmed photos, the whole plan made again must have the digest. Null in a job an earlier build queued, which checks
/// the frames version only.
/// </summary>
/// <param name="Photos">The frame ids of the photos converted, in the plan's order, as <see cref="EncodePhotos"/> writes them.</param>
public sealed record ConvertSequence(Guid Id, int FramesVersion, string? Frames = null, string? Photos = null)
{
/// <summary>
/// Frame ids as a job's parameters hold them: their 16 bytes each, one after the other, in base64 (a whole-archive
/// conversion lists every photo it converts, so the job row stays about half the size of a list of ids).
/// </summary>
public static string EncodePhotos(IEnumerable<Guid> ids)
{
ArgumentNullException.ThrowIfNull(ids);
IReadOnlyCollection<Guid> frames = ids as IReadOnlyCollection<Guid> ?? [.. ids];
var bytes = new byte[frames.Count * 16];
var at = 0;
foreach (var frame in frames)
{
frame.TryWriteBytes(bytes.AsSpan(at, 16));
at += 16;
}
return Convert.ToBase64String(bytes);
}
public sealed record ConvertSequence(Guid Id, int FramesVersion, string? Frames = null);
/// <summary>The frame ids <see cref="Photos"/> holds, in order; null when it holds none (a job an earlier build queued).</summary>
/// <exception cref="FormatException">The text is not ids as <see cref="EncodePhotos"/> writes them.</exception>
public IReadOnlyList<Guid>? PhotoIds()
{
if (Photos is null) return null;
var bytes = Convert.FromBase64String(Photos);
if (bytes.Length % 16 != 0) throw new FormatException($"The photos of sequence {Id} are not frame ids.");
var ids = new Guid[bytes.Length / 16];
for (var i = 0; i < ids.Length; i++) ids[i] = new Guid(bytes.AsSpan(i * 16, 16));
return ids;
}
}
/// <summary>A job ready to enqueue.</summary>
public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets, string? Label, string? ParametersJson, JobPriority Priority);
/// <summary>
/// A job ready to enqueue; <paramref name="Id"/>, when set, is the id it is queued under (chosen first, for what the job
/// reads beside its journal: <see cref="ConfirmedPhotos"/>), otherwise the engine gives it one.
/// </summary>
public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets, string? Label, string? ParametersJson, JobPriority Priority, Guid? Id = null);
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)]
[JsonSerializable(typeof(ScanParameters))]
@@ -120,6 +94,7 @@ public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets
[JsonSerializable(typeof(ConvertJournal))]
[JsonSerializable(typeof(StagedLine))]
[JsonSerializable(typeof(PinnedSequence))]
[JsonSerializable(typeof(ConfirmedLine))]
[JsonSerializable(typeof(CardProfile))]
internal sealed partial class ArchiveJobJson : JsonSerializerContext
{
@@ -17,7 +17,9 @@ namespace Chronos.Data.Archiving.Jobs;
/// one when the source had it, thumbnails and proxies) are queued as follow-ups, as soon as it is done. Sequences left as
/// they were (changed after the plan, busy, a photo another program holds, a photo that failed) are logged and fail the
/// job at the end, after the others are converted; a retry tries again those busy, held or failed, with what they staged,
/// and leaves one that changed after the plan as it is (its log line says to plan again).
/// and leaves one that changed after the plan as it is (its log line says to plan again). A retry of a New sequence
/// conversion finishes the copies it made and makes none of a source another conversion copied in the same format since
/// this job was queued (its creation time: when the plan was confirmed).
/// </summary>
internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter converter, Func<Guid, string, CancellationToken, Task>? sequenceChanged) : IJobHandler
{
@@ -39,7 +41,8 @@ internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter conver
var archive = await db.Archives.GetAsync(parameters.ArchiveId, context.Cancellation).ConfigureAwait(false)
?? throw new InvalidOperationException($"Convert job {context.Job.Id} does not name a known archive.");
var name = context.Job.CreatedBy is { } user ? (await db.Users.GetAsync(user, context.Cancellation).ConfigureAwait(false))?.Username : null;
var result = await converter.RunAsync(archive, parameters, context.Job.Id, new ConvertRequester(name, context.Job.CreatedBy), new ConvertCallbacks(
var by = new ConvertRequester(name, context.Job.CreatedBy, context.Job.Created);
var result = await converter.RunAsync(archive, parameters, context.Job.Id, by, new ConvertCallbacks(
(done, total) => context.ReportAsync(done, total),
(done, total) => context.CheckpointAsync(Checkpoint, done, total),
context.Log, TellAsync, requests => QueueAsync(context, requests)), context.Cancellation).ConfigureAwait(false);
+423 -96
View File
@@ -1,3 +1,4 @@
using System.Collections.Concurrent;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
@@ -29,6 +30,12 @@ public enum ConvertOutcome
/// <summary>It had nothing to convert.</summary>
Nothing,
/// <summary>
/// Done with, no copy made (<see cref="ConvertMode.NewSequence"/>): another conversion made its copy in this format since
/// this one was confirmed (a retry of an older conversion, or a duplicate queued beside it), so a second copy is not made.
/// </summary>
Superseded,
/// <summary>
/// Left as it was: a photo it converts no longer holds the content confirmed (it changed, is gone, or no longer converts),
/// or it is no longer in the archive. A retry finds the same: the user plans again.
@@ -52,8 +59,12 @@ public enum ConvertOutcome
public sealed record ConvertSequenceResult(Guid SequenceId, string Title, ConvertOutcome Outcome, int Frames, Guid? NewSequenceId = null, string? Error = null,
int LeftOut = 0)
{
/// <summary>Whether the sequence is done with: converted, made, or with nothing to convert.</summary>
public bool Final => Outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing;
/// <summary>Whether the sequence is done with: converted, made, with nothing to convert, or copied by another conversion since.</summary>
public bool Final => IsFinal(Outcome);
/// <summary>Whether <paramref name="outcome"/> leaves a sequence done with (<see cref="Final"/>).</summary>
internal static bool IsFinal(ConvertOutcome? outcome) =>
outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing or ConvertOutcome.Superseded;
}
/// <summary>What a conversion did, and the passes the sequences it changed or made need (queued as follow-ups as each one ended).</summary>
@@ -75,8 +86,12 @@ public sealed record ConvertResult(IReadOnlyList<ConvertSequenceResult> Sequence
internal sealed record ConvertCallbacks(Func<long, long, ValueTask> Progress, Func<long, long, ValueTask> Checkpoint, Action<string> Log,
Func<ConvertSequenceResult, ValueTask>? Done = null, Func<IReadOnlyList<ArchiveJobRequest>, ValueTask>? Queue = null);
/// <summary>Who asked for a conversion: the user's name, kept in the trash records of a Replace, and id, the owner of a new sequence when its source has none.</summary>
internal sealed record ConvertRequester(string? Name, Guid? Id);
/// <summary>
/// Who asked for a conversion: the user's name, kept in the trash records of a Replace, and id, the owner of a new sequence
/// when its source has none; and when the plan was confirmed (the job queued): a copy another conversion made of a source
/// since then supersedes this one's (<see cref="ConvertOutcome.Superseded"/>). Null: no copy made by another counts.
/// </summary>
internal sealed record ConvertRequester(string? Name, Guid? Id, DateTimeOffset? Confirmed = null);
/// <summary>
/// A conversion of sequences a confirmed plan converts is queued, running or paused already, so its job was not queued
@@ -94,7 +109,7 @@ public sealed class ConversionQueuedException(IReadOnlyList<Guid> sequenceIds) :
/// the conversion writes, <see cref="ArchiveServiceOptions.FreeSpace"/>), and a <see cref="JobKind.Convert"/> job applies the
/// confirmed plan (queued by <see cref="QueueAsync"/>, which refuses a second conversion of the same sequences) one sequence
/// at a time. Under the sequence's lease, its sidecar settled where the index has it then (<see cref="SequenceWriter.ReadLocked"/>),
/// the job plans it again and takes the photos confirmed (<see cref="ConvertSequence.Photos"/>), going on only while each is
/// the job plans it again and takes the photos confirmed (<see cref="ConfirmedPhotos"/>), going on only while each is
/// still planned with its content confirmed (<see cref="ConvertSequence.Frames"/>); it pins them then (<see cref="PinLog"/>),
/// so a photo that came in, was verified or could be opened again since the plan stays as it is, on a retry too. Stage A,
/// without the lease: each photo is hashed against its record and converted (<see cref="FrameConverter"/>, several at once
@@ -107,14 +122,19 @@ public sealed class ConversionQueuedException(IReadOnlyList<Guid> sequenceIds) :
/// <see cref="ConvertMode.NewSequence"/> files the converted copies into a new sequence through the archive ingest (session
/// key <c>convert:&lt;job&gt;:&lt;source&gt;</c>, so a resume finds it again), titled "title (FORMAT)", owned as the source is,
/// with the source's tags, rating, notes and project, the source left as it is (the photos the plan did not convert stay
/// there alone, as the job log says). A sequence whose photos changed, that is busy, or has a photo that fails is left as it
/// was and reported.
/// there alone, as the job log says). A retry or a resume finishes the copy it made already rather than make another, and
/// makes none of a source that another conversion copied in the same format since this one was confirmed
/// (<see cref="ConvertOutcome.Superseded"/>). A sequence whose photos changed, that is busy, or has a photo that fails is
/// left as it was and reported.
/// </summary>
public sealed class MassConverter
{
/// <summary>What the frames of a sequence no longer have once their files were replaced (<see cref="FrameTrash.ReplaceAsync"/>).</summary>
private const SequenceStatus Uncovered = SequenceStatus.AnalyzedLight | SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
/// <summary>How the session key of a new sequence a conversion makes begins (<see cref="SessionKey"/>).</summary>
private const string KeyPrefix = "convert:";
/// <summary>Held while a confirmed conversion is checked against those pending and queued (<see cref="QueueAsync"/>): one at a time in this process.</summary>
private static readonly SemaphoreSlim Queueing = new(1, 1);
@@ -186,8 +206,8 @@ public sealed class MassConverter
var busy = new HashSet<Guid>(await _trash.BusyAsync(job, cancellationToken).ConfigureAwait(false));
if (job is not { } self)
busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken).ConfigureAwait(false));
else if (await _db.Jobs.GetAsync(self, cancellationToken).ConfigureAwait(false) is { } running)
busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, running.Created, cancellationToken).ConfigureAwait(false));
else if (await _db.Jobs.GetCreatedAsync(self, cancellationToken).ConfigureAwait(false) is { } queued)
busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, queued, cancellationToken).ConfigureAwait(false));
return busy;
}
@@ -195,8 +215,11 @@ public sealed class MassConverter
/// Queues the job of the confirmed <paramref name="plan"/> (<see cref="ArchiveJobs.Convert"/>) through <paramref name="enqueue"/>,
/// unless a conversion of a sequence it converts is queued, running or paused already: another dialog or tab, an answer
/// lost and Apply pressed again, a dry run made before that one was queued or none at all. Checked and queued under one
/// lock of this process, so two confirmations at once queue one job. <paramref name="enqueue"/> runs in the caller's
/// context (a desktop's tells the user on its UI thread).
/// lock of this process, so two confirmations at once queue one job. First, which photos it converts are written under
/// the job's id beside the journal it starts (<see cref="ConfirmedPhotos"/>); <paramref name="enqueue"/> queues the
/// request under that id (<see cref="ArchiveJobRequest.Id"/>), and runs in the caller's context (a desktop's tells the
/// user on its UI thread). When they cannot be written (the archive went away since its dry run), the job is queued all
/// the same and holds each sequence to its whole plan.
/// </summary>
/// <exception cref="ConversionQueuedException">A conversion of one of its sequences waits or runs; nothing was queued.</exception>
/// <exception cref="ArgumentException">The plan converts nothing.</exception>
@@ -204,13 +227,25 @@ public sealed class MassConverter
{
ArgumentNullException.ThrowIfNull(enqueue);
var request = ArchiveJobs.Convert(archive, plan);
var paths = new ArchivePaths(archive.RootPath);
var job = request.Id!.Value;
// No ConfigureAwait(false) here: each wait comes back to the caller's context, where enqueue is called.
await Queueing.WaitAsync(cancellationToken);
try
{
var pending = await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken);
if (request.Targets.Where(pending.Contains).ToList() is { Count: > 0 } queued) throw new ConversionQueuedException(queued);
await enqueue(request);
var written = await Task.Run(() => ConfirmedPhotos.TryWrite(paths, job, plan), cancellationToken);
try
{
await enqueue(request);
}
catch
{
// Not queued: no job will read them.
if (written) ConfirmedPhotos.Delete(paths, job);
throw;
}
}
finally
{
@@ -260,6 +295,7 @@ public sealed class MassConverter
var steps = parameters.Sequences.DistinctBy(s => s.Id).ToList();
var staged = journal.Staged(paths);
var pins = journal.Pins(paths);
var confirmed = ConfirmedPhotos.Read(paths, jobId);
// What is still to do: every sequence not final in an earlier attempt that is still in the archive, its sidecar settled
// first when a writer a crash interrupted left a journal in its folder (this job's replacement among them).
var rows = new Dictionary<Guid, Sequence>();
@@ -277,11 +313,15 @@ public sealed class MassConverter
var unstaged = new Dictionary<Guid, long>();
foreach (var p in plan.Sequences.Where(p => p.Blocked.Count == 0))
{
if (Confirmed(byId[p.SequenceId], p, pins.Of(p.SequenceId)) is not { Frames.Count: > 0 } converts) continue;
if (Confirmed(byId[p.SequenceId], confirmed.Of(p.SequenceId), p, pins.Of(p.SequenceId)) is not { Frames.Count: > 0 } converts) continue;
progress.Count(converts.SequenceId, converts.Frames.Count, staged.Of(converts.SequenceId, converts.FramesVersion).Count);
unstaged[converts.SequenceId] = Unstaged(paths, jobId, converts, staged);
// What a new sequence this job made holds already takes no space again.
var filed = parameters.Mode == ConvertMode.NewSequence
? await FiledAsync(archive, SessionKey(jobId, converts.SequenceId), cancellationToken).ConfigureAwait(false)
: null;
unstaged[converts.SequenceId] = Unstaged(paths, jobId, converts, staged, filed);
}
var space = new SpaceWatch(this, paths);
var space = new SpaceWatch(() => FreeSpace(paths));
EnsureSpace(unstaged.Values.Sum(), space);
await callbacks.Progress(progress.Done, progress.Total).ConfigureAwait(false);
@@ -309,8 +349,8 @@ public sealed class MassConverter
// The space for what is left, measured again: the volume may have filled since, and the files staged so far tell
// how far off the estimate is.
EnsureSpace(unstaged.Values.Sum(), space);
done = await ConvertOneAsync(archive, paths, step, row, options, parameters.Mode, jobId, by, journal, pins, staged, progress, space, callbacks,
cancellationToken).ConfigureAwait(false);
done = await ConvertOneAsync(archive, paths, step, row, options, parameters.Mode, jobId, by, journal, confirmed, pins, staged, progress, space,
callbacks, cancellationToken).ConfigureAwait(false);
}
unstaged.Remove(step.Id);
var result = done.Result;
@@ -349,16 +389,37 @@ public sealed class MassConverter
/// <summary>
/// One sequence: under its lease, its sidecar settled, planned again and held to what was confirmed (the photos pinned the
/// first time); then stage A without the lease, and stage B.
/// first time); then stage A without the lease, and stage B. A New sequence that has its copy already is not converted
/// again (<see cref="CopiedAsync"/>).
/// </summary>
private async Task<StepDone> ConvertOneAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options,
ConvertMode mode, Guid jobId, ConvertRequester by, ConvertJournal journal, PinLog pins, StagedLog staged, Progress progress, SpaceWatch space,
ConvertCallbacks callbacks, CancellationToken cancellationToken)
ConvertMode mode, Guid jobId, ConvertRequester by, ConvertJournal journal, ConfirmedPhotos confirmedPhotos, PinLog pins, StagedLog staged,
Progress progress, SpaceWatch space, ConvertCallbacks callbacks, CancellationToken cancellationToken)
{
// Another job holds it now (a Reorganize paused since the job started, a render, a conversion of it queued first): it
// waits for the retry.
var busy = await HeldAsync(jobId, cancellationToken).ConfigureAwait(false);
if (busy.Contains(step.Id) || busy.Contains(archive.Id)) return new StepDone(Result(row, ConvertOutcome.Blocked));
var key = SessionKey(jobId, row.Id);
if (mode == ConvertMode.NewSequence)
{
try
{
if (await CopiedAsync(archive, paths, step, row, options, jobId, by, journal, pins, staged, key, progress, cancellationToken).ConfigureAwait(false)
is { } copied)
return copied;
}
catch (ArchiveOfflineException)
{
throw;
}
catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
{
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the copy this conversion made of it could not be finished yet", ex);
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex)));
}
}
var photos = confirmedPhotos.Of(step.Id);
SequencePaths folder;
SequenceManifest manifest;
ConvertSequencePlan planned;
@@ -379,7 +440,7 @@ public sealed class MassConverter
manifest = SequenceWriter.ReadLocked(folder, row.Id);
FrameTrash.EnsureReplaced(folder);
var pin = pins.Of(step.Id);
if (mode == ConvertMode.Replace && ReplacedEarlier(manifest, step, pin, staged.Of(step.Id, step.FramesVersion), options) is { } frames)
if (mode == ConvertMode.Replace && ReplacedEarlier(manifest, step, photos, pin, staged.Of(step.Id, step.FramesVersion), options) is { } frames)
{
// Replaced by this job already: a crash after its stage B, before the journal said so (the settle may just have
// completed it).
@@ -394,11 +455,11 @@ public sealed class MassConverter
IReadOnlySet<string> files = new HashSet<string>(FileRenames.ListPresent(at, sidecar), StringComparer.OrdinalIgnoreCase);
return (ConvertPlanner.Plan(sidecar, at, files, options, decoder, title), files);
}, cancellationToken).ConfigureAwait(false);
if (Confirmed(step, fresh, pin) is not { } confirmed)
if (Confirmed(step, photos, fresh, pin) is not { } confirmed)
{
// A photo it converts that another program holds now cannot be told from one that changed: the retry, once it
// is free, can.
if (await Task.Run(() => Held(step, pin, sidecar, at, present, fresh, options, decoder), cancellationToken).ConfigureAwait(false))
if (await Task.Run(() => Held(photos, pin, sidecar, at, present, fresh, options, decoder), cancellationToken).ConfigureAwait(false))
{
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: a photo it converts cannot be opened now; it is not converted");
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.PhotoHeld")));
@@ -426,7 +487,6 @@ public sealed class MassConverter
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex)));
}
var staging = StagingOf(paths, jobId, row.Id);
var key = SessionKey(jobId, row.Id);
// A new sequence a resumed job finds again holds the photos it filed before it stopped.
var filed = mode == ConvertMode.NewSequence
? await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false)
@@ -443,7 +503,7 @@ public sealed class MassConverter
{
return mode == ConvertMode.Replace
? await ReplaceAsync(archive, paths, row, planned, ready, staging, options, jobId, by.Name, cancellationToken).ConfigureAwait(false)
: await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, cancellationToken).ConfigureAwait(false);
: await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, journal, cancellationToken).ConfigureAwait(false);
}
catch (LeaseUnavailableException ex)
{
@@ -479,15 +539,97 @@ public sealed class MassConverter
};
/// <summary>
/// What the job converts of a sequence whose plan is now <paramref name="fresh"/>: the photos it pinned when it first came to
/// it, each still planned with the content it had then; the first time, the photos confirmed (<see cref="ConvertSequence.Photos"/>)
/// while each is still planned with the content confirmed (the step's digest). Photos that came in, were verified or could
/// be opened again since stay as they are, so an upload while the job waits its turn neither changes what it converts
/// nor holds it back. A job an earlier build queued: the whole plan while it has the step's digest, or its frames
/// version. Null when a photo confirmed no longer converts as it was: the sequence changed after the plan, and the user
/// plans again.
/// What a <see cref="ConvertMode.NewSequence"/> conversion does with a source that has its copy already when the job comes
/// to it (a retry, a resume): the copy this job made (its journal names it, or its session key when a crash came before
/// the journal did) is finished, whatever changed in the source since, once it holds every photo the job pinned, and one
/// gone since is not made again without a new plan; a copy in this format that another conversion made since this one was
/// confirmed (<see cref="ConvertRequester.Confirmed"/>) supersedes this one's. Null when neither: the sequence converts,
/// and a copy holding only some of its photos (a crash while they were filed) gets the rest.
/// </summary>
internal static ConvertSequencePlan? Confirmed(ConvertSequence step, ConvertSequencePlan fresh, PinnedSequence? pin)
private async Task<StepDone?> CopiedAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options, Guid jobId,
ConvertRequester by, ConvertJournal journal, PinLog pins, StagedLog staged, string key, Progress progress, CancellationToken cancellationToken)
{
if ((journal.Of(step.Id)?.NewSequenceId ?? await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, cancellationToken).ConfigureAwait(false)) is { } made)
{
if (await _db.Sequences.GetAsync(made, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id)
{
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it is gone; no other is made without a new plan");
return new StepDone(Result(row, ConvertOutcome.Changed));
}
return pins.Of(step.Id) is { } pin
? await FinishCopyAsync(archive, paths, row, pin, staged, key, made, progress, cancellationToken).ConfigureAwait(false)
: null;
}
if (by.Confirmed is not { } confirmed
|| await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } other)
return null;
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed; no second copy is made");
progress.Count(row.Id, 0, 0);
return new StepDone(Result(row, ConvertOutcome.Superseded));
}
/// <summary>
/// The copy this job made of a source, finished when it holds the conversion of every photo pinned (a crash came after
/// they were filed, before the journal said so): it takes the source's tags, rating, notes and project as stage B gives
/// them, whatever became of the photos of the source since. Null while a photo pinned is not in it, or when the source
/// is no longer where this index has it: the job converts the sequence, which files the rest.
/// </summary>
private async Task<StepDone?> FinishCopyAsync(Archive archive, ArchivePaths paths, Sequence row, PinnedSequence pin, StagedLog staged, string key, Guid made,
Progress progress, CancellationToken cancellationToken)
{
var conversions = staged.Of(row.Id, pin.FramesVersion).ToDictionary(c => c.FrameId);
var filed = await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false);
var ready = new Dictionary<Guid, StagedConversion>();
foreach (var photo in pin.Frames)
{
if (!conversions.TryGetValue(photo.Id, out var conversion) || !string.Equals(conversion.Source, photo.Sha256, StringComparison.OrdinalIgnoreCase)
|| !filed.Contains(conversion.Sha256))
return null;
ready[photo.Id] = conversion;
}
SequenceManifest manifest;
string? project;
await using (await _writer.LockAsync(paths, row.Id, cancellationToken).ConfigureAwait(false))
{
if (await _db.Sequences.GetAsync(row.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id) return null;
row = now;
var folder = paths.Sequence(row.Folder);
if (!SequenceFolders.Holds(folder, row.Id)) return null;
manifest = SequenceWriter.ReadLocked(folder, row.Id);
project = row.SettingsJson ?? (File.Exists(folder.ProjectPath) ? AtomicFile.ReadAllText(folder.ProjectPath) : null);
}
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it holds every photo already; it is finished, no other is made");
progress.Count(row.Id, pin.Frames.Count, pin.Frames.Count);
return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, made, pin.Frames.Count, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// A copy of <paramref name="source"/> in this conversion's format that another conversion made since <paramref name="confirmed"/>:
/// a sequence filed under another job's session key for that source (<see cref="SessionKey"/>) whose sidecar says it was
/// made since then and holds photos of that format. Null when there is none.
/// </summary>
private async Task<Guid?> LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed,
CancellationToken cancellationToken)
{
foreach (var (key, id) in await _db.Sequences.ListSessionKeysAsync(archive.Id, $"{KeyPrefix}%:{source:N}", cancellationToken).ConfigureAwait(false))
{
if (JobOfKey(key) is not { } other || other == jobId) continue;
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id) continue;
if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format)) return id;
}
return null;
}
/// <summary>
/// What the job converts of a sequence whose plan is now <paramref name="fresh"/>: the photos it pinned when it first came to
/// it, each still planned with the content it had then; the first time, the photos confirmed (<paramref name="photos"/>,
/// <see cref="ConfirmedPhotos"/>) while each is still planned with the content confirmed (the step's digest). Photos that
/// came in, were verified or could be opened again since stay as they are, so an upload while the job waits its turn
/// neither changes what it converts nor holds it back. Without the photos confirmed (they could not be written, or a job
/// an earlier build queued): the whole plan while it has the step's digest, or its frames version. Null when a photo
/// confirmed no longer converts as it was: the sequence changed after the plan, and the user plans again.
/// </summary>
internal static ConvertSequencePlan? Confirmed(ConvertSequence step, IReadOnlyList<Guid>? photos, ConvertSequencePlan fresh, PinnedSequence? pin)
{
var now = fresh.Frames.ToDictionary(f => f.FrameId);
if (pin is not null)
@@ -500,7 +642,7 @@ public sealed class MassConverter
}
return frames.Count > 0 ? fresh with { FramesVersion = pin.FramesVersion, Frames = frames } : null;
}
if (step is { Frames: { } confirmedDigest } && step.PhotoIds() is { Count: > 0 } photos)
if (step is { Frames: { } confirmedDigest } && photos is { Count: > 0 })
{
var frames = new List<ConvertFramePlan>(photos.Count);
foreach (var id in photos)
@@ -520,15 +662,15 @@ public sealed class MassConverter
/// <summary>
/// Whether the plan made now (<paramref name="fresh"/>) leaves out a photo the job converts of the sequence (one it pinned,
/// else one confirmed) only because its file could not be opened (<see cref="ConvertSkip.Inaccessible"/>: another program
/// holds it): the sequence then waits for the retry rather than counting as changed. Photos the job does not convert do
/// not count, nor does a confirmed photo that changed meanwhile. A job an earlier build queued, which names no photos:
/// any photo of the sequence that could not be opened.
/// else one of <paramref name="photos"/>, those confirmed) only because its file could not be opened
/// (<see cref="ConvertSkip.Inaccessible"/>: another program holds it): the sequence then waits for the retry rather than
/// counting as changed. Photos the job does not convert do not count, nor does a confirmed photo that changed meanwhile.
/// Without the photos confirmed: any photo of the sequence that could not be opened.
/// </summary>
private static bool Held(ConvertSequence step, PinnedSequence? pin, SequenceManifest manifest, SequencePaths folder, IReadOnlySet<string> present,
private static bool Held(IReadOnlyList<Guid>? photos, PinnedSequence? pin, SequenceManifest manifest, SequencePaths folder, IReadOnlySet<string> present,
ConvertSequencePlan fresh, ConvertOptions options, ImageDecoder decoder)
{
IReadOnlyList<Guid>? converts = pin is not null ? [.. pin.Frames.Select(f => f.Id)] : step.PhotoIds();
IReadOnlyList<Guid>? converts = pin is not null ? [.. pin.Frames.Select(f => f.Id)] : photos;
if (converts is null) return fresh.Skipped.Any(s => s.Reason == ConvertSkip.Inaccessible);
var planned = fresh.Frames.Select(f => f.FrameId).ToHashSet();
var frames = manifest.Frames.ToDictionary(f => f.Id);
@@ -579,12 +721,13 @@ public sealed class MassConverter
/// <summary>
/// Stage A: converts the planned frames not staged yet, several at once, each staged file appended to the journal as it
/// is consumed (in frame order). A photo starts only while the volume has the space it and the photos after it that may
/// convert at once with it take (as many as the host converts at once, never photos that are not there to start), with
/// the reserve: short of it the job stops (<c>Archive.Error.ConvertSpace</c>) before the volume fills. The space the
/// photos converting already take is in the volume's free space as they write it, so it is not counted twice. Returns
/// the first failure (the file and why, localised; the exception goes to the diagnostics log); the frames converted
/// meanwhile stay staged for a retry.
/// is consumed (in frame order). A photo starts only while the volume has the space for it and for every photo handed to
/// the work before it whose file has not landed yet (<see cref="SpaceWatch.Start"/>: converting or waiting for its turn,
/// up to twice as many as convert at once, at the end of a sequence as anywhere), with the margin and the reserve: short
/// of it the job stops (<c>Archive.Error.ConvertSpace</c>) before the volume fills. A converted file is encoded in memory
/// and lands whole, by a rename, before it is verified: what those photos will write is not in the free space measured
/// yet, and what landed is, so each counts until its file lands, never twice. Returns the first failure (the file and why,
/// localised; the exception goes to the diagnostics log); the frames converted meanwhile stay staged for a retry.
/// </summary>
/// <exception cref="IOException">The volume lacks the space for the next photo; what was staged stays.</exception>
private async Task<string?> StageAsync(Sequence row, SequenceManifest manifest, SequencePaths folder, ConvertSequencePlan planned,
@@ -622,13 +765,26 @@ public sealed class MassConverter
}, cancellationToken);
for (var i = 0; i < todo.Count && failure is null; i++)
{
if (space.Short(Ahead(todo, i, atOnce)) is { } message)
var frame = frames[todo[i].FrameId];
var target = Path.Combine(staging, frame.Id.ToString("N") + options.Extension);
if (space.Start(todo[i].EstimatedBytes, target) is { } message)
{
full = message;
break;
}
var frame = frames[todo[i].FrameId];
await work.AddAsync(token => Stage(frame, manifest, folder, staging, options, decoder, threads, token)).ConfigureAwait(false);
// A photo that never starts (the job stops: cancelled, or a write of the journal failed) is never counted off,
// and the watch goes with the job.
await work.AddAsync(token =>
{
try
{
return Stage(frame, manifest, folder, target, options, decoder, threads, token);
}
finally
{
space.Ended(target);
}
}).ConfigureAwait(false);
}
await work.CompleteAsync().ConfigureAwait(false);
if (full is not null)
@@ -640,20 +796,8 @@ public sealed class MassConverter
return failure;
}
/// <summary>
/// The bytes photo <paramref name="index"/> of <paramref name="todo"/> and the photos after it that may convert at once with
/// it are estimated at: <paramref name="atOnce"/> of them, fewer near the end (the space a photo of stage A starts with).
/// </summary>
internal static long Ahead(IReadOnlyList<ConvertFramePlan> todo, int index, int atOnce)
{
ArgumentNullException.ThrowIfNull(todo);
long bytes = 0;
for (var i = index; i < Math.Min(todo.Count, index + Math.Max(1, atOnce)); i++) bytes += todo[i].EstimatedBytes;
return bytes;
}
/// <summary>One photo of stage A: hashed against its record, converted into the staging folder and verified.</summary>
private static Staged Stage(SidecarFrame frame, SequenceManifest manifest, SequencePaths folder, string staging, ConvertOptions options, ImageDecoder decoder,
/// <summary>One photo of stage A: hashed against its record, converted into <paramref name="target"/> (in the staging folder) and verified.</summary>
private static Staged Stage(SidecarFrame frame, SequenceManifest manifest, SequencePaths folder, string target, ConvertOptions options, ImageDecoder decoder,
int threads, CancellationToken cancellationToken)
{
var source = folder.File(frame.File);
@@ -666,7 +810,6 @@ public sealed class MassConverter
if (!string.Equals(sha, frame.Sha256, StringComparison.OrdinalIgnoreCase))
throw new ConversionException(ConversionFailure.Changed, $"{frame.File} no longer matches its SHA-256; verify the sequence");
cancellationToken.ThrowIfCancellationRequested();
var target = Path.Combine(staging, frame.Id.ToString("N") + options.Extension);
var file = FrameConverter.Convert(new ConvertSource(source, frame, manifest.Camera, manifest.Lens), options, decoder, target, threads, cancellationToken);
return new Staged(frame.File, new StagedConversion(frame.Id, sha, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height), null);
}
@@ -734,9 +877,12 @@ public sealed class MassConverter
/// source's capture facts; then the source's tags, rating, notes and project (its positions translated to the new frames).
/// Every conversion not filed yet must still be staged whole: one gone since (a clean-up, a quarantine) fails the sequence
/// rather than leave its photo out. The photos the plan did not convert are not in the new sequence (<see cref="ConvertSequenceResult.LeftOut"/>).
/// Once its photos are filed, the journal names the new sequence (<see cref="ConvertJournal.Made"/>), so that a retry or a
/// resume finishes it and never makes a second one.
/// </summary>
private async Task<StepDone> CreateAsync(Archive archive, ArchivePaths paths, Sequence row, ConvertSequencePlan planned,
Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options, string key, Guid? requestedBy, CancellationToken cancellationToken)
Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options, string key, Guid? requestedBy, ConvertJournal journal,
CancellationToken cancellationToken)
{
SequenceManifest manifest;
string? project;
@@ -775,13 +921,26 @@ public sealed class MassConverter
await _ingest.CommitSessionAsync(archive, new SequenceTarget(key, title, null, owner), pending, CancellationToken.None).ConfigureAwait(false);
var id = await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, CancellationToken.None).ConfigureAwait(false)
?? throw new IOException($"No converted photo of {row.DisplayTitle} could be filed into a new sequence.");
journal.Made(row.Id, id);
journal.Write(paths);
return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, id, planned.Frames.Count, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// The end of stage B of <see cref="ConvertMode.NewSequence"/>, once the new sequence <paramref name="id"/> holds the
/// <paramref name="frames"/> photos converted: the source's tags, rating, notes and project given to it (<see cref="CopyAsync"/>),
/// and the passes it needs, from what the source has. The photos of the source it does not hold are counted as left out.
/// </summary>
private async Task<StepDone> CompleteCopyAsync(Archive archive, ArchivePaths paths, Sequence row, SequenceManifest manifest, string? project,
Dictionary<Guid, StagedConversion> ready, Guid id, int frames, CancellationToken cancellationToken)
{
await CopyAsync(paths, row, manifest, project, ready, id, cancellationToken).ConfigureAwait(false);
var made = await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false)
?? throw new InvalidOperationException($"The new sequence {id} is not indexed.");
var leftOut = manifest.Frames.Count - planned.Frames.Count;
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: {planned.Frames.Count} photo(s) converted into the new sequence {made.DisplayTitle}, {leftOut} left in the source alone");
var leftOut = manifest.Frames.Count(f => !ready.ContainsKey(f.Id));
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: {frames} photo(s) converted into the new sequence {made.DisplayTitle}, {leftOut} left in the source alone");
var had = row.Status & (SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt);
return new StepDone(new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Created, planned.Frames.Count, id, LeftOut: leftOut), had,
return new StepDone(new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Created, frames, id, LeftOut: leftOut), had,
_frames.FollowUps(archive, made, had));
}
@@ -855,13 +1014,13 @@ public sealed class MassConverter
/// <summary>
/// The photos converted, when this job replaced the sequence's frames already (a crash after its stage B, before the
/// journal said so): every photo it pinned is a frame holding a file of the target's format converted from the content
/// pinned. Without a pin (its journal lost), every photo confirmed (<see cref="ConvertSequence.Photos"/>) is a frame
/// holding a conversion to the target whose sources have the step's digest; a job an earlier build queued: the frames
/// holding a conversion to the target are, by their sources, the photos of the step's digest, or every staged conversion
/// is a frame holding the converted content. Null otherwise.
/// pinned. Without a pin (its journal lost), every photo confirmed (<paramref name="photos"/>) is a frame holding a
/// conversion to the target whose sources have the step's digest; without those either: the frames holding a conversion
/// to the target are, by their sources, the photos of the step's digest, or (a job an earlier build queued) every staged
/// conversion is a frame holding the converted content. Null otherwise.
/// </summary>
private static int? ReplacedEarlier(SequenceManifest manifest, ConvertSequence step, PinnedSequence? pin, IReadOnlyList<StagedConversion> noted,
ConvertOptions options)
private static int? ReplacedEarlier(SequenceManifest manifest, ConvertSequence step, IReadOnlyList<Guid>? photos, PinnedSequence? pin,
IReadOnlyList<StagedConversion> noted, ConvertOptions options)
{
var frames = manifest.Frames.ToDictionary(f => f.Id);
if (pin is not null)
@@ -872,7 +1031,7 @@ public sealed class MassConverter
? pin.Frames.Count
: null;
}
if (step is { Frames: { } confirmed } && step.PhotoIds() is { Count: > 0 } photos)
if (step is { Frames: { } confirmed } && photos is { Count: > 0 })
{
var sources = new List<(Guid, string?)>(photos.Count);
foreach (var id in photos)
@@ -987,11 +1146,16 @@ public sealed class MassConverter
return null;
}
/// <summary>The bytes the photos <paramref name="plan"/> converts are estimated at, but for those whose staged file is there already.</summary>
private static long Unstaged(ArchivePaths paths, Guid jobId, ConvertSequencePlan plan, StagedLog staged)
/// <summary>
/// The bytes the photos <paramref name="plan"/> converts are estimated at, but for those whose staged file is there already,
/// or which the new sequence this job made holds (<paramref name="filed"/>, the SHA-256 of its photos).
/// </summary>
private static long Unstaged(ArchivePaths paths, Guid jobId, ConvertSequencePlan plan, StagedLog staged, IReadOnlySet<string>? filed)
{
var folder = StagingOf(paths, jobId, plan.SequenceId);
var there = staged.Of(plan.SequenceId, plan.FramesVersion).Where(c => Present(Path.Combine(folder, c.File), c)).Select(c => c.FrameId).ToHashSet();
var there = staged.Of(plan.SequenceId, plan.FramesVersion)
.Where(c => filed?.Contains(c.Sha256) == true || Present(Path.Combine(folder, c.File), c))
.Select(c => c.FrameId).ToHashSet();
return plan.Frames.Where(f => !there.Contains(f.FrameId)).Sum(f => f.EstimatedBytes);
}
@@ -1042,6 +1206,7 @@ public sealed class MassConverter
ConvertOutcome.Converted => Loc.L("Archive.Log.Converted", result.Title, result.Frames),
ConvertOutcome.Created => Loc.L("Archive.Log.ConvertedNew", result.Title, result.Frames),
ConvertOutcome.Nothing => Loc.L("Archive.Log.ConvertNothing", result.Title),
ConvertOutcome.Superseded => Loc.L("Archive.Log.ConvertSuperseded", result.Title),
ConvertOutcome.Changed => Loc.L("Archive.Log.ConvertChanged", result.Title),
// Busy, or why the retry may do better (a photo another program holds, a folder another index moved), in one sentence.
ConvertOutcome.Blocked when result.Error is null => Loc.L("Archive.Log.ConvertBlocked", result.Title),
@@ -1067,7 +1232,11 @@ public sealed class MassConverter
: Loc.L("Common.Size.MB", (bytes / 1e6).ToString(digits, Loc.Culture));
/// <summary>The session key of the new sequence a conversion makes from <paramref name="source"/>.</summary>
internal static string SessionKey(Guid job, Guid source) => $"convert:{job:N}:{source:N}";
internal static string SessionKey(Guid job, Guid source) => $"{KeyPrefix}{job:N}:{source:N}";
/// <summary>The conversion job a session key of <see cref="SessionKey"/> names; null for any other key.</summary>
private static Guid? JobOfKey(string key) =>
key.Split(':') is ["convert", var job, _] && Guid.TryParseExact(job, "N", out var id) ? id : null;
/// <summary>The folder under <c>.chronos/staging</c> a conversion job stages into, one subfolder per sequence.</summary>
internal static string StagingFolder(Guid job) => ConvertJournal.Prefix + job.ToString("N");
@@ -1112,34 +1281,68 @@ public sealed class MassConverter
}
/// <summary>
/// The free space a running conversion watches: measured where it writes, against estimates scaled by how far the files it
/// staged came out above theirs (photos of a 14-bit RAW carry more noise than the estimate's bits per pixel allow for).
/// The free space a running conversion watches: measured where it writes (<paramref name="free"/>), against estimates scaled
/// by how far the files it staged came out above theirs (photos of a 14-bit RAW carry more noise than the estimate's bits
/// per pixel allow for), and, in stage A, with the photos started whose converted file is not there yet (<see cref="Start"/>):
/// a converted file is encoded in memory and lands whole, by a rename, before its photo's work verifies it, so the free
/// space measured holds every file landed, and none of what the others will write.
/// </summary>
private sealed class SpaceWatch(MassConverter converter, ArchivePaths paths)
internal sealed class SpaceWatch(Func<long?> free)
{
private readonly ConcurrentDictionary<string, Starting> _started = new(StringComparer.Ordinal);
private long _actual;
private long _estimated;
/// <summary>The bytes staged over their estimates, at least 1.</summary>
public double Ratio => _estimated > 0 ? Math.Max(1, (double)_actual / _estimated) : 1;
/// <summary>
/// The estimated bytes of the photos started (<see cref="Start"/>) whose converted file has not landed yet: the file is
/// not at its path, or is still the one that was there when the photo started (one a crash left, which it replaces).
/// </summary>
public long Writing => _started.Sum(s => Landed(s.Key) == s.Value.Before ? s.Value.Estimate : 0);
public void Staged(long actual, long estimated)
{
_actual += actual;
_estimated += Math.Max(1, estimated);
}
/// <summary>
/// Why a photo estimated at <paramref name="estimated"/> bytes, converted into <paramref name="target"/>, cannot start now:
/// the volume lacks the space for it and for the photos started before it whose file has not landed yet
/// (<see cref="Writing"/>, <see cref="Short"/>), localized. Null when it can: it counts among those until its file lands
/// or its work ends (<see cref="Ended"/>).
/// </summary>
public string? Start(long estimated, string target)
{
var before = Landed(target);
if (Short(Math.Max(0, estimated) + Writing) is { } message) return message;
_started[target] = new Starting(Math.Max(0, estimated), before);
return null;
}
/// <summary>The work of the photo converted into <paramref name="target"/> ended: its file landed, or never will.</summary>
public void Ended(string target) => _started.TryRemove(target, out _);
/// <summary>
/// Why the volume cannot take <paramref name="estimated"/> bytes more (scaled, with the margin and the reserve), localized;
/// null when it can, or when its free space cannot be measured.
/// </summary>
public string? Short(long estimated)
{
var space = new ConvertSpace(ConvertPlanner.Required((long)Math.Ceiling(estimated * Ratio)), converter.FreeSpace(paths));
var space = new ConvertSpace(ConvertPlanner.Required((long)Math.Ceiling(estimated * Ratio)), free());
if (space.Enough) return null;
var (required, free) = Sizes(space.Required, space.Free ?? 0);
return Loc.L("Archive.Error.ConvertSpace", required, free);
var (required, left) = Sizes(space.Required, space.Free ?? 0);
return Loc.L("Archive.Error.ConvertSpace", required, left);
}
/// <summary>The file at <paramref name="path"/> as a landing changes it, its size and time of writing; null when there is none.</summary>
private static (long Size, long Written)? Landed(string path) =>
new FileInfo(path) is { Exists: true } file ? (file.Length, file.LastWriteTimeUtc.Ticks) : null;
/// <summary>A photo started: its estimate, and the file at its path then.</summary>
private sealed record Starting(long Estimate, (long Size, long Written)? Before);
}
/// <summary>What a sequence came to: its result, the statuses its follow-ups follow from, and those follow-ups (null: none to queue).</summary>
@@ -1171,12 +1374,13 @@ internal sealed record PinnedFrame(Guid Id, string Sha256);
/// <summary>
/// The journal of a conversion job (<c>.chronos/journal/convert-&lt;job&gt;.json</c>, owner review 2, R7): what each sequence
/// came to, and whether the passes it needs were queued, written after each one; with it, the pin log
/// (<c>convert-&lt;job&gt;.pins</c>, <see cref="PinLog"/>), the photos the job converts of each sequence, and the staged log
/// (<c>convert-&lt;job&gt;.staged</c>), one line per photo staged, both appended and flushed as they grow, so a job of thousands
/// of photos journals each once. A resumed or retried job keeps the sequences that are final, the photos pinned, and the
/// staged files whose hash still verifies. All of it goes when every sequence is final, or once the job is gone
/// (<see cref="ArchiveMaintenance"/>).
/// came to, the new sequence a New sequence conversion made of it as soon as its photos are filed, and whether the passes
/// it needs were queued, written after each one; with it, the photos confirmed (<c>convert-&lt;job&gt;.photos</c>,
/// <see cref="ConfirmedPhotos"/>, written when the job is queued), the pin log (<c>convert-&lt;job&gt;.pins</c>,
/// <see cref="PinLog"/>), the photos the job converts of each sequence, and the staged log (<c>convert-&lt;job&gt;.staged</c>),
/// one line per photo staged, both appended and flushed as they grow, so a job of thousands of photos journals each once. A
/// resumed or retried job keeps the sequences that are final, the copies made, the photos pinned, and the staged files
/// whose hash still verifies. All of it goes when every sequence is final, or once the job is gone (<see cref="ArchiveMaintenance"/>).
/// </summary>
internal sealed class ConvertJournal
{
@@ -1198,7 +1402,10 @@ internal sealed class ConvertJournal
noted.Had = had;
}
/// <summary>Notes what a sequence came to and, when it was converted or made, the statuses its follow-ups follow from.</summary>
/// <summary>
/// Notes what a sequence came to and, when it was converted or made, the statuses its follow-ups follow from. The new
/// sequence a New sequence conversion made of it stays named whatever it came to since (<see cref="Made"/>).
/// </summary>
public void Note(ConvertSequenceResult result, SequenceStatus? had = null)
{
var noted = Of(result.SequenceId);
@@ -1206,10 +1413,21 @@ internal sealed class ConvertJournal
noted.Title = result.Title;
noted.Outcome = result.Outcome;
noted.Frames = result.Frames;
noted.NewSequenceId = result.NewSequenceId;
noted.NewSequenceId = result.NewSequenceId ?? noted.NewSequenceId;
if (had is { } status) noted.Had = status;
}
/// <summary>
/// Notes the new sequence a New sequence conversion filed the photos of <paramref name="id"/> into, before anything else is
/// done with it: a retry or a resume finishes that one and makes no other.
/// </summary>
public void Made(Guid id, Guid newSequenceId)
{
var noted = Of(id);
if (noted is null) Sequences.Add(noted = new ConvertJournalSequence { Id = id });
noted.NewSequenceId = newSequenceId;
}
public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.json");
public static string StagedPathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.staged");
@@ -1256,8 +1474,9 @@ internal sealed class ConvertJournal
/// <summary>The pin log of this job, read now.</summary>
public PinLog Pins(ArchivePaths paths) => PinLog.Open(PinsPathFor(paths, JobId));
/// <summary>The journal files of <paramref name="job"/>, whichever exist.</summary>
public static IEnumerable<string> Files(ArchivePaths paths, Guid job) => [StagedPathFor(paths, job), PinsPathFor(paths, job), PathFor(paths, job)];
/// <summary>The journal files of <paramref name="job"/>, whichever exist: the photos confirmed among them (<see cref="ConfirmedPhotos"/>).</summary>
public static IEnumerable<string> Files(ArchivePaths paths, Guid job) =>
[StagedPathFor(paths, job), PinsPathFor(paths, job), ConfirmedPhotos.PathFor(paths, job), PathFor(paths, job)];
public static void Delete(ArchivePaths paths, Guid job)
{
@@ -1291,7 +1510,7 @@ internal sealed class ConvertJournalSequence
public bool Queued { get; set; }
[JsonIgnore]
public bool Final => Outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing;
public bool Final => ConvertSequenceResult.IsFinal(Outcome);
}
/// <summary>The staged log of a conversion job (<see cref="ConvertJournal"/>): read once, appended to line by line.</summary>
@@ -1380,6 +1599,114 @@ internal sealed class PinLog
}
}
/// <summary>A line of <see cref="ConfirmedPhotos"/>: a sequence, and the frame ids of the photos converted of it, 16 bytes each, in base64.</summary>
internal sealed record ConfirmedLine(Guid Sequence, string Photos);
/// <summary>
/// The photos a conversion job converts of each sequence, as its plan was confirmed (<c>.chronos/journal/convert-&lt;job&gt;.photos</c>,
/// owner review 2, R7): written whole under the job's id before it is queued (<see cref="MassConverter.QueueAsync"/>), one
/// line per sequence with the frame ids in the plan's order. The job's row keeps each sequence's digest of those photos
/// with their content (<see cref="ConvertSequence.Frames"/>), never the ids: it stays as small as the number of sequences
/// whatever the number of photos, for every list and check that reads jobs. Under each sequence's lease the job takes these
/// photos from its plan made again and goes on only while they have that digest; a sequence this file does not name (it
/// could not be written, or a job an earlier build queued) is held to its whole plan. It goes with the journal
/// (<see cref="ConvertJournal.Files"/>).
/// </summary>
internal sealed class ConfirmedPhotos
{
private readonly Dictionary<Guid, IReadOnlyList<Guid>> _photos = [];
public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{ConvertJournal.Prefix}{job:N}.photos");
/// <summary>The photos confirmed of <paramref name="sequence"/>, in the plan's order; null when the file does not name it.</summary>
public IReadOnlyList<Guid>? Of(Guid sequence) => _photos.GetValueOrDefault(sequence);
/// <summary>The photos confirmed for <paramref name="job"/>: none when the file is not there; a line that cannot be read names nothing.</summary>
public static ConfirmedPhotos Read(ArchivePaths paths, Guid job)
{
var read = new ConfirmedPhotos();
foreach (var line in JournalLines.Read(PathFor(paths, job)))
{
try
{
if (JsonSerializer.Deserialize(line, ArchiveJobJson.Default.ConfirmedLine) is { } confirmed) read._photos[confirmed.Sequence] = Decode(confirmed.Photos);
}
catch (Exception ex) when (ex is JsonException or FormatException)
{
Core.Diagnostics.Log.Warning("archive", $"A line of the photos confirmed for conversion job {job} cannot be read; that sequence is held to its whole plan", ex);
}
}
return read;
}
/// <summary>
/// Writes the photos <paramref name="plan"/> converts of each sequence it converts, for the job <paramref name="job"/>, whole
/// (a reader finds all of it or none); false, logged, when it cannot be written.
/// </summary>
public static bool TryWrite(ArchivePaths paths, Guid job, ConvertPlan plan)
{
ArgumentNullException.ThrowIfNull(plan);
try
{
using var lines = new MemoryStream();
foreach (var sequence in plan.Converted)
{
lines.Write(JsonSerializer.SerializeToUtf8Bytes(new ConfirmedLine(sequence.SequenceId, Encode(sequence.Frames.Select(f => f.FrameId))),
ArchiveJobJson.Default.ConfirmedLine));
lines.WriteByte((byte)'\n');
}
System.IO.Directory.CreateDirectory(paths.JournalDirectory);
AtomicFile.WriteAllBytes(PathFor(paths, job), lines.GetBuffer().AsSpan(0, (int)lines.Length));
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be written; it holds each sequence to its whole plan", ex);
return false;
}
}
/// <summary>Deletes the file of <paramref name="job"/>; a failure is logged, and maintenance removes it later.</summary>
public static void Delete(ArchivePaths paths, Guid job)
{
try
{
File.Delete(PathFor(paths, job));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be deleted; maintenance removes them later", ex);
}
}
/// <summary>Frame ids as a line holds them: their 16 bytes each, one after the other, in base64.</summary>
public static string Encode(IEnumerable<Guid> ids)
{
ArgumentNullException.ThrowIfNull(ids);
IReadOnlyCollection<Guid> frames = ids as IReadOnlyCollection<Guid> ?? [.. ids];
var bytes = new byte[frames.Count * 16];
var at = 0;
foreach (var frame in frames)
{
frame.TryWriteBytes(bytes.AsSpan(at, 16));
at += 16;
}
return Convert.ToBase64String(bytes);
}
/// <summary>The frame ids <see cref="Encode"/> wrote, in order.</summary>
/// <exception cref="FormatException">The text is not frame ids as <see cref="Encode"/> writes them.</exception>
public static IReadOnlyList<Guid> Decode(string text)
{
ArgumentNullException.ThrowIfNull(text);
var bytes = Convert.FromBase64String(text);
if (bytes.Length % 16 != 0) throw new FormatException("The text is not frame ids.");
var ids = new Guid[bytes.Length / 16];
for (var i = 0; i < ids.Length; i++) ids[i] = new Guid(bytes.AsSpan(i * 16, 16));
return ids;
}
}
/// <summary>Files of JSON lines a job appends to as it goes: each line written whole and flushed to disk, read back skipping blank ones.</summary>
internal static class JournalLines
{
+9 -16
View File
@@ -6,6 +6,7 @@ using Chronos.Core.IO;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Repositories;
namespace Chronos.Data.Archiving;
@@ -115,7 +116,6 @@ public sealed partial class SequenceTrash
public const string FolderName = "trash";
private const string RecordExtension = ".json";
private const int JobPage = 200;
/// <summary>Bytes of a sidecar read to find its id (written second, after the schema version).</summary>
private const int SidecarHead = 4096;
@@ -493,30 +493,23 @@ public sealed partial class SequenceTrash
/// </summary>
internal Task<IReadOnlySet<Guid>> BusyAsync(CancellationToken cancellationToken) => BusyAsync(null, cancellationToken);
/// <summary>What running and paused jobs hold, but the job <paramref name="except"/> (the one asking).</summary>
/// <summary>
/// What running and paused jobs hold, but the job <paramref name="except"/> (the one asking), read from their targets
/// alone (<see cref="JobRepository.ListHoldsAsync"/>): nothing reads the parameters a conversion or an import keeps.
/// </summary>
internal async Task<IReadOnlySet<Guid>> BusyAsync(Guid? except, CancellationToken cancellationToken)
{
var busy = new HashSet<Guid>();
foreach (var state in new[] { JobState.Running, JobState.Paused })
{
string? cursor = null;
do
{
var page = await _db.Jobs.ListAsync(state, JobPage, cursor, cancellationToken).ConfigureAwait(false);
foreach (var job in page.Items.Where(j => j.Id != except)) busy.UnionWith(Holds(job));
cursor = page.NextCursor;
}
while (cursor is not null);
}
foreach (var job in await _db.Jobs.ListHoldsAsync(except, cancellationToken).ConfigureAwait(false)) busy.UnionWith(Holds(job));
return busy;
}
private static IReadOnlyList<Guid> Holds(Job job)
private static IReadOnlyList<Guid> Holds(JobHold job)
{
if (job.Kind != JobKind.Verify) return job.TargetIds;
if (job is not { Kind: JobKind.Verify, ParametersJson: { Length: > 0 } json }) return job.TargetIds;
try
{
return ArchiveJobJson.Read(job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null)).SequenceIds ?? job.TargetIds;
return JsonSerializer.Deserialize(json, ArchiveJobJson.Default.VerifyParameters)?.SequenceIds ?? job.TargetIds;
}
catch (JsonException)
{
+11 -2
View File
@@ -122,6 +122,14 @@ public sealed class JobEngine : IAsyncDisposable
Guid? createdBy, Guid? scheduleId, CancellationToken cancellationToken = default) =>
EnqueueAsync(Guid.CreateVersion7(), kind, targets, targetLabel, parametersJson, priority, createdBy, scheduleId, cancellationToken);
/// <summary>
/// Queues a job under the id <paramref name="id"/>, chosen before (what it reads beside its journal is written under that
/// id first, as a conversion's confirmed photos are), and wakes a worker.
/// </summary>
public Task<Job> EnqueueAsync(Guid id, JobKind kind, IReadOnlyList<Guid> targets, string? targetLabel, string? parametersJson, JobPriority priority,
Guid? createdBy, CancellationToken cancellationToken = default) =>
EnqueueAsync(id, kind, targets, targetLabel, parametersJson, priority, createdBy, null, cancellationToken);
/// <summary>
/// True when this engine runs jobs of <paramref name="kind"/>: it has a handler for it and <see cref="JobEngineOptions.Kinds"/>
/// does not leave it out. Known before <see cref="StartAsync"/>, so a host that does not start the engine still answers.
@@ -377,7 +385,7 @@ public sealed class JobEngine : IAsyncDisposable
{
var candidate = await AcceptedAsync(queue, null, cancellationToken).ConfigureAwait(false);
if (candidate is null) return _others.Count > 0 ? await _db.Jobs.ClaimNextAsync(workerId, _others, queue, cancellationToken).ConfigureAwait(false) : null;
if (_others.Count > 0 && (await _db.Jobs.ListQueuedAsync(_others, 1, queue, cancellationToken).ConfigureAwait(false)).FirstOrDefault() is { } head
if (_others.Count > 0 && (await _db.Jobs.ListQueuedBriefAsync(_others, 1, queue, cancellationToken).ConfigureAwait(false)).FirstOrDefault() is { } head
&& queue.ClaimsBefore(head, candidate))
{
if (await _db.Jobs.ClaimNextAsync(workerId, _others, queue, cancellationToken).ConfigureAwait(false) is { } next) return next;
@@ -397,7 +405,8 @@ public sealed class JobEngine : IAsyncDisposable
Job? head = null;
if (_others.Count > 0)
{
foreach (var queued in await _db.Jobs.ListQueuedAsync(_others, ReviewWindow, queue, cancellationToken).ConfigureAwait(false))
// Only who comes first counts here: the job that yields to it reads its kind, and a claim reads a job whole.
foreach (var queued in await _db.Jobs.ListQueuedBriefAsync(_others, ReviewWindow, queue, cancellationToken).ConfigureAwait(false))
{
if (running.FollowUps.ContainsKey(queued.Id)) continue;
head = queued;
+60 -5
View File
@@ -7,6 +7,12 @@ namespace Chronos.Data.Repositories;
/// <summary>Jobs per state; <see cref="DoneSince"/> and <see cref="FailedSince"/> count those finished since a given time.</summary>
public sealed record JobCounts(int Queued, int Running, int Paused, int DoneSince, int FailedSince);
/// <summary>
/// What a running or paused job works on (<see cref="JobRepository.ListHoldsAsync"/>): its targets and, for a verify, its
/// parameters, which name the sequences it holds; null for every other kind, whose parameters are not read.
/// </summary>
public sealed record JobHold(Guid Id, JobKind Kind, IReadOnlyList<Guid> TargetIds, string? ParametersJson);
/// <summary>
/// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically,
/// write checkpoints while running, and on startup requeue what a crash left running.
@@ -17,6 +23,14 @@ public sealed class JobRepository
"id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " +
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id";
/// <summary>
/// <see cref="Columns"/> as lists read them (<see cref="ListBriefAsync"/>): the parameters left out, null. A conversion's or
/// an import's name every sequence or file they work on; the job reads them, and so do the screens that act on that one job.
/// </summary>
private const string BriefColumns =
"id, kind, target_ids, target_label, CAST(NULL AS TEXT), state, priority, progress_done, progress_total, items_per_second, " +
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id";
/// <summary>The states of a job that has not ended: queued, running, paused.</summary>
private static readonly string PendingStates =
string.Join(", ", new[] { JobState.Queued, JobState.Running, JobState.Paused }.Select(s => ((int)s).ToString(CultureInfo.InvariantCulture)));
@@ -37,6 +51,11 @@ public sealed class JobRepository
public Task<Job?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken);
/// <summary>When the job <paramref name="id"/> was queued; null when there is no such job. Reads that column alone.</summary>
public async Task<DateTimeOffset?> GetCreatedAsync(Guid id, CancellationToken cancellationToken = default) =>
(await _db.ListAsync("SELECT created FROM jobs WHERE id = $1", [id], r => r.GetDateTimeOffset(0), cancellationToken).ConfigureAwait(false))
is [var created] ? created : null;
/// <summary>
/// Marks the next queued job (highest priority, then oldest) of the given kinds as running for
/// <paramref name="workerId"/> and returns it, or null when none waits. Concurrent claimers never get the same
@@ -93,7 +112,17 @@ public sealed class JobRepository
ListAsync(state, kind: null, limit, cursor, cancellationToken);
/// <summary>Jobs, newest first, optionally of one state and one kind.</summary>
public Task<Page<Job>> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default)
public Task<Page<Job>> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
PageAsync(Columns, state, kind, limit, cursor, cancellationToken);
/// <summary>
/// <see cref="ListAsync(JobState?, JobKind?, int, string?, CancellationToken)"/> as lists show jobs: every field but
/// <see cref="Job.ParametersJson"/>, which stays null (<see cref="GetAsync"/> reads a job whole).
/// </summary>
public Task<Page<Job>> ListBriefAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
PageAsync(BriefColumns, state, kind, limit, cursor, cancellationToken);
private Task<Page<Job>> PageAsync(string columns, JobState? state, JobKind? kind, int limit, string? cursor, CancellationToken cancellationToken)
{
var p = new SqlParameters();
var where = new List<string>();
@@ -105,7 +134,7 @@ public sealed class JobRepository
where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add));
}
var take = Cursor.ClampLimit(limit);
var sql = $"SELECT {Columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
var sql = $"SELECT {columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
$"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}";
return _db.RunAsync(async session =>
{
@@ -158,11 +187,22 @@ public sealed class JobRepository
ListQueuedAsync(kinds, limit, JobPriorityQueue.Off, cancellationToken);
/// <summary>Queued jobs of the given kinds in the claim order of <paramref name="queue"/>.</summary>
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default)
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) =>
QueuedAsync(Columns, kinds, limit, queue, cancellationToken);
/// <summary>
/// <see cref="ListQueuedAsync(IReadOnlyCollection{JobKind}, int, JobPriorityQueue, CancellationToken)"/> without the
/// parameters (null), for a look at what would be claimed next: claiming a job reads it whole.
/// </summary>
public Task<IReadOnlyList<Job>> ListQueuedBriefAsync(IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) =>
QueuedAsync(BriefColumns, kinds, limit, queue, cancellationToken);
private Task<IReadOnlyList<Job>> QueuedAsync(string columns, IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(kinds);
ArgumentNullException.ThrowIfNull(queue);
return kinds.Count == 0 ? Task.FromResult<IReadOnlyList<Job>>([]) : _db.ListAsync(
$"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1",
$"SELECT {columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1",
[Cursor.ClampLimit(limit)], Map, cancellationToken);
}
@@ -195,6 +235,18 @@ public sealed class JobRepository
return await _db.ScalarAsync<long>(sql, p.ToArray(), cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>
/// What the running and paused jobs hold, but the job <paramref name="except"/> (the one asking): each one's id, kind and
/// targets, and only a verify's parameters (<see cref="JobHold"/>). Neither the other parameters nor the targets of
/// <paramref name="except"/> are read: a conversion asks at every sequence it comes to, and its own row names them all.
/// </summary>
public Task<IReadOnlyList<JobHold>> ListHoldsAsync(Guid? except = null, CancellationToken cancellationToken = default) =>
_db.ListAsync(
$"SELECT id, kind, target_ids, CASE WHEN kind = {(int)JobKind.Verify} THEN parameters_json END FROM jobs " +
$"WHERE state IN ({(int)JobState.Running}, {(int)JobState.Paused}){(except is null ? string.Empty : " AND id <> $1")} ORDER BY created, id",
except is { } self ? [self] : [],
r => new JobHold(r.GetGuid(0), (JobKind)r.GetInt32(1), Targets(r.GetString(2)), r.GetStringOrNull(3)), cancellationToken);
/// <summary>The targets of the queued, running and paused jobs of <paramref name="kind"/> (the newest 5000 such jobs).</summary>
public Task<HashSet<Guid>> ListPendingTargetsAsync(JobKind kind, CancellationToken cancellationToken = default) =>
PendingTargetsAsync(kind, null, cancellationToken);
@@ -306,11 +358,14 @@ public sealed class JobRepository
private static string KindList(IEnumerable<JobKind> kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture)));
/// <summary>Target ids as the table stores them: comma-joined, empty for none.</summary>
private static Guid[] Targets(string ids) => ids.Length > 0 ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [];
private static Job Map(DbRow r) => new()
{
Id = r.GetGuid(0),
Kind = (JobKind)r.GetInt32(1),
TargetIds = r.GetString(2) is { Length: > 0 } ids ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [],
TargetIds = Targets(r.GetString(2)),
TargetLabel = r.GetStringOrNull(3),
ParametersJson = r.GetStringOrNull(4),
State = (JobState)r.GetInt32(5),
@@ -565,6 +565,14 @@ public sealed class SequenceRepository
_db.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
[archiveId, sessionKey], cancellationToken);
/// <summary>
/// The session keys of an archive that match <paramref name="pattern"/> (a <c>LIKE</c> pattern, <c>\</c> its escape), each
/// with the sequence it resolves to, by key: the new sequences conversions made of one source, say.
/// </summary>
public Task<IReadOnlyList<(string Key, Guid SequenceId)>> ListSessionKeysAsync(Guid archiveId, string pattern, CancellationToken cancellationToken = default) =>
_db.ListAsync("SELECT session_key, sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key LIKE $2 ESCAPE '\\' ORDER BY session_key",
[archiveId, pattern], r => (r.GetString(0), r.GetGuid(1)), cancellationToken);
/// <summary>Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to.</summary>
public Task<Guid> MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
+2 -2
View File
@@ -48,8 +48,8 @@ public static class AdminEndpoints
var users = await db.Users.ListAsync(ct);
var names = users.ToDictionary(u => u.Id, u => u.Username);
var counts = await db.Jobs.CountAsync(today, ct);
var running = await db.Jobs.ListAsync(JobState.Running, null, 10, null, ct);
var queued = await db.Jobs.ListAsync(JobState.Queued, null, 10, null, ct);
var running = await db.Jobs.ListBriefAsync(JobState.Running, null, 10, null, ct);
var queued = await db.Jobs.ListBriefAsync(JobState.Queued, null, 10, null, ct);
var nodes = await db.RenderNodes.ListAsync(ct);
var uploads = await db.Uploads.CommittedPerDayAsync(today.AddDays(-29), ct);
var activity = await db.Audit.ListAsync(null, null, 12, null, ct);
+2 -1
View File
@@ -56,7 +56,8 @@ public static class JobEndpoints
if (!JobWire.TryParseKind(kind, out var k)) return ApiErrors.BadRequest(context, "invalid_filter", "kind");
kindFilter = k;
}
var page = await db.Jobs.ListAsync(stateFilter, kindFilter, limit ?? 50, string.IsNullOrEmpty(cursor) ? null : cursor, context.RequestAborted);
// A job's answer holds no parameters (a conversion's or an import's can name every sequence or file): the list reads none.
var page = await db.Jobs.ListBriefAsync(stateFilter, kindFilter, limit ?? 50, string.IsNullOrEmpty(cursor) ? null : cursor, context.RequestAborted);
return Results.Json(new Page<JobDto>([.. page.Items.Select(Dto.Job)], page.NextCursor, null), ProtocolJson.Default.PageJobDto);
}