diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx
index 39be0d17..bba3d8b3 100644
--- a/i18n/en/Archive.resx
+++ b/i18n/en/Archive.resx
@@ -224,6 +224,7 @@
Left as it was: {0}. {1}long: job log line; {0} sequence title, {1} what failed and why (sentences)
{0}: photos not converted, left in the source alone: {1}long: job log line after a conversion into a new sequence; {0} sequence title, {1} photos
Nothing to convert: {0}long: job log line; {0} sequence title
+ {0}: another conversion made its copy in this format since this one was confirmed; no second copy is made.long: job log line of a conversion into new sequences; {0} sequence title
{0}: photos replaced by their conversion: {1}long: job log line; {0} sequence title, {1} photos
{0}: photos converted into a new sequence: {1}long: job log line; {0} sequence title, {1} photos
Dry run: nothing was changed.long: job log line
diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx
index 91523868..cc064662 100644
--- a/i18n/it/Archive.it.resx
+++ b/i18n/it/Archive.it.resx
@@ -218,6 +218,7 @@
Lasciata com'era: {0}. {1}
{0}: foto non convertite, rimaste solo nella sorgente: {1}
Niente da convertire: {0}
+ {0}: dopo la conferma di questa conversione, un'altra ne ha già creato la copia in questo formato; non ne viene creata una seconda.
{0}: foto sostituite dalla loro conversione: {1}
{0}: foto convertite in una nuova sequenza: {1}
Simulazione: nulla è stato modificato.
diff --git a/src/Chronos.Data/Archiving/ArchiveService.cs b/src/Chronos.Data/Archiving/ArchiveService.cs
index ed69f220..6fe247f5 100644
--- a/src/Chronos.Data/Archiving/ArchiveService.cs
+++ b/src/Chronos.Data/Archiving/ArchiveService.cs
@@ -697,29 +697,35 @@ public static class ArchiveJobs
///
/// : applies the conversion ()
- /// as confirmed (owner review 2, R7): the sequences it converts, in its order with their frames versions, the photos each
- /// converts and their digest (), are the job's targets, so jobs that would touch
- /// them wait. The job converts those photos alone, and leaves as it is and reports a sequence where one of them no longer
- /// holds the content confirmed when the job comes to it. Queue it through , which
- /// refuses a second conversion of the same sequences.
+ /// as confirmed (owner review 2, R7): the sequences it converts, in its order with their frames versions and the digest
+ /// of the photos each converts (), are the job's targets, so jobs that would touch
+ /// them wait. The request names the job's id, under which (the way to queue it,
+ /// which refuses a second conversion of the same sequences) writes which photos those are beside the job's journal
+ /// (). The job converts those photos alone, and leaves as it is and reports a sequence where
+ /// one of them no longer holds the content confirmed when the job comes to it.
///
/// The plan converts nothing.
public static ArchiveJobRequest Convert(Archive archive, ConvertPlan plan)
{
ArgumentNullException.ThrowIfNull(archive);
ArgumentNullException.ThrowIfNull(plan);
- var sequences = plan.Converted
- .Select(s => new ConvertSequence(s.SequenceId, s.FramesVersion, s.Digest(), ConvertSequence.EncodePhotos(s.Frames.Select(f => f.FrameId))))
- .ToList();
+ var sequences = plan.Converted.Select(s => new ConvertSequence(s.SequenceId, s.FramesVersion, s.Digest())).ToList();
if (sequences.Count == 0) throw new ArgumentException("The plan converts nothing.", nameof(plan));
var label = sequences.Count == 1 ? plan.Converted.First().Title : archive.Name;
return new(JobKind.Convert, [.. sequences.Select(s => s.Id)], label,
Json(new ConvertJobParameters(archive.Id, plan.Options.Target, plan.Options.Quality, plan.Mode, sequences, plan.Fingerprint()),
- ArchiveJobJson.Default.ConvertJobParameters), JobPriority.User);
+ ArchiveJobJson.Default.ConvertJobParameters), JobPriority.User, Guid.CreateVersion7());
}
- public static Task EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) =>
- engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
+ /// Queues , under its own id when it names one ().
+ public static Task EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(engine);
+ ArgumentNullException.ThrowIfNull(request);
+ return request.Id is { } id
+ ? engine.EnqueueAsync(id, request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken)
+ : engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
+ }
private static string Json(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo typeInfo) => JsonSerializer.Serialize(value, typeInfo);
}
diff --git a/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs b/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs
index cd34d322..8a467a4a 100644
--- a/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs
+++ b/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs
@@ -56,55 +56,29 @@ public sealed record MoveSequence(Guid Id);
/// (owner review 2, R7): targets = the sequences converted, so the jobs that would touch them
/// wait; the archive, the target format and JPEG quality, the mode chosen for this run, the sequences of the confirmed plan
/// in its order, each with what it converts, and the plan's fingerprint (). A sequence
-/// whose photos to convert are no longer those confirmed is left as it is and reported.
+/// whose photos to convert are no longer those confirmed is left as it is and reported. Which photos those are is not
+/// here but in the job's own file beside its journal (), so that the row stays as small as the
+/// number of sequences, whatever the number of photos, for everything that reads jobs.
///
public sealed record ConvertJobParameters(Guid ArchiveId, ConvertTarget Target, int Quality, ConvertMode Mode, IReadOnlyList Sequences,
string Fingerprint);
///
-/// A sequence of a : the frames version it was planned at, the digest of the photos it
-/// converts, each with its content (), and which photos those are ().
-/// Under the sequence's lease, before it converts anything, the job takes those photos from its plan made again and goes
-/// on only while they have the digest confirmed: photos that came in since (an upload while the job waited its turn) stay
-/// as they are, and a photo the plan did not list (one another program held while it was made) is never converted. Null
-/// in a job an earlier build queued, which checks the frames version only (no digest), or the whole plan's digest (no photos).
+/// A sequence of a : the frames version it was planned at, and the digest of the photos it
+/// converts, each with its content (). Under the sequence's lease, before it
+/// converts anything, the job takes the photos confirmed () from its plan made again and goes
+/// on only while they have that digest: photos that came in since (an upload while the job waited its turn) stay as they
+/// are, and a photo the plan did not list (one another program held while it was made) is never converted. Without the
+/// confirmed photos, the whole plan made again must have the digest. Null in a job an earlier build queued, which checks
+/// the frames version only.
///
-/// The frame ids of the photos converted, in the plan's order, as writes them.
-public sealed record ConvertSequence(Guid Id, int FramesVersion, string? Frames = null, string? Photos = null)
-{
- ///
- /// Frame ids as a job's parameters hold them: their 16 bytes each, one after the other, in base64 (a whole-archive
- /// conversion lists every photo it converts, so the job row stays about half the size of a list of ids).
- ///
- public static string EncodePhotos(IEnumerable ids)
- {
- ArgumentNullException.ThrowIfNull(ids);
- IReadOnlyCollection frames = ids as IReadOnlyCollection ?? [.. ids];
- var bytes = new byte[frames.Count * 16];
- var at = 0;
- foreach (var frame in frames)
- {
- frame.TryWriteBytes(bytes.AsSpan(at, 16));
- at += 16;
- }
- return Convert.ToBase64String(bytes);
- }
+public sealed record ConvertSequence(Guid Id, int FramesVersion, string? Frames = null);
- /// The frame ids holds, in order; null when it holds none (a job an earlier build queued).
- /// The text is not ids as writes them.
- public IReadOnlyList? PhotoIds()
- {
- if (Photos is null) return null;
- var bytes = Convert.FromBase64String(Photos);
- if (bytes.Length % 16 != 0) throw new FormatException($"The photos of sequence {Id} are not frame ids.");
- var ids = new Guid[bytes.Length / 16];
- for (var i = 0; i < ids.Length; i++) ids[i] = new Guid(bytes.AsSpan(i * 16, 16));
- return ids;
- }
-}
-
-/// A job ready to enqueue.
-public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets, string? Label, string? ParametersJson, JobPriority Priority);
+///
+/// A job ready to enqueue; , when set, is the id it is queued under (chosen first, for what the job
+/// reads beside its journal: ), otherwise the engine gives it one.
+///
+public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets, string? Label, string? ParametersJson, JobPriority Priority, Guid? Id = null);
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)]
[JsonSerializable(typeof(ScanParameters))]
@@ -120,6 +94,7 @@ public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets
[JsonSerializable(typeof(ConvertJournal))]
[JsonSerializable(typeof(StagedLine))]
[JsonSerializable(typeof(PinnedSequence))]
+[JsonSerializable(typeof(ConfirmedLine))]
[JsonSerializable(typeof(CardProfile))]
internal sealed partial class ArchiveJobJson : JsonSerializerContext
{
diff --git a/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs b/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs
index 6dc02824..c7026207 100644
--- a/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs
+++ b/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs
@@ -17,7 +17,9 @@ namespace Chronos.Data.Archiving.Jobs;
/// one when the source had it, thumbnails and proxies) are queued as follow-ups, as soon as it is done. Sequences left as
/// they were (changed after the plan, busy, a photo another program holds, a photo that failed) are logged and fail the
/// job at the end, after the others are converted; a retry tries again those busy, held or failed, with what they staged,
-/// and leaves one that changed after the plan as it is (its log line says to plan again).
+/// and leaves one that changed after the plan as it is (its log line says to plan again). A retry of a New sequence
+/// conversion finishes the copies it made and makes none of a source another conversion copied in the same format since
+/// this job was queued (its creation time: when the plan was confirmed).
///
internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter converter, Func? sequenceChanged) : IJobHandler
{
@@ -39,7 +41,8 @@ internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter conver
var archive = await db.Archives.GetAsync(parameters.ArchiveId, context.Cancellation).ConfigureAwait(false)
?? throw new InvalidOperationException($"Convert job {context.Job.Id} does not name a known archive.");
var name = context.Job.CreatedBy is { } user ? (await db.Users.GetAsync(user, context.Cancellation).ConfigureAwait(false))?.Username : null;
- var result = await converter.RunAsync(archive, parameters, context.Job.Id, new ConvertRequester(name, context.Job.CreatedBy), new ConvertCallbacks(
+ var by = new ConvertRequester(name, context.Job.CreatedBy, context.Job.Created);
+ var result = await converter.RunAsync(archive, parameters, context.Job.Id, by, new ConvertCallbacks(
(done, total) => context.ReportAsync(done, total),
(done, total) => context.CheckpointAsync(Checkpoint, done, total),
context.Log, TellAsync, requests => QueueAsync(context, requests)), context.Cancellation).ConfigureAwait(false);
diff --git a/src/Chronos.Data/Archiving/MassConverter.cs b/src/Chronos.Data/Archiving/MassConverter.cs
index 3a7f1ce1..1c1bb014 100644
--- a/src/Chronos.Data/Archiving/MassConverter.cs
+++ b/src/Chronos.Data/Archiving/MassConverter.cs
@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
@@ -29,6 +30,12 @@ public enum ConvertOutcome
/// It had nothing to convert.
Nothing,
+ ///
+ /// Done with, no copy made (): another conversion made its copy in this format since
+ /// this one was confirmed (a retry of an older conversion, or a duplicate queued beside it), so a second copy is not made.
+ ///
+ Superseded,
+
///
/// Left as it was: a photo it converts no longer holds the content confirmed (it changed, is gone, or no longer converts),
/// or it is no longer in the archive. A retry finds the same: the user plans again.
@@ -52,8 +59,12 @@ public enum ConvertOutcome
public sealed record ConvertSequenceResult(Guid SequenceId, string Title, ConvertOutcome Outcome, int Frames, Guid? NewSequenceId = null, string? Error = null,
int LeftOut = 0)
{
- /// Whether the sequence is done with: converted, made, or with nothing to convert.
- public bool Final => Outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing;
+ /// Whether the sequence is done with: converted, made, with nothing to convert, or copied by another conversion since.
+ public bool Final => IsFinal(Outcome);
+
+ /// Whether leaves a sequence done with ().
+ internal static bool IsFinal(ConvertOutcome? outcome) =>
+ outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing or ConvertOutcome.Superseded;
}
/// What a conversion did, and the passes the sequences it changed or made need (queued as follow-ups as each one ended).
@@ -75,8 +86,12 @@ public sealed record ConvertResult(IReadOnlyList Sequence
internal sealed record ConvertCallbacks(Func Progress, Func Checkpoint, Action Log,
Func? Done = null, Func, ValueTask>? Queue = null);
-/// Who asked for a conversion: the user's name, kept in the trash records of a Replace, and id, the owner of a new sequence when its source has none.
-internal sealed record ConvertRequester(string? Name, Guid? Id);
+///
+/// Who asked for a conversion: the user's name, kept in the trash records of a Replace, and id, the owner of a new sequence
+/// when its source has none; and when the plan was confirmed (the job queued): a copy another conversion made of a source
+/// since then supersedes this one's (). Null: no copy made by another counts.
+///
+internal sealed record ConvertRequester(string? Name, Guid? Id, DateTimeOffset? Confirmed = null);
///
/// A conversion of sequences a confirmed plan converts is queued, running or paused already, so its job was not queued
@@ -94,7 +109,7 @@ public sealed class ConversionQueuedException(IReadOnlyList sequenceIds) :
/// the conversion writes, ), and a job applies the
/// confirmed plan (queued by , which refuses a second conversion of the same sequences) one sequence
/// at a time. Under the sequence's lease, its sidecar settled where the index has it then (),
-/// the job plans it again and takes the photos confirmed (), going on only while each is
+/// the job plans it again and takes the photos confirmed (), going on only while each is
/// still planned with its content confirmed (); it pins them then (),
/// so a photo that came in, was verified or could be opened again since the plan stays as it is, on a retry too. Stage A,
/// without the lease: each photo is hashed against its record and converted (, several at once
@@ -107,14 +122,19 @@ public sealed class ConversionQueuedException(IReadOnlyList sequenceIds) :
/// files the converted copies into a new sequence through the archive ingest (session
/// key convert:<job>:<source>, so a resume finds it again), titled "title (FORMAT)", owned as the source is,
/// with the source's tags, rating, notes and project, the source left as it is (the photos the plan did not convert stay
-/// there alone, as the job log says). A sequence whose photos changed, that is busy, or has a photo that fails is left as it
-/// was and reported.
+/// there alone, as the job log says). A retry or a resume finishes the copy it made already rather than make another, and
+/// makes none of a source that another conversion copied in the same format since this one was confirmed
+/// (). A sequence whose photos changed, that is busy, or has a photo that fails is
+/// left as it was and reported.
///
public sealed class MassConverter
{
/// What the frames of a sequence no longer have once their files were replaced ().
private const SequenceStatus Uncovered = SequenceStatus.AnalyzedLight | SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
+ /// How the session key of a new sequence a conversion makes begins ().
+ private const string KeyPrefix = "convert:";
+
/// Held while a confirmed conversion is checked against those pending and queued (): one at a time in this process.
private static readonly SemaphoreSlim Queueing = new(1, 1);
@@ -186,8 +206,8 @@ public sealed class MassConverter
var busy = new HashSet(await _trash.BusyAsync(job, cancellationToken).ConfigureAwait(false));
if (job is not { } self)
busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken).ConfigureAwait(false));
- else if (await _db.Jobs.GetAsync(self, cancellationToken).ConfigureAwait(false) is { } running)
- busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, running.Created, cancellationToken).ConfigureAwait(false));
+ else if (await _db.Jobs.GetCreatedAsync(self, cancellationToken).ConfigureAwait(false) is { } queued)
+ busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, queued, cancellationToken).ConfigureAwait(false));
return busy;
}
@@ -195,8 +215,11 @@ public sealed class MassConverter
/// Queues the job of the confirmed () through ,
/// unless a conversion of a sequence it converts is queued, running or paused already: another dialog or tab, an answer
/// lost and Apply pressed again, a dry run made before that one was queued or none at all. Checked and queued under one
- /// lock of this process, so two confirmations at once queue one job. runs in the caller's
- /// context (a desktop's tells the user on its UI thread).
+ /// lock of this process, so two confirmations at once queue one job. First, which photos it converts are written under
+ /// the job's id beside the journal it starts (); queues the
+ /// request under that id (), and runs in the caller's context (a desktop's tells the
+ /// user on its UI thread). When they cannot be written (the archive went away since its dry run), the job is queued all
+ /// the same and holds each sequence to its whole plan.
///
/// A conversion of one of its sequences waits or runs; nothing was queued.
/// The plan converts nothing.
@@ -204,13 +227,25 @@ public sealed class MassConverter
{
ArgumentNullException.ThrowIfNull(enqueue);
var request = ArchiveJobs.Convert(archive, plan);
+ var paths = new ArchivePaths(archive.RootPath);
+ var job = request.Id!.Value;
// No ConfigureAwait(false) here: each wait comes back to the caller's context, where enqueue is called.
await Queueing.WaitAsync(cancellationToken);
try
{
var pending = await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken);
if (request.Targets.Where(pending.Contains).ToList() is { Count: > 0 } queued) throw new ConversionQueuedException(queued);
- await enqueue(request);
+ var written = await Task.Run(() => ConfirmedPhotos.TryWrite(paths, job, plan), cancellationToken);
+ try
+ {
+ await enqueue(request);
+ }
+ catch
+ {
+ // Not queued: no job will read them.
+ if (written) ConfirmedPhotos.Delete(paths, job);
+ throw;
+ }
}
finally
{
@@ -260,6 +295,7 @@ public sealed class MassConverter
var steps = parameters.Sequences.DistinctBy(s => s.Id).ToList();
var staged = journal.Staged(paths);
var pins = journal.Pins(paths);
+ var confirmed = ConfirmedPhotos.Read(paths, jobId);
// What is still to do: every sequence not final in an earlier attempt that is still in the archive, its sidecar settled
// first when a writer a crash interrupted left a journal in its folder (this job's replacement among them).
var rows = new Dictionary();
@@ -277,11 +313,15 @@ public sealed class MassConverter
var unstaged = new Dictionary();
foreach (var p in plan.Sequences.Where(p => p.Blocked.Count == 0))
{
- if (Confirmed(byId[p.SequenceId], p, pins.Of(p.SequenceId)) is not { Frames.Count: > 0 } converts) continue;
+ if (Confirmed(byId[p.SequenceId], confirmed.Of(p.SequenceId), p, pins.Of(p.SequenceId)) is not { Frames.Count: > 0 } converts) continue;
progress.Count(converts.SequenceId, converts.Frames.Count, staged.Of(converts.SequenceId, converts.FramesVersion).Count);
- unstaged[converts.SequenceId] = Unstaged(paths, jobId, converts, staged);
+ // What a new sequence this job made holds already takes no space again.
+ var filed = parameters.Mode == ConvertMode.NewSequence
+ ? await FiledAsync(archive, SessionKey(jobId, converts.SequenceId), cancellationToken).ConfigureAwait(false)
+ : null;
+ unstaged[converts.SequenceId] = Unstaged(paths, jobId, converts, staged, filed);
}
- var space = new SpaceWatch(this, paths);
+ var space = new SpaceWatch(() => FreeSpace(paths));
EnsureSpace(unstaged.Values.Sum(), space);
await callbacks.Progress(progress.Done, progress.Total).ConfigureAwait(false);
@@ -309,8 +349,8 @@ public sealed class MassConverter
// The space for what is left, measured again: the volume may have filled since, and the files staged so far tell
// how far off the estimate is.
EnsureSpace(unstaged.Values.Sum(), space);
- done = await ConvertOneAsync(archive, paths, step, row, options, parameters.Mode, jobId, by, journal, pins, staged, progress, space, callbacks,
- cancellationToken).ConfigureAwait(false);
+ done = await ConvertOneAsync(archive, paths, step, row, options, parameters.Mode, jobId, by, journal, confirmed, pins, staged, progress, space,
+ callbacks, cancellationToken).ConfigureAwait(false);
}
unstaged.Remove(step.Id);
var result = done.Result;
@@ -349,16 +389,37 @@ public sealed class MassConverter
///
/// One sequence: under its lease, its sidecar settled, planned again and held to what was confirmed (the photos pinned the
- /// first time); then stage A without the lease, and stage B.
+ /// first time); then stage A without the lease, and stage B. A New sequence that has its copy already is not converted
+ /// again ().
///
private async Task ConvertOneAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options,
- ConvertMode mode, Guid jobId, ConvertRequester by, ConvertJournal journal, PinLog pins, StagedLog staged, Progress progress, SpaceWatch space,
- ConvertCallbacks callbacks, CancellationToken cancellationToken)
+ ConvertMode mode, Guid jobId, ConvertRequester by, ConvertJournal journal, ConfirmedPhotos confirmedPhotos, PinLog pins, StagedLog staged,
+ Progress progress, SpaceWatch space, ConvertCallbacks callbacks, CancellationToken cancellationToken)
{
// Another job holds it now (a Reorganize paused since the job started, a render, a conversion of it queued first): it
// waits for the retry.
var busy = await HeldAsync(jobId, cancellationToken).ConfigureAwait(false);
if (busy.Contains(step.Id) || busy.Contains(archive.Id)) return new StepDone(Result(row, ConvertOutcome.Blocked));
+ var key = SessionKey(jobId, row.Id);
+ if (mode == ConvertMode.NewSequence)
+ {
+ try
+ {
+ if (await CopiedAsync(archive, paths, step, row, options, jobId, by, journal, pins, staged, key, progress, cancellationToken).ConfigureAwait(false)
+ is { } copied)
+ return copied;
+ }
+ catch (ArchiveOfflineException)
+ {
+ throw;
+ }
+ catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
+ {
+ Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the copy this conversion made of it could not be finished yet", ex);
+ return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex)));
+ }
+ }
+ var photos = confirmedPhotos.Of(step.Id);
SequencePaths folder;
SequenceManifest manifest;
ConvertSequencePlan planned;
@@ -379,7 +440,7 @@ public sealed class MassConverter
manifest = SequenceWriter.ReadLocked(folder, row.Id);
FrameTrash.EnsureReplaced(folder);
var pin = pins.Of(step.Id);
- if (mode == ConvertMode.Replace && ReplacedEarlier(manifest, step, pin, staged.Of(step.Id, step.FramesVersion), options) is { } frames)
+ if (mode == ConvertMode.Replace && ReplacedEarlier(manifest, step, photos, pin, staged.Of(step.Id, step.FramesVersion), options) is { } frames)
{
// Replaced by this job already: a crash after its stage B, before the journal said so (the settle may just have
// completed it).
@@ -394,11 +455,11 @@ public sealed class MassConverter
IReadOnlySet files = new HashSet(FileRenames.ListPresent(at, sidecar), StringComparer.OrdinalIgnoreCase);
return (ConvertPlanner.Plan(sidecar, at, files, options, decoder, title), files);
}, cancellationToken).ConfigureAwait(false);
- if (Confirmed(step, fresh, pin) is not { } confirmed)
+ if (Confirmed(step, photos, fresh, pin) is not { } confirmed)
{
// A photo it converts that another program holds now cannot be told from one that changed: the retry, once it
// is free, can.
- if (await Task.Run(() => Held(step, pin, sidecar, at, present, fresh, options, decoder), cancellationToken).ConfigureAwait(false))
+ if (await Task.Run(() => Held(photos, pin, sidecar, at, present, fresh, options, decoder), cancellationToken).ConfigureAwait(false))
{
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: a photo it converts cannot be opened now; it is not converted");
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.PhotoHeld")));
@@ -426,7 +487,6 @@ public sealed class MassConverter
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex)));
}
var staging = StagingOf(paths, jobId, row.Id);
- var key = SessionKey(jobId, row.Id);
// A new sequence a resumed job finds again holds the photos it filed before it stopped.
var filed = mode == ConvertMode.NewSequence
? await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false)
@@ -443,7 +503,7 @@ public sealed class MassConverter
{
return mode == ConvertMode.Replace
? await ReplaceAsync(archive, paths, row, planned, ready, staging, options, jobId, by.Name, cancellationToken).ConfigureAwait(false)
- : await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, cancellationToken).ConfigureAwait(false);
+ : await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, journal, cancellationToken).ConfigureAwait(false);
}
catch (LeaseUnavailableException ex)
{
@@ -479,15 +539,97 @@ public sealed class MassConverter
};
///
- /// What the job converts of a sequence whose plan is now : the photos it pinned when it first came to
- /// it, each still planned with the content it had then; the first time, the photos confirmed ()
- /// while each is still planned with the content confirmed (the step's digest). Photos that came in, were verified or could
- /// be opened again since stay as they are, so an upload while the job waits its turn neither changes what it converts
- /// nor holds it back. A job an earlier build queued: the whole plan while it has the step's digest, or its frames
- /// version. Null when a photo confirmed no longer converts as it was: the sequence changed after the plan, and the user
- /// plans again.
+ /// What a conversion does with a source that has its copy already when the job comes
+ /// to it (a retry, a resume): the copy this job made (its journal names it, or its session key when a crash came before
+ /// the journal did) is finished, whatever changed in the source since, once it holds every photo the job pinned, and one
+ /// gone since is not made again without a new plan; a copy in this format that another conversion made since this one was
+ /// confirmed () supersedes this one's. Null when neither: the sequence converts,
+ /// and a copy holding only some of its photos (a crash while they were filed) gets the rest.
///
- internal static ConvertSequencePlan? Confirmed(ConvertSequence step, ConvertSequencePlan fresh, PinnedSequence? pin)
+ private async Task CopiedAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options, Guid jobId,
+ ConvertRequester by, ConvertJournal journal, PinLog pins, StagedLog staged, string key, Progress progress, CancellationToken cancellationToken)
+ {
+ if ((journal.Of(step.Id)?.NewSequenceId ?? await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, cancellationToken).ConfigureAwait(false)) is { } made)
+ {
+ if (await _db.Sequences.GetAsync(made, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id)
+ {
+ Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it is gone; no other is made without a new plan");
+ return new StepDone(Result(row, ConvertOutcome.Changed));
+ }
+ return pins.Of(step.Id) is { } pin
+ ? await FinishCopyAsync(archive, paths, row, pin, staged, key, made, progress, cancellationToken).ConfigureAwait(false)
+ : null;
+ }
+ if (by.Confirmed is not { } confirmed
+ || await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } other)
+ return null;
+ Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed; no second copy is made");
+ progress.Count(row.Id, 0, 0);
+ return new StepDone(Result(row, ConvertOutcome.Superseded));
+ }
+
+ ///
+ /// The copy this job made of a source, finished when it holds the conversion of every photo pinned (a crash came after
+ /// they were filed, before the journal said so): it takes the source's tags, rating, notes and project as stage B gives
+ /// them, whatever became of the photos of the source since. Null while a photo pinned is not in it, or when the source
+ /// is no longer where this index has it: the job converts the sequence, which files the rest.
+ ///
+ private async Task FinishCopyAsync(Archive archive, ArchivePaths paths, Sequence row, PinnedSequence pin, StagedLog staged, string key, Guid made,
+ Progress progress, CancellationToken cancellationToken)
+ {
+ var conversions = staged.Of(row.Id, pin.FramesVersion).ToDictionary(c => c.FrameId);
+ var filed = await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false);
+ var ready = new Dictionary();
+ foreach (var photo in pin.Frames)
+ {
+ if (!conversions.TryGetValue(photo.Id, out var conversion) || !string.Equals(conversion.Source, photo.Sha256, StringComparison.OrdinalIgnoreCase)
+ || !filed.Contains(conversion.Sha256))
+ return null;
+ ready[photo.Id] = conversion;
+ }
+ SequenceManifest manifest;
+ string? project;
+ await using (await _writer.LockAsync(paths, row.Id, cancellationToken).ConfigureAwait(false))
+ {
+ if (await _db.Sequences.GetAsync(row.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id) return null;
+ row = now;
+ var folder = paths.Sequence(row.Folder);
+ if (!SequenceFolders.Holds(folder, row.Id)) return null;
+ manifest = SequenceWriter.ReadLocked(folder, row.Id);
+ project = row.SettingsJson ?? (File.Exists(folder.ProjectPath) ? AtomicFile.ReadAllText(folder.ProjectPath) : null);
+ }
+ Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it holds every photo already; it is finished, no other is made");
+ progress.Count(row.Id, pin.Frames.Count, pin.Frames.Count);
+ return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, made, pin.Frames.Count, cancellationToken).ConfigureAwait(false);
+ }
+
+ ///
+ /// A copy of in this conversion's format that another conversion made since :
+ /// a sequence filed under another job's session key for that source () whose sidecar says it was
+ /// made since then and holds photos of that format. Null when there is none.
+ ///
+ private async Task LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed,
+ CancellationToken cancellationToken)
+ {
+ foreach (var (key, id) in await _db.Sequences.ListSessionKeysAsync(archive.Id, $"{KeyPrefix}%:{source:N}", cancellationToken).ConfigureAwait(false))
+ {
+ if (JobOfKey(key) is not { } other || other == jobId) continue;
+ if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id) continue;
+ if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format)) return id;
+ }
+ return null;
+ }
+
+ ///
+ /// What the job converts of a sequence whose plan is now : the photos it pinned when it first came to
+ /// it, each still planned with the content it had then; the first time, the photos confirmed (,
+ /// ) while each is still planned with the content confirmed (the step's digest). Photos that
+ /// came in, were verified or could be opened again since stay as they are, so an upload while the job waits its turn
+ /// neither changes what it converts nor holds it back. Without the photos confirmed (they could not be written, or a job
+ /// an earlier build queued): the whole plan while it has the step's digest, or its frames version. Null when a photo
+ /// confirmed no longer converts as it was: the sequence changed after the plan, and the user plans again.
+ ///
+ internal static ConvertSequencePlan? Confirmed(ConvertSequence step, IReadOnlyList? photos, ConvertSequencePlan fresh, PinnedSequence? pin)
{
var now = fresh.Frames.ToDictionary(f => f.FrameId);
if (pin is not null)
@@ -500,7 +642,7 @@ public sealed class MassConverter
}
return frames.Count > 0 ? fresh with { FramesVersion = pin.FramesVersion, Frames = frames } : null;
}
- if (step is { Frames: { } confirmedDigest } && step.PhotoIds() is { Count: > 0 } photos)
+ if (step is { Frames: { } confirmedDigest } && photos is { Count: > 0 })
{
var frames = new List(photos.Count);
foreach (var id in photos)
@@ -520,15 +662,15 @@ public sealed class MassConverter
///
/// Whether the plan made now () leaves out a photo the job converts of the sequence (one it pinned,
- /// else one confirmed) only because its file could not be opened (: another program
- /// holds it): the sequence then waits for the retry rather than counting as changed. Photos the job does not convert do
- /// not count, nor does a confirmed photo that changed meanwhile. A job an earlier build queued, which names no photos:
- /// any photo of the sequence that could not be opened.
+ /// else one of , those confirmed) only because its file could not be opened
+ /// (: another program holds it): the sequence then waits for the retry rather than
+ /// counting as changed. Photos the job does not convert do not count, nor does a confirmed photo that changed meanwhile.
+ /// Without the photos confirmed: any photo of the sequence that could not be opened.
///
- private static bool Held(ConvertSequence step, PinnedSequence? pin, SequenceManifest manifest, SequencePaths folder, IReadOnlySet present,
+ private static bool Held(IReadOnlyList? photos, PinnedSequence? pin, SequenceManifest manifest, SequencePaths folder, IReadOnlySet present,
ConvertSequencePlan fresh, ConvertOptions options, ImageDecoder decoder)
{
- IReadOnlyList? converts = pin is not null ? [.. pin.Frames.Select(f => f.Id)] : step.PhotoIds();
+ IReadOnlyList? converts = pin is not null ? [.. pin.Frames.Select(f => f.Id)] : photos;
if (converts is null) return fresh.Skipped.Any(s => s.Reason == ConvertSkip.Inaccessible);
var planned = fresh.Frames.Select(f => f.FrameId).ToHashSet();
var frames = manifest.Frames.ToDictionary(f => f.Id);
@@ -579,12 +721,13 @@ public sealed class MassConverter
///
/// Stage A: converts the planned frames not staged yet, several at once, each staged file appended to the journal as it
- /// is consumed (in frame order). A photo starts only while the volume has the space it and the photos after it that may
- /// convert at once with it take (as many as the host converts at once, never photos that are not there to start), with
- /// the reserve: short of it the job stops (Archive.Error.ConvertSpace) before the volume fills. The space the
- /// photos converting already take is in the volume's free space as they write it, so it is not counted twice. Returns
- /// the first failure (the file and why, localised; the exception goes to the diagnostics log); the frames converted
- /// meanwhile stay staged for a retry.
+ /// is consumed (in frame order). A photo starts only while the volume has the space for it and for every photo handed to
+ /// the work before it whose file has not landed yet (: converting or waiting for its turn,
+ /// up to twice as many as convert at once, at the end of a sequence as anywhere), with the margin and the reserve: short
+ /// of it the job stops (Archive.Error.ConvertSpace) before the volume fills. A converted file is encoded in memory
+ /// and lands whole, by a rename, before it is verified: what those photos will write is not in the free space measured
+ /// yet, and what landed is, so each counts until its file lands, never twice. Returns the first failure (the file and why,
+ /// localised; the exception goes to the diagnostics log); the frames converted meanwhile stay staged for a retry.
///
/// The volume lacks the space for the next photo; what was staged stays.
private async Task StageAsync(Sequence row, SequenceManifest manifest, SequencePaths folder, ConvertSequencePlan planned,
@@ -622,13 +765,26 @@ public sealed class MassConverter
}, cancellationToken);
for (var i = 0; i < todo.Count && failure is null; i++)
{
- if (space.Short(Ahead(todo, i, atOnce)) is { } message)
+ var frame = frames[todo[i].FrameId];
+ var target = Path.Combine(staging, frame.Id.ToString("N") + options.Extension);
+ if (space.Start(todo[i].EstimatedBytes, target) is { } message)
{
full = message;
break;
}
- var frame = frames[todo[i].FrameId];
- await work.AddAsync(token => Stage(frame, manifest, folder, staging, options, decoder, threads, token)).ConfigureAwait(false);
+ // A photo that never starts (the job stops: cancelled, or a write of the journal failed) is never counted off,
+ // and the watch goes with the job.
+ await work.AddAsync(token =>
+ {
+ try
+ {
+ return Stage(frame, manifest, folder, target, options, decoder, threads, token);
+ }
+ finally
+ {
+ space.Ended(target);
+ }
+ }).ConfigureAwait(false);
}
await work.CompleteAsync().ConfigureAwait(false);
if (full is not null)
@@ -640,20 +796,8 @@ public sealed class MassConverter
return failure;
}
- ///
- /// The bytes photo of and the photos after it that may convert at once with
- /// it are estimated at: of them, fewer near the end (the space a photo of stage A starts with).
- ///
- internal static long Ahead(IReadOnlyList todo, int index, int atOnce)
- {
- ArgumentNullException.ThrowIfNull(todo);
- long bytes = 0;
- for (var i = index; i < Math.Min(todo.Count, index + Math.Max(1, atOnce)); i++) bytes += todo[i].EstimatedBytes;
- return bytes;
- }
-
- /// One photo of stage A: hashed against its record, converted into the staging folder and verified.
- private static Staged Stage(SidecarFrame frame, SequenceManifest manifest, SequencePaths folder, string staging, ConvertOptions options, ImageDecoder decoder,
+ /// One photo of stage A: hashed against its record, converted into (in the staging folder) and verified.
+ private static Staged Stage(SidecarFrame frame, SequenceManifest manifest, SequencePaths folder, string target, ConvertOptions options, ImageDecoder decoder,
int threads, CancellationToken cancellationToken)
{
var source = folder.File(frame.File);
@@ -666,7 +810,6 @@ public sealed class MassConverter
if (!string.Equals(sha, frame.Sha256, StringComparison.OrdinalIgnoreCase))
throw new ConversionException(ConversionFailure.Changed, $"{frame.File} no longer matches its SHA-256; verify the sequence");
cancellationToken.ThrowIfCancellationRequested();
- var target = Path.Combine(staging, frame.Id.ToString("N") + options.Extension);
var file = FrameConverter.Convert(new ConvertSource(source, frame, manifest.Camera, manifest.Lens), options, decoder, target, threads, cancellationToken);
return new Staged(frame.File, new StagedConversion(frame.Id, sha, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height), null);
}
@@ -734,9 +877,12 @@ public sealed class MassConverter
/// source's capture facts; then the source's tags, rating, notes and project (its positions translated to the new frames).
/// Every conversion not filed yet must still be staged whole: one gone since (a clean-up, a quarantine) fails the sequence
/// rather than leave its photo out. The photos the plan did not convert are not in the new sequence ().
+ /// Once its photos are filed, the journal names the new sequence (), so that a retry or a
+ /// resume finishes it and never makes a second one.
///
private async Task CreateAsync(Archive archive, ArchivePaths paths, Sequence row, ConvertSequencePlan planned,
- Dictionary ready, string staging, ConvertOptions options, string key, Guid? requestedBy, CancellationToken cancellationToken)
+ Dictionary ready, string staging, ConvertOptions options, string key, Guid? requestedBy, ConvertJournal journal,
+ CancellationToken cancellationToken)
{
SequenceManifest manifest;
string? project;
@@ -775,13 +921,26 @@ public sealed class MassConverter
await _ingest.CommitSessionAsync(archive, new SequenceTarget(key, title, null, owner), pending, CancellationToken.None).ConfigureAwait(false);
var id = await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, CancellationToken.None).ConfigureAwait(false)
?? throw new IOException($"No converted photo of {row.DisplayTitle} could be filed into a new sequence.");
+ journal.Made(row.Id, id);
+ journal.Write(paths);
+ return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, id, planned.Frames.Count, cancellationToken).ConfigureAwait(false);
+ }
+
+ ///
+ /// The end of stage B of , once the new sequence holds the
+ /// photos converted: the source's tags, rating, notes and project given to it (),
+ /// and the passes it needs, from what the source has. The photos of the source it does not hold are counted as left out.
+ ///
+ private async Task CompleteCopyAsync(Archive archive, ArchivePaths paths, Sequence row, SequenceManifest manifest, string? project,
+ Dictionary ready, Guid id, int frames, CancellationToken cancellationToken)
+ {
await CopyAsync(paths, row, manifest, project, ready, id, cancellationToken).ConfigureAwait(false);
var made = await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false)
?? throw new InvalidOperationException($"The new sequence {id} is not indexed.");
- var leftOut = manifest.Frames.Count - planned.Frames.Count;
- Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: {planned.Frames.Count} photo(s) converted into the new sequence {made.DisplayTitle}, {leftOut} left in the source alone");
+ var leftOut = manifest.Frames.Count(f => !ready.ContainsKey(f.Id));
+ Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: {frames} photo(s) converted into the new sequence {made.DisplayTitle}, {leftOut} left in the source alone");
var had = row.Status & (SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt);
- return new StepDone(new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Created, planned.Frames.Count, id, LeftOut: leftOut), had,
+ return new StepDone(new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Created, frames, id, LeftOut: leftOut), had,
_frames.FollowUps(archive, made, had));
}
@@ -855,13 +1014,13 @@ public sealed class MassConverter
///
/// The photos converted, when this job replaced the sequence's frames already (a crash after its stage B, before the
/// journal said so): every photo it pinned is a frame holding a file of the target's format converted from the content
- /// pinned. Without a pin (its journal lost), every photo confirmed () is a frame
- /// holding a conversion to the target whose sources have the step's digest; a job an earlier build queued: the frames
- /// holding a conversion to the target are, by their sources, the photos of the step's digest, or every staged conversion
- /// is a frame holding the converted content. Null otherwise.
+ /// pinned. Without a pin (its journal lost), every photo confirmed () is a frame holding a
+ /// conversion to the target whose sources have the step's digest; without those either: the frames holding a conversion
+ /// to the target are, by their sources, the photos of the step's digest, or (a job an earlier build queued) every staged
+ /// conversion is a frame holding the converted content. Null otherwise.
///
- private static int? ReplacedEarlier(SequenceManifest manifest, ConvertSequence step, PinnedSequence? pin, IReadOnlyList noted,
- ConvertOptions options)
+ private static int? ReplacedEarlier(SequenceManifest manifest, ConvertSequence step, IReadOnlyList? photos, PinnedSequence? pin,
+ IReadOnlyList noted, ConvertOptions options)
{
var frames = manifest.Frames.ToDictionary(f => f.Id);
if (pin is not null)
@@ -872,7 +1031,7 @@ public sealed class MassConverter
? pin.Frames.Count
: null;
}
- if (step is { Frames: { } confirmed } && step.PhotoIds() is { Count: > 0 } photos)
+ if (step is { Frames: { } confirmed } && photos is { Count: > 0 })
{
var sources = new List<(Guid, string?)>(photos.Count);
foreach (var id in photos)
@@ -987,11 +1146,16 @@ public sealed class MassConverter
return null;
}
- /// The bytes the photos converts are estimated at, but for those whose staged file is there already.
- private static long Unstaged(ArchivePaths paths, Guid jobId, ConvertSequencePlan plan, StagedLog staged)
+ ///
+ /// The bytes the photos converts are estimated at, but for those whose staged file is there already,
+ /// or which the new sequence this job made holds (, the SHA-256 of its photos).
+ ///
+ private static long Unstaged(ArchivePaths paths, Guid jobId, ConvertSequencePlan plan, StagedLog staged, IReadOnlySet? filed)
{
var folder = StagingOf(paths, jobId, plan.SequenceId);
- var there = staged.Of(plan.SequenceId, plan.FramesVersion).Where(c => Present(Path.Combine(folder, c.File), c)).Select(c => c.FrameId).ToHashSet();
+ var there = staged.Of(plan.SequenceId, plan.FramesVersion)
+ .Where(c => filed?.Contains(c.Sha256) == true || Present(Path.Combine(folder, c.File), c))
+ .Select(c => c.FrameId).ToHashSet();
return plan.Frames.Where(f => !there.Contains(f.FrameId)).Sum(f => f.EstimatedBytes);
}
@@ -1042,6 +1206,7 @@ public sealed class MassConverter
ConvertOutcome.Converted => Loc.L("Archive.Log.Converted", result.Title, result.Frames),
ConvertOutcome.Created => Loc.L("Archive.Log.ConvertedNew", result.Title, result.Frames),
ConvertOutcome.Nothing => Loc.L("Archive.Log.ConvertNothing", result.Title),
+ ConvertOutcome.Superseded => Loc.L("Archive.Log.ConvertSuperseded", result.Title),
ConvertOutcome.Changed => Loc.L("Archive.Log.ConvertChanged", result.Title),
// Busy, or why the retry may do better (a photo another program holds, a folder another index moved), in one sentence.
ConvertOutcome.Blocked when result.Error is null => Loc.L("Archive.Log.ConvertBlocked", result.Title),
@@ -1067,7 +1232,11 @@ public sealed class MassConverter
: Loc.L("Common.Size.MB", (bytes / 1e6).ToString(digits, Loc.Culture));
/// The session key of the new sequence a conversion makes from .
- internal static string SessionKey(Guid job, Guid source) => $"convert:{job:N}:{source:N}";
+ internal static string SessionKey(Guid job, Guid source) => $"{KeyPrefix}{job:N}:{source:N}";
+
+ /// The conversion job a session key of names; null for any other key.
+ private static Guid? JobOfKey(string key) =>
+ key.Split(':') is ["convert", var job, _] && Guid.TryParseExact(job, "N", out var id) ? id : null;
/// The folder under .chronos/staging a conversion job stages into, one subfolder per sequence.
internal static string StagingFolder(Guid job) => ConvertJournal.Prefix + job.ToString("N");
@@ -1112,34 +1281,68 @@ public sealed class MassConverter
}
///
- /// The free space a running conversion watches: measured where it writes, against estimates scaled by how far the files it
- /// staged came out above theirs (photos of a 14-bit RAW carry more noise than the estimate's bits per pixel allow for).
+ /// The free space a running conversion watches: measured where it writes (), against estimates scaled
+ /// by how far the files it staged came out above theirs (photos of a 14-bit RAW carry more noise than the estimate's bits
+ /// per pixel allow for), and, in stage A, with the photos started whose converted file is not there yet ():
+ /// a converted file is encoded in memory and lands whole, by a rename, before its photo's work verifies it, so the free
+ /// space measured holds every file landed, and none of what the others will write.
///
- private sealed class SpaceWatch(MassConverter converter, ArchivePaths paths)
+ internal sealed class SpaceWatch(Func free)
{
+ private readonly ConcurrentDictionary _started = new(StringComparer.Ordinal);
private long _actual;
private long _estimated;
/// The bytes staged over their estimates, at least 1.
public double Ratio => _estimated > 0 ? Math.Max(1, (double)_actual / _estimated) : 1;
+ ///
+ /// The estimated bytes of the photos started () whose converted file has not landed yet: the file is
+ /// not at its path, or is still the one that was there when the photo started (one a crash left, which it replaces).
+ ///
+ public long Writing => _started.Sum(s => Landed(s.Key) == s.Value.Before ? s.Value.Estimate : 0);
+
public void Staged(long actual, long estimated)
{
_actual += actual;
_estimated += Math.Max(1, estimated);
}
+ ///
+ /// Why a photo estimated at bytes, converted into , cannot start now:
+ /// the volume lacks the space for it and for the photos started before it whose file has not landed yet
+ /// (, ), localized. Null when it can: it counts among those until its file lands
+ /// or its work ends ().
+ ///
+ public string? Start(long estimated, string target)
+ {
+ var before = Landed(target);
+ if (Short(Math.Max(0, estimated) + Writing) is { } message) return message;
+ _started[target] = new Starting(Math.Max(0, estimated), before);
+ return null;
+ }
+
+ /// The work of the photo converted into ended: its file landed, or never will.
+ public void Ended(string target) => _started.TryRemove(target, out _);
+
///
/// Why the volume cannot take bytes more (scaled, with the margin and the reserve), localized;
/// null when it can, or when its free space cannot be measured.
///
public string? Short(long estimated)
{
- var space = new ConvertSpace(ConvertPlanner.Required((long)Math.Ceiling(estimated * Ratio)), converter.FreeSpace(paths));
+ var space = new ConvertSpace(ConvertPlanner.Required((long)Math.Ceiling(estimated * Ratio)), free());
if (space.Enough) return null;
- var (required, free) = Sizes(space.Required, space.Free ?? 0);
- return Loc.L("Archive.Error.ConvertSpace", required, free);
+ var (required, left) = Sizes(space.Required, space.Free ?? 0);
+ return Loc.L("Archive.Error.ConvertSpace", required, left);
}
+
+ /// The file at as a landing changes it, its size and time of writing; null when there is none.
+ private static (long Size, long Written)? Landed(string path) =>
+ new FileInfo(path) is { Exists: true } file ? (file.Length, file.LastWriteTimeUtc.Ticks) : null;
+
+ /// A photo started: its estimate, and the file at its path then.
+ private sealed record Starting(long Estimate, (long Size, long Written)? Before);
}
/// What a sequence came to: its result, the statuses its follow-ups follow from, and those follow-ups (null: none to queue).
@@ -1171,12 +1374,13 @@ internal sealed record PinnedFrame(Guid Id, string Sha256);
///
/// The journal of a conversion job (.chronos/journal/convert-<job>.json, owner review 2, R7): what each sequence
-/// came to, and whether the passes it needs were queued, written after each one; with it, the pin log
-/// (convert-<job>.pins, ), the photos the job converts of each sequence, and the staged log
-/// (convert-<job>.staged), one line per photo staged, both appended and flushed as they grow, so a job of thousands
-/// of photos journals each once. A resumed or retried job keeps the sequences that are final, the photos pinned, and the
-/// staged files whose hash still verifies. All of it goes when every sequence is final, or once the job is gone
-/// ().
+/// came to, the new sequence a New sequence conversion made of it as soon as its photos are filed, and whether the passes
+/// it needs were queued, written after each one; with it, the photos confirmed (convert-<job>.photos,
+/// , written when the job is queued), the pin log (convert-<job>.pins,
+/// ), the photos the job converts of each sequence, and the staged log (convert-<job>.staged),
+/// one line per photo staged, both appended and flushed as they grow, so a job of thousands of photos journals each once. A
+/// resumed or retried job keeps the sequences that are final, the copies made, the photos pinned, and the staged files
+/// whose hash still verifies. All of it goes when every sequence is final, or once the job is gone ().
///
internal sealed class ConvertJournal
{
@@ -1198,7 +1402,10 @@ internal sealed class ConvertJournal
noted.Had = had;
}
- /// Notes what a sequence came to and, when it was converted or made, the statuses its follow-ups follow from.
+ ///
+ /// Notes what a sequence came to and, when it was converted or made, the statuses its follow-ups follow from. The new
+ /// sequence a New sequence conversion made of it stays named whatever it came to since ().
+ ///
public void Note(ConvertSequenceResult result, SequenceStatus? had = null)
{
var noted = Of(result.SequenceId);
@@ -1206,10 +1413,21 @@ internal sealed class ConvertJournal
noted.Title = result.Title;
noted.Outcome = result.Outcome;
noted.Frames = result.Frames;
- noted.NewSequenceId = result.NewSequenceId;
+ noted.NewSequenceId = result.NewSequenceId ?? noted.NewSequenceId;
if (had is { } status) noted.Had = status;
}
+ ///
+ /// Notes the new sequence a New sequence conversion filed the photos of into, before anything else is
+ /// done with it: a retry or a resume finishes that one and makes no other.
+ ///
+ public void Made(Guid id, Guid newSequenceId)
+ {
+ var noted = Of(id);
+ if (noted is null) Sequences.Add(noted = new ConvertJournalSequence { Id = id });
+ noted.NewSequenceId = newSequenceId;
+ }
+
public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.json");
public static string StagedPathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.staged");
@@ -1256,8 +1474,9 @@ internal sealed class ConvertJournal
/// The pin log of this job, read now.
public PinLog Pins(ArchivePaths paths) => PinLog.Open(PinsPathFor(paths, JobId));
- /// The journal files of , whichever exist.
- public static IEnumerable Files(ArchivePaths paths, Guid job) => [StagedPathFor(paths, job), PinsPathFor(paths, job), PathFor(paths, job)];
+ /// The journal files of , whichever exist: the photos confirmed among them ().
+ public static IEnumerable Files(ArchivePaths paths, Guid job) =>
+ [StagedPathFor(paths, job), PinsPathFor(paths, job), ConfirmedPhotos.PathFor(paths, job), PathFor(paths, job)];
public static void Delete(ArchivePaths paths, Guid job)
{
@@ -1291,7 +1510,7 @@ internal sealed class ConvertJournalSequence
public bool Queued { get; set; }
[JsonIgnore]
- public bool Final => Outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing;
+ public bool Final => ConvertSequenceResult.IsFinal(Outcome);
}
/// The staged log of a conversion job (): read once, appended to line by line.
@@ -1380,6 +1599,114 @@ internal sealed class PinLog
}
}
+/// A line of : a sequence, and the frame ids of the photos converted of it, 16 bytes each, in base64.
+internal sealed record ConfirmedLine(Guid Sequence, string Photos);
+
+///
+/// The photos a conversion job converts of each sequence, as its plan was confirmed (.chronos/journal/convert-<job>.photos,
+/// owner review 2, R7): written whole under the job's id before it is queued (), one
+/// line per sequence with the frame ids in the plan's order. The job's row keeps each sequence's digest of those photos
+/// with their content (), never the ids: it stays as small as the number of sequences
+/// whatever the number of photos, for every list and check that reads jobs. Under each sequence's lease the job takes these
+/// photos from its plan made again and goes on only while they have that digest; a sequence this file does not name (it
+/// could not be written, or a job an earlier build queued) is held to its whole plan. It goes with the journal
+/// ().
+///
+internal sealed class ConfirmedPhotos
+{
+ private readonly Dictionary> _photos = [];
+
+ public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{ConvertJournal.Prefix}{job:N}.photos");
+
+ /// The photos confirmed of , in the plan's order; null when the file does not name it.
+ public IReadOnlyList? Of(Guid sequence) => _photos.GetValueOrDefault(sequence);
+
+ /// The photos confirmed for : none when the file is not there; a line that cannot be read names nothing.
+ public static ConfirmedPhotos Read(ArchivePaths paths, Guid job)
+ {
+ var read = new ConfirmedPhotos();
+ foreach (var line in JournalLines.Read(PathFor(paths, job)))
+ {
+ try
+ {
+ if (JsonSerializer.Deserialize(line, ArchiveJobJson.Default.ConfirmedLine) is { } confirmed) read._photos[confirmed.Sequence] = Decode(confirmed.Photos);
+ }
+ catch (Exception ex) when (ex is JsonException or FormatException)
+ {
+ Core.Diagnostics.Log.Warning("archive", $"A line of the photos confirmed for conversion job {job} cannot be read; that sequence is held to its whole plan", ex);
+ }
+ }
+ return read;
+ }
+
+ ///
+ /// Writes the photos converts of each sequence it converts, for the job , whole
+ /// (a reader finds all of it or none); false, logged, when it cannot be written.
+ ///
+ public static bool TryWrite(ArchivePaths paths, Guid job, ConvertPlan plan)
+ {
+ ArgumentNullException.ThrowIfNull(plan);
+ try
+ {
+ using var lines = new MemoryStream();
+ foreach (var sequence in plan.Converted)
+ {
+ lines.Write(JsonSerializer.SerializeToUtf8Bytes(new ConfirmedLine(sequence.SequenceId, Encode(sequence.Frames.Select(f => f.FrameId))),
+ ArchiveJobJson.Default.ConfirmedLine));
+ lines.WriteByte((byte)'\n');
+ }
+ System.IO.Directory.CreateDirectory(paths.JournalDirectory);
+ AtomicFile.WriteAllBytes(PathFor(paths, job), lines.GetBuffer().AsSpan(0, (int)lines.Length));
+ return true;
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be written; it holds each sequence to its whole plan", ex);
+ return false;
+ }
+ }
+
+ /// Deletes the file of ; a failure is logged, and maintenance removes it later.
+ public static void Delete(ArchivePaths paths, Guid job)
+ {
+ try
+ {
+ File.Delete(PathFor(paths, job));
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be deleted; maintenance removes them later", ex);
+ }
+ }
+
+ /// Frame ids as a line holds them: their 16 bytes each, one after the other, in base64.
+ public static string Encode(IEnumerable ids)
+ {
+ ArgumentNullException.ThrowIfNull(ids);
+ IReadOnlyCollection frames = ids as IReadOnlyCollection ?? [.. ids];
+ var bytes = new byte[frames.Count * 16];
+ var at = 0;
+ foreach (var frame in frames)
+ {
+ frame.TryWriteBytes(bytes.AsSpan(at, 16));
+ at += 16;
+ }
+ return Convert.ToBase64String(bytes);
+ }
+
+ /// The frame ids wrote, in order.
+ /// The text is not frame ids as writes them.
+ public static IReadOnlyList Decode(string text)
+ {
+ ArgumentNullException.ThrowIfNull(text);
+ var bytes = Convert.FromBase64String(text);
+ if (bytes.Length % 16 != 0) throw new FormatException("The text is not frame ids.");
+ var ids = new Guid[bytes.Length / 16];
+ for (var i = 0; i < ids.Length; i++) ids[i] = new Guid(bytes.AsSpan(i * 16, 16));
+ return ids;
+ }
+}
+
/// Files of JSON lines a job appends to as it goes: each line written whole and flushed to disk, read back skipping blank ones.
internal static class JournalLines
{
diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs
index 1c8b3cc5..7a9312ee 100644
--- a/src/Chronos.Data/Archiving/SequenceTrash.cs
+++ b/src/Chronos.Data/Archiving/SequenceTrash.cs
@@ -6,6 +6,7 @@ using Chronos.Core.IO;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Data.Archiving.Jobs;
+using Chronos.Data.Repositories;
namespace Chronos.Data.Archiving;
@@ -115,7 +116,6 @@ public sealed partial class SequenceTrash
public const string FolderName = "trash";
private const string RecordExtension = ".json";
- private const int JobPage = 200;
/// Bytes of a sidecar read to find its id (written second, after the schema version).
private const int SidecarHead = 4096;
@@ -493,30 +493,23 @@ public sealed partial class SequenceTrash
///
internal Task> BusyAsync(CancellationToken cancellationToken) => BusyAsync(null, cancellationToken);
- /// What running and paused jobs hold, but the job (the one asking).
+ ///
+ /// What running and paused jobs hold, but the job (the one asking), read from their targets
+ /// alone (): nothing reads the parameters a conversion or an import keeps.
+ ///
internal async Task> BusyAsync(Guid? except, CancellationToken cancellationToken)
{
var busy = new HashSet();
- foreach (var state in new[] { JobState.Running, JobState.Paused })
- {
- string? cursor = null;
- do
- {
- var page = await _db.Jobs.ListAsync(state, JobPage, cursor, cancellationToken).ConfigureAwait(false);
- foreach (var job in page.Items.Where(j => j.Id != except)) busy.UnionWith(Holds(job));
- cursor = page.NextCursor;
- }
- while (cursor is not null);
- }
+ foreach (var job in await _db.Jobs.ListHoldsAsync(except, cancellationToken).ConfigureAwait(false)) busy.UnionWith(Holds(job));
return busy;
}
- private static IReadOnlyList Holds(Job job)
+ private static IReadOnlyList Holds(JobHold job)
{
- if (job.Kind != JobKind.Verify) return job.TargetIds;
+ if (job is not { Kind: JobKind.Verify, ParametersJson: { Length: > 0 } json }) return job.TargetIds;
try
{
- return ArchiveJobJson.Read(job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null)).SequenceIds ?? job.TargetIds;
+ return JsonSerializer.Deserialize(json, ArchiveJobJson.Default.VerifyParameters)?.SequenceIds ?? job.TargetIds;
}
catch (JsonException)
{
diff --git a/src/Chronos.Data/Jobs/JobEngine.cs b/src/Chronos.Data/Jobs/JobEngine.cs
index e84eac82..e739aad8 100644
--- a/src/Chronos.Data/Jobs/JobEngine.cs
+++ b/src/Chronos.Data/Jobs/JobEngine.cs
@@ -122,6 +122,14 @@ public sealed class JobEngine : IAsyncDisposable
Guid? createdBy, Guid? scheduleId, CancellationToken cancellationToken = default) =>
EnqueueAsync(Guid.CreateVersion7(), kind, targets, targetLabel, parametersJson, priority, createdBy, scheduleId, cancellationToken);
+ ///
+ /// Queues a job under the id , chosen before (what it reads beside its journal is written under that
+ /// id first, as a conversion's confirmed photos are), and wakes a worker.
+ ///
+ public Task EnqueueAsync(Guid id, JobKind kind, IReadOnlyList targets, string? targetLabel, string? parametersJson, JobPriority priority,
+ Guid? createdBy, CancellationToken cancellationToken = default) =>
+ EnqueueAsync(id, kind, targets, targetLabel, parametersJson, priority, createdBy, null, cancellationToken);
+
///
/// True when this engine runs jobs of : it has a handler for it and
/// does not leave it out. Known before , so a host that does not start the engine still answers.
@@ -377,7 +385,7 @@ public sealed class JobEngine : IAsyncDisposable
{
var candidate = await AcceptedAsync(queue, null, cancellationToken).ConfigureAwait(false);
if (candidate is null) return _others.Count > 0 ? await _db.Jobs.ClaimNextAsync(workerId, _others, queue, cancellationToken).ConfigureAwait(false) : null;
- if (_others.Count > 0 && (await _db.Jobs.ListQueuedAsync(_others, 1, queue, cancellationToken).ConfigureAwait(false)).FirstOrDefault() is { } head
+ if (_others.Count > 0 && (await _db.Jobs.ListQueuedBriefAsync(_others, 1, queue, cancellationToken).ConfigureAwait(false)).FirstOrDefault() is { } head
&& queue.ClaimsBefore(head, candidate))
{
if (await _db.Jobs.ClaimNextAsync(workerId, _others, queue, cancellationToken).ConfigureAwait(false) is { } next) return next;
@@ -397,7 +405,8 @@ public sealed class JobEngine : IAsyncDisposable
Job? head = null;
if (_others.Count > 0)
{
- foreach (var queued in await _db.Jobs.ListQueuedAsync(_others, ReviewWindow, queue, cancellationToken).ConfigureAwait(false))
+ // Only who comes first counts here: the job that yields to it reads its kind, and a claim reads a job whole.
+ foreach (var queued in await _db.Jobs.ListQueuedBriefAsync(_others, ReviewWindow, queue, cancellationToken).ConfigureAwait(false))
{
if (running.FollowUps.ContainsKey(queued.Id)) continue;
head = queued;
diff --git a/src/Chronos.Data/Repositories/JobRepository.cs b/src/Chronos.Data/Repositories/JobRepository.cs
index 8f29cffc..356a66d0 100644
--- a/src/Chronos.Data/Repositories/JobRepository.cs
+++ b/src/Chronos.Data/Repositories/JobRepository.cs
@@ -7,6 +7,12 @@ namespace Chronos.Data.Repositories;
/// Jobs per state; and count those finished since a given time.
public sealed record JobCounts(int Queued, int Running, int Paused, int DoneSince, int FailedSince);
+///
+/// What a running or paused job works on (): its targets and, for a verify, its
+/// parameters, which name the sequences it holds; null for every other kind, whose parameters are not read.
+///
+public sealed record JobHold(Guid Id, JobKind Kind, IReadOnlyList TargetIds, string? ParametersJson);
+
///
/// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically,
/// write checkpoints while running, and on startup requeue what a crash left running.
@@ -17,6 +23,14 @@ public sealed class JobRepository
"id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " +
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id";
+ ///
+ /// as lists read them (): the parameters left out, null. A conversion's or
+ /// an import's name every sequence or file they work on; the job reads them, and so do the screens that act on that one job.
+ ///
+ private const string BriefColumns =
+ "id, kind, target_ids, target_label, CAST(NULL AS TEXT), state, priority, progress_done, progress_total, items_per_second, " +
+ "eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id";
+
/// The states of a job that has not ended: queued, running, paused.
private static readonly string PendingStates =
string.Join(", ", new[] { JobState.Queued, JobState.Running, JobState.Paused }.Select(s => ((int)s).ToString(CultureInfo.InvariantCulture)));
@@ -37,6 +51,11 @@ public sealed class JobRepository
public Task GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken);
+ /// When the job was queued; null when there is no such job. Reads that column alone.
+ public async Task GetCreatedAsync(Guid id, CancellationToken cancellationToken = default) =>
+ (await _db.ListAsync("SELECT created FROM jobs WHERE id = $1", [id], r => r.GetDateTimeOffset(0), cancellationToken).ConfigureAwait(false))
+ is [var created] ? created : null;
+
///
/// Marks the next queued job (highest priority, then oldest) of the given kinds as running for
/// and returns it, or null when none waits. Concurrent claimers never get the same
@@ -93,7 +112,17 @@ public sealed class JobRepository
ListAsync(state, kind: null, limit, cursor, cancellationToken);
/// Jobs, newest first, optionally of one state and one kind.
- public Task> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default)
+ public Task> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
+ PageAsync(Columns, state, kind, limit, cursor, cancellationToken);
+
+ ///
+ /// as lists show jobs: every field but
+ /// , which stays null ( reads a job whole).
+ ///
+ public Task> ListBriefAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
+ PageAsync(BriefColumns, state, kind, limit, cursor, cancellationToken);
+
+ private Task> PageAsync(string columns, JobState? state, JobKind? kind, int limit, string? cursor, CancellationToken cancellationToken)
{
var p = new SqlParameters();
var where = new List();
@@ -105,7 +134,7 @@ public sealed class JobRepository
where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add));
}
var take = Cursor.ClampLimit(limit);
- var sql = $"SELECT {Columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
+ var sql = $"SELECT {columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
$"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}";
return _db.RunAsync(async session =>
{
@@ -158,11 +187,22 @@ public sealed class JobRepository
ListQueuedAsync(kinds, limit, JobPriorityQueue.Off, cancellationToken);
/// Queued jobs of the given kinds in the claim order of .
- public Task> ListQueuedAsync(IReadOnlyCollection kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default)
+ public Task> ListQueuedAsync(IReadOnlyCollection kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) =>
+ QueuedAsync(Columns, kinds, limit, queue, cancellationToken);
+
+ ///
+ /// without the
+ /// parameters (null), for a look at what would be claimed next: claiming a job reads it whole.
+ ///
+ public Task> ListQueuedBriefAsync(IReadOnlyCollection kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) =>
+ QueuedAsync(BriefColumns, kinds, limit, queue, cancellationToken);
+
+ private Task> QueuedAsync(string columns, IReadOnlyCollection kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken)
{
+ ArgumentNullException.ThrowIfNull(kinds);
ArgumentNullException.ThrowIfNull(queue);
return kinds.Count == 0 ? Task.FromResult>([]) : _db.ListAsync(
- $"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1",
+ $"SELECT {columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1",
[Cursor.ClampLimit(limit)], Map, cancellationToken);
}
@@ -195,6 +235,18 @@ public sealed class JobRepository
return await _db.ScalarAsync(sql, p.ToArray(), cancellationToken).ConfigureAwait(false) > 0;
}
+ ///
+ /// What the running and paused jobs hold, but the job (the one asking): each one's id, kind and
+ /// targets, and only a verify's parameters (). Neither the other parameters nor the targets of
+ /// are read: a conversion asks at every sequence it comes to, and its own row names them all.
+ ///
+ public Task> ListHoldsAsync(Guid? except = null, CancellationToken cancellationToken = default) =>
+ _db.ListAsync(
+ $"SELECT id, kind, target_ids, CASE WHEN kind = {(int)JobKind.Verify} THEN parameters_json END FROM jobs " +
+ $"WHERE state IN ({(int)JobState.Running}, {(int)JobState.Paused}){(except is null ? string.Empty : " AND id <> $1")} ORDER BY created, id",
+ except is { } self ? [self] : [],
+ r => new JobHold(r.GetGuid(0), (JobKind)r.GetInt32(1), Targets(r.GetString(2)), r.GetStringOrNull(3)), cancellationToken);
+
/// The targets of the queued, running and paused jobs of (the newest 5000 such jobs).
public Task> ListPendingTargetsAsync(JobKind kind, CancellationToken cancellationToken = default) =>
PendingTargetsAsync(kind, null, cancellationToken);
@@ -306,11 +358,14 @@ public sealed class JobRepository
private static string KindList(IEnumerable kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture)));
+ /// Target ids as the table stores them: comma-joined, empty for none.
+ private static Guid[] Targets(string ids) => ids.Length > 0 ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [];
+
private static Job Map(DbRow r) => new()
{
Id = r.GetGuid(0),
Kind = (JobKind)r.GetInt32(1),
- TargetIds = r.GetString(2) is { Length: > 0 } ids ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [],
+ TargetIds = Targets(r.GetString(2)),
TargetLabel = r.GetStringOrNull(3),
ParametersJson = r.GetStringOrNull(4),
State = (JobState)r.GetInt32(5),
diff --git a/src/Chronos.Data/Repositories/SequenceRepository.cs b/src/Chronos.Data/Repositories/SequenceRepository.cs
index d2d94ef6..a3338f62 100644
--- a/src/Chronos.Data/Repositories/SequenceRepository.cs
+++ b/src/Chronos.Data/Repositories/SequenceRepository.cs
@@ -565,6 +565,14 @@ public sealed class SequenceRepository
_db.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
[archiveId, sessionKey], cancellationToken);
+ ///
+ /// The session keys of an archive that match (a LIKE pattern, \ its escape), each
+ /// with the sequence it resolves to, by key: the new sequences conversions made of one source, say.
+ ///
+ public Task> ListSessionKeysAsync(Guid archiveId, string pattern, CancellationToken cancellationToken = default) =>
+ _db.ListAsync("SELECT session_key, sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key LIKE $2 ESCAPE '\\' ORDER BY session_key",
+ [archiveId, pattern], r => (r.GetString(0), r.GetGuid(1)), cancellationToken);
+
/// Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to.
public Task MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
diff --git a/src/Chronos.Server/Api/AdminEndpoints.cs b/src/Chronos.Server/Api/AdminEndpoints.cs
index f59ca983..3e504403 100644
--- a/src/Chronos.Server/Api/AdminEndpoints.cs
+++ b/src/Chronos.Server/Api/AdminEndpoints.cs
@@ -48,8 +48,8 @@ public static class AdminEndpoints
var users = await db.Users.ListAsync(ct);
var names = users.ToDictionary(u => u.Id, u => u.Username);
var counts = await db.Jobs.CountAsync(today, ct);
- var running = await db.Jobs.ListAsync(JobState.Running, null, 10, null, ct);
- var queued = await db.Jobs.ListAsync(JobState.Queued, null, 10, null, ct);
+ var running = await db.Jobs.ListBriefAsync(JobState.Running, null, 10, null, ct);
+ var queued = await db.Jobs.ListBriefAsync(JobState.Queued, null, 10, null, ct);
var nodes = await db.RenderNodes.ListAsync(ct);
var uploads = await db.Uploads.CommittedPerDayAsync(today.AddDays(-29), ct);
var activity = await db.Audit.ListAsync(null, null, 12, null, ct);
diff --git a/src/Chronos.Server/Api/JobEndpoints.cs b/src/Chronos.Server/Api/JobEndpoints.cs
index bf219330..6817fe88 100644
--- a/src/Chronos.Server/Api/JobEndpoints.cs
+++ b/src/Chronos.Server/Api/JobEndpoints.cs
@@ -56,7 +56,8 @@ public static class JobEndpoints
if (!JobWire.TryParseKind(kind, out var k)) return ApiErrors.BadRequest(context, "invalid_filter", "kind");
kindFilter = k;
}
- var page = await db.Jobs.ListAsync(stateFilter, kindFilter, limit ?? 50, string.IsNullOrEmpty(cursor) ? null : cursor, context.RequestAborted);
+ // A job's answer holds no parameters (a conversion's or an import's can name every sequence or file): the list reads none.
+ var page = await db.Jobs.ListBriefAsync(stateFilter, kindFilter, limit ?? 50, string.IsNullOrEmpty(cursor) ? null : cursor, context.RequestAborted);
return Results.Json(new Page([.. page.Items.Select(Dto.Job)], page.NextCursor, null), ProtocolJson.Default.PageJobDto);
}
diff --git a/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs b/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs
index 4395d681..cea35108 100644
--- a/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs
+++ b/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs
@@ -76,18 +76,30 @@ public sealed class ConvertTests(TestContext context)
return ValueTask.CompletedTask;
});
- /// The conversion job of , run here as its handler runs it.
- private static Task RunHereAsync(ArchiveFixture fx, ConvertPlan plan, Guid jobId, ConvertCallbacks? callbacks = null) =>
- fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), jobId, new ConvertRequester(null, null), callbacks ?? Callbacks(), fx.Ct);
+ ///
+ /// The conversion job of , run here as its handler runs it, with the photos
+ /// confirmed written beside its journal as writes them (once: a run again of the
+ /// same job finds them), and as the time it was queued.
+ ///
+ private static Task RunHereAsync(ArchiveFixture fx, ConvertPlan plan, Guid jobId, ConvertCallbacks? callbacks = null, DateTimeOffset? confirmed = null)
+ {
+ if (!File.Exists(ConfirmedPhotos.PathFor(fx.Paths, jobId))) Assert.True(ConfirmedPhotos.TryWrite(fx.Paths, jobId, plan));
+ return fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), jobId, new ConvertRequester(null, null, confirmed), callbacks ?? Callbacks(), fx.Ct);
+ }
private static string StagingOf(ArchiveFixture fx, Guid job, Guid sequence) => Path.Combine(Staging(fx), $"convert-{job:N}", $"{sequence:N}");
- private static async Task RunAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, JobState state = JobState.Done)
+ /// The job of , queued as the Library, the Archive manager and the panel queue it ().
+ private static async Task QueueAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, Guid? createdBy = null)
{
- var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct);
- return await fx.WaitForAsync(job.Id, state, (int)TestTiming.Budget(60));
+ Job? job = null;
+ await fx.Service.Converter.QueueAsync(fx.Archive, plan, async r => job = await engine.EnqueueAsync(r, createdBy, fx.Ct), fx.Ct);
+ return Assert.NotNull(job);
}
+ private static async Task RunAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, JobState state = JobState.Done) =>
+ await fx.WaitForAsync((await QueueAsync(fx, engine, plan)).Id, state, (int)TestTiming.Budget(60));
+
private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
@@ -721,9 +733,16 @@ public sealed class ConvertTests(TestContext context)
File.AppendAllText(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId), "{\"sequence\":");
var pinned = Assert.NotNull(read.Pins(fx.Paths).Of(first));
Assert.Equal((5, photo, "bb"), (pinned.FramesVersion, pinned.Frames.Single().Id, pinned.Frames.Single().Sha256));
- // Its files and its staging folder are known by their names.
- Assert.SequenceEqual([journal.JobId, journal.JobId, journal.JobId],
+ // Its files (the photos confirmed among them) and its staging folder are known by their names.
+ Assert.SequenceEqual([journal.JobId, journal.JobId, journal.JobId, journal.JobId],
ConvertJournal.Files(fx.Paths, journal.JobId).Select(p => ConvertJournal.JobOf(Path.GetFileName(p))));
+ Assert.Contains(ConfirmedPhotos.PathFor(fx.Paths, journal.JobId), ConvertJournal.Files(fx.Paths, journal.JobId));
+ // The copy a New sequence conversion made stays named whatever the sequence comes to after it.
+ var copy = Guid.NewGuid();
+ journal.Made(harbour, copy);
+ journal.Note(new ConvertSequenceResult(harbour, "Harbour", ConvertOutcome.Failed, 0, Error: "no"));
+ journal.Write(fx.Paths);
+ Assert.Equal(copy, ConvertJournal.Read(fx.Paths, journal.JobId)!.Of(harbour)!.NewSequenceId);
Assert.Equal(journal.JobId, ConvertJournal.JobOf(MassConverter.StagingFolder(journal.JobId)));
Assert.Null(ConvertJournal.JobOf("convert-test"));
Assert.Null(ConvertJournal.JobOf("rename-" + journal.JobId.ToString("N") + ".json"));
@@ -1257,16 +1276,85 @@ public sealed class ConvertTests(TestContext context)
}
///
- /// Before each photo of stage A, the space is checked for it and the photos after it that may convert at once with it:
- /// fewer near the end, so a job whose dry run had the space never stops short for photos that are not there.
+ /// Before each photo of stage A, the space is checked for it and for every photo handed to the work before it whose file
+ /// has not landed yet: a converted file lands whole, by a rename, before its photo's work verifies it, so the free space
+ /// measured holds the files landed and none of what the photos still converting will write. Near the end of a sequence
+ /// those are as many as anywhere else, never fewer; photos that are not there to start are never counted.
///
[Test]
- public async Task TheLastPhotosToStartNeedOnlyTheirOwnSpace()
+ public void APhotoStartsOnlyWithTheSpaceForItAndThePhotosStillWriting()
{
- ConvertFramePlan[] todo = [.. new long[] { 10, 20, 30, 40 }.Select((bytes, i) => new ConvertFramePlan(Guid.NewGuid(), i, $"originals/{i}.JPG", ImageFileFormat.Jpeg, 1, bytes))];
- Assert.SequenceEqual([60L, 90, 70, 40], Enumerable.Range(0, todo.Length).Select(i => MassConverter.Ahead(todo, i, 3)));
- Assert.SequenceEqual([10L, 20, 30, 40], Enumerable.Range(0, todo.Length).Select(i => MassConverter.Ahead(todo, i, 1)));
+ const long photo = 100_000_000;
+ var staging = context.CreateTempDirectory("staging");
+ string Target(int i) => Path.Combine(staging, $"{i:00}.tif");
+ // The volume: what it had, less the files landed so far (those still converting are not on it yet).
+ var landed = 0L;
+ var volume = ConvertPlanner.Required(3 * photo);
+ var space = new MassConverter.SpaceWatch(() => volume - landed);
+ void Land(int i)
+ {
+ File.WriteAllBytes(Target(i), [1]);
+ landed += photo;
+ }
+ // The last photos of a sequence, four converting at once: none of the three started has landed, so the fourth, the
+ // last one, needs the space of all four, not its own alone.
+ for (var i = 0; i < 3; i++) Assert.Null(space.Start(photo, Target(i)));
+ Assert.Equal(3 * photo, space.Writing);
+ var refused = Assert.NotNull(space.Start(photo, Target(3)));
+ Assert.True(refused.StartsWith("The conversion needs", StringComparison.Ordinal), refused);
+ Assert.Equal(3 * photo, space.Writing);
+
+ // One lands, and its work verifies it: the volume holds it, so the watch counts it no more (never twice). The last
+ // photo still waits: the two others and it need more than what is left.
+ Land(0);
+ Assert.Equal(2 * photo, space.Writing);
+ Assert.NotNull(space.Start(photo, Target(3)));
+ space.Ended(Target(0));
+ Land(1);
+ Land(2);
+ Assert.Equal(0, space.Writing);
+ space.Ended(Target(1));
+ space.Ended(Target(2));
+ // All three landed: what is left, with the margin and the reserve, cannot take a fourth; the reserve is whole.
+ Assert.NotNull(space.Start(photo, Target(3)));
+ Assert.True(volume - landed >= ConvertPlanner.SpaceReserve);
+
+ // Room for one more: it starts, and a photo whose work ends without a file (it failed) counts no more either.
+ volume = landed + ConvertPlanner.Required(photo);
+ Assert.Null(space.Start(photo, Target(3)));
+ Assert.NotNull(space.Start(photo, Target(4)));
+ space.Ended(Target(3));
+ Assert.Null(space.Start(photo, Target(4)));
+ space.Ended(Target(4));
+
+ // A file a crash left where a photo converts is not its file: the photo counts until the new one replaces it.
+ File.WriteAllBytes(Target(5), [1, 2, 3]);
+ File.SetLastWriteTimeUtc(Target(5), new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc));
+ Assert.Null(space.Start(photo, Target(5)));
+ Assert.Equal(photo, space.Writing);
+ File.WriteAllBytes(Target(5), new byte[10]);
+ Assert.Equal(0, space.Writing);
+
+ // Files staged over their estimate scale what is still writing too.
+ var scaled = new MassConverter.SpaceWatch(() => ConvertPlanner.Required(3 * photo));
+ scaled.Staged(2 * photo, photo);
+ Assert.Equal(2, scaled.Ratio);
+ Assert.Null(scaled.Start(photo, Target(6)));
+ Assert.NotNull(scaled.Start(photo, Target(7)));
+ scaled.Ended(Target(6));
+ Assert.Null(scaled.Start(photo, Target(7)));
+ // A volume that cannot be measured stops nothing.
+ Assert.Null(new MassConverter.SpaceWatch(() => null).Start(long.MaxValue / 4, Target(8)));
+ }
+
+ ///
+ /// A job whose dry run had the space never stops short at the end of a sequence: the last photo to start counts only
+ /// the photos that started and have not written yet, none that are not there.
+ ///
+ [Test]
+ public async Task TheLastPhotosToStartNeedOnlyTheSpaceOfThePhotosStillWriting()
+ {
long free = 1L << 50;
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free, Parallelism = () => 4 });
var id = await UploadedAsync(fx, 1);
@@ -1280,6 +1368,41 @@ public sealed class ConvertTests(TestContext context)
Assert.Equal((ConvertOutcome.Converted, 1), (done.Outcome, done.Frames));
}
+ ///
+ /// A sequence that fits converts to its last photo, four at once, while its converted files take the volume as they land:
+ /// at the end the photos still writing are counted, those landed (verified meanwhile) are in the free space measured, and
+ /// none is counted twice.
+ ///
+ [Test]
+ public async Task ASequenceThatFitsConvertsToItsLastPhotoWhileItsFilesFillTheVolume()
+ {
+ long volume = 1L << 50;
+ string? staging = null;
+ // The volume as the converted files land on it; an atomic write's temporary file is left out, being there only while
+ // its photo still counts.
+ long Landed() => staging is not null && Directory.Exists(staging)
+ ? Directory.EnumerateFiles(staging).Where(f => !Path.GetFileName(f).StartsWith('.')).Sum(f => new FileInfo(f).Length)
+ : 0;
+ await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => volume - Landed(), Parallelism = () => 4 });
+ var id = await UploadedAsync(fx, 12);
+ var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
+ // What the twelve converted files take, each converted beforehand as the job converts it.
+ var sequence = (await fx.Db.Sequences.GetAsync(id))!;
+ var folder = fx.Paths.Sequence(sequence.Folder);
+ var manifest = Sidecar(fx, sequence);
+ var sizes = context.CreateTempDirectory("sizes");
+ var converted = manifest.Frames.Sum(f => FrameConverter.Convert(new ConvertSource(folder.File(f.File), f, manifest.Camera, manifest.Lens),
+ new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), Path.Combine(sizes, f.Id.ToString("N") + ".tif"), cancellationToken: fx.Ct).Size);
+ // Just the space they need with the margin and the reserve, and never less than the dry run asks.
+ volume = ConvertPlanner.Required(Math.Max(converted, plan.BytesAfter));
+ var jobId = Guid.CreateVersion7();
+ staging = StagingOf(fx, jobId, id);
+
+ var done = (await RunHereAsync(fx, plan, jobId)).Sequences.Single();
+ Assert.Equal((ConvertOutcome.Converted, 12), (done.Outcome, done.Frames));
+ Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
+ }
+
[Test]
public async Task ASecondConversionOfTheSameSequencesIsRefusedWithOrWithoutADryRun()
{
@@ -1333,26 +1456,239 @@ public sealed class ConvertTests(TestContext context)
var first = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-2));
var second = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-1));
- var held = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null), Callbacks(), fx.Ct))
+ var held = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null, second.Created), Callbacks(), fx.Ct))
.Sequences.Single();
Assert.Equal(ConvertOutcome.Blocked, held.Outcome);
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(second.Id, id)));
// The one queued first is not held back by the one after it.
- var made = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), first.Id, new ConvertRequester(null, null), Callbacks(), fx.Ct))
+ var made = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), first.Id, new ConvertRequester(null, null, first.Created), Callbacks(), fx.Ct))
.Sequences.Single();
Assert.Equal((ConvertOutcome.Created, 2), (made.Outcome, made.Frames));
+ await fx.Db.Jobs.SetStateAsync(first.Id, JobState.Done, null, fx.Ct);
+ // The duplicate, tried again once the first is done, finds the copy made since it was queued: it makes no second one.
+ var again = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null, second.Created), Callbacks(), fx.Ct))
+ .Sequences.Single();
+ Assert.Equal(ConvertOutcome.Superseded, again.Outcome);
+ Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(second.Id, id)));
+ }
+
+ ///
+ /// A job's row names no photo, so it stays as small as the number of sequences however many photos convert: the photos
+ /// confirmed are written whole beside the journal, under the job's id, before it is queued, and go with the journal.
+ ///
+ [Test]
+ public async Task TheJobRowNamesNoPhotoAndThePhotosConfirmedWaitBesideItsJournal()
+ {
+ await using var fx = await FixtureAsync();
+ var id = await UploadedAsync(fx, 3);
+ var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
+ var frames = plan.Sequences.Single().Frames.Select(f => f.FrameId).ToList();
+ // Queued and not run yet: an engine that is not started.
+ await using var idle = new JobEngine(fx.Db, []);
+ Job? job = null;
+ await fx.Service.Converter.QueueAsync(fx.Archive, plan, async r =>
+ {
+ Assert.FileExists(ConfirmedPhotos.PathFor(fx.Paths, r.Id!.Value));
+ job = await idle.EnqueueAsync(r, cancellationToken: fx.Ct);
+ }, fx.Ct);
+ var queued = Assert.NotNull(await fx.Db.Jobs.GetAsync(job!.Id, fx.Ct));
+ Assert.SequenceEqual(frames, ConfirmedPhotos.Read(fx.Paths, queued.Id).Of(id)!);
+ Assert.Null(ConfirmedPhotos.Read(fx.Paths, queued.Id).Of(Guid.NewGuid()));
+ foreach (var frame in frames)
+ {
+ Assert.False(queued.ParametersJson!.Contains(frame.ToString(), StringComparison.OrdinalIgnoreCase), queued.ParametersJson);
+ Assert.False(queued.ParametersJson!.Contains(ConfirmedPhotos.Encode([frame])[..20], StringComparison.Ordinal), queued.ParametersJson);
+ }
+ Assert.Equal(plan.Sequences.Single().Digest(), Parameters(fx, plan).Sequences.Single().Frames);
+ // Lists of jobs read none of their parameters.
+ Assert.Null((await fx.Db.Jobs.ListBriefAsync(JobState.Queued, JobKind.Convert, 10, null, fx.Ct)).Items.Single().ParametersJson);
+
+ // A confirmation the queue refuses leaves nothing behind.
+ var other = await UploadedAsync(fx, 1, "upload-2", seed: 300);
+ var refused = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, other);
+ Guid? named = null;
+ await Assert.ThrowsAsync(() => fx.Service.Converter.QueueAsync(fx.Archive, refused, r =>
+ {
+ named = r.Id;
+ throw new InvalidOperationException("the index is away");
+ }, fx.Ct));
+ Assert.FileMissing(ConfirmedPhotos.PathFor(fx.Paths, named!.Value));
+
+ // The job runs, and once done with every sequence, its photos go with its journal.
+ var engine = await fx.StartEngineAsync();
+ engine.Wake();
+ await fx.WaitForAsync(queued.Id, JobState.Done, (int)TestTiming.Budget(60));
+ Assert.FileMissing(ConfirmedPhotos.PathFor(fx.Paths, queued.Id));
+ Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
}
[Test]
- public void AJobsParametersNameThePhotosItConverts()
+ public void ThePhotosConfirmedReadBackAsWrittenAndALineThatCannotBeReadNamesNothing()
{
Guid[] ids = [Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.NewGuid()];
- var step = new ConvertSequence(Guid.NewGuid(), 3, "digest", ConvertSequence.EncodePhotos(ids));
- Assert.SequenceEqual(ids, step.PhotoIds()!);
- var json = JsonSerializer.Serialize(step, ArchiveJobJson.Default.ConvertSequence);
- Assert.SequenceEqual(ids, JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ConvertSequence)!.PhotoIds()!);
- // A job an earlier build queued names none.
- Assert.Null(new ConvertSequence(Guid.NewGuid(), 3, "digest").PhotoIds());
- Assert.Empty(new ConvertSequence(Guid.NewGuid(), 3, "digest", ConvertSequence.EncodePhotos([])).PhotoIds()!);
+ Assert.SequenceEqual(ids, ConfirmedPhotos.Decode(ConfirmedPhotos.Encode(ids)));
+ Assert.Empty(ConfirmedPhotos.Decode(ConfirmedPhotos.Encode([])));
+ Assert.Throws(() => ConfirmedPhotos.Decode(Convert.ToBase64String(new byte[15])));
+
+ var paths = new ArchivePaths(context.CreateTempDirectory("archive"));
+ var job = Guid.CreateVersion7();
+ var (good, bad) = (Guid.NewGuid(), Guid.NewGuid());
+ Directory.CreateDirectory(paths.JournalDirectory);
+ File.WriteAllLines(ConfirmedPhotos.PathFor(paths, job),
+ [
+ JsonSerializer.Serialize(new ConfirmedLine(good, ConfirmedPhotos.Encode(ids)), ArchiveJobJson.Default.ConfirmedLine),
+ JsonSerializer.Serialize(new ConfirmedLine(bad, "not ids"), ArchiveJobJson.Default.ConfirmedLine),
+ "{\"sequence\":",
+ ]);
+ var read = ConfirmedPhotos.Read(paths, job);
+ Assert.SequenceEqual(ids, read.Of(good)!);
+ Assert.Null(read.Of(bad));
+ // None at all where nothing was written: every sequence is then held to its whole plan.
+ Assert.Null(ConfirmedPhotos.Read(paths, Guid.CreateVersion7()).Of(good));
+ }
+
+ ///
+ /// Without the photos confirmed (they could not be written when the job was queued, or an earlier build queued it), a
+ /// sequence converts only while its whole plan made again is the one confirmed: an upload since leaves it as it is.
+ ///
+ [Test]
+ public async Task WithoutThePhotosConfirmedASequenceIsHeldToItsWholePlan()
+ {
+ await using var fx = await FixtureAsync();
+ var still = await UploadedAsync(fx, 2);
+ var grown = await UploadedAsync(fx, 2, "upload-2", seed: 300);
+ var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, still, grown);
+ var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
+ await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, grown, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
+ var before = await FramesAsync(fx, grown);
+
+ var result = await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), Guid.CreateVersion7(), new ConvertRequester(null, null), Callbacks(), fx.Ct);
+ Assert.SequenceEqual([ConvertOutcome.Converted, ConvertOutcome.Changed], result.Sequences.Select(s => s.Outcome));
+ Assert.SequenceEqual(before.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, grown)).Select(f => (f.Id, f.Sha256)));
+ }
+
+ ///
+ /// A New sequence job that stopped once its copy held the photos, before its journal said the sequence was done (a crash,
+ /// a stop): retried after the source changed, it finishes that copy and makes no other; the copy is reported as made.
+ ///
+ [Test]
+ public async Task ARetryAfterACrashOnceTheCopyWasFiledFinishesThatCopyAndMakesNoOther()
+ {
+ await using var fx = await FixtureAsync();
+ var id = await UploadedAsync(fx, 3);
+ await fx.Service.SetSequenceInfoAsync(id, "Roof", ["sunset"], 4, "From the roof", fx.Ct);
+ var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
+ var jobId = Guid.CreateVersion7();
+ // What the attempt that stopped there left: the photos confirmed, pinned and staged, and the copy filed, the journal
+ // not saying more (it is written once the copy is finished).
+ var left = new Dictionary();
+ var callbacks = Callbacks(done: _ =>
+ {
+ foreach (var file in new[] { ConfirmedPhotos.PathFor(fx.Paths, jobId), ConvertJournal.PinsPathFor(fx.Paths, jobId), ConvertJournal.StagedPathFor(fx.Paths, jobId) })
+ left[file] = File.ReadAllBytes(file);
+ });
+ var first = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single();
+ Assert.Equal(ConvertOutcome.Created, first.Outcome);
+ var copy = first.NewSequenceId!.Value;
+ await fx.Db.Sequences.EditAsync(copy, new SequenceEdit { Tags = ["other"], Rating = 1, Notes = "Scrambled" }, null, fx.Ct);
+ foreach (var (file, bytes) in left) File.WriteAllBytes(file, bytes);
+ // The source changed since: a photo of it deleted.
+ var sequence = (await fx.Db.Sequences.GetAsync(id))!;
+ var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [(await FramesAsync(fx, id))[1].Id], sequence.FramesVersion, fx.Ct);
+ await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
+ var sequences = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count;
+
+ var retried = (await RunHereAsync(fx, plan, jobId)).Sequences.Single();
+ Assert.Equal((ConvertOutcome.Created, 3, (Guid?)copy, 0), (retried.Outcome, retried.Frames, retried.NewSequenceId, retried.LeftOut));
+ Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
+ Assert.Count(3, await FramesAsync(fx, copy));
+ // Finished as stage B finishes a copy: the source's tags, rating and notes.
+ var finished = (await fx.Db.Sequences.GetAsync(copy))!;
+ Assert.Equal((4, "From the roof"), (finished.Rating, finished.Notes));
+ Assert.SequenceEqual(["sunset"], finished.Tags);
+ Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId));
+
+ // Its journal names the copy as soon as the copy holds its photos: a copy deleted since is not made again.
+ var again = Guid.CreateVersion7();
+ var journal = new ConvertJournal { JobId = again };
+ journal.Made(id, Guid.CreateVersion7());
+ journal.Write(fx.Paths);
+ var gone = (await RunHereAsync(fx, plan, again)).Sequences.Single();
+ Assert.Equal(ConvertOutcome.Changed, gone.Outcome);
+ Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(again, id)));
+ Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
+ }
+
+ /// The photos a New sequence job filed into its copy before it stopped take no space again: its retry finishes on a full volume.
+ [Test]
+ public async Task ARetryOnceTheCopyWasFiledNeedsNoSpaceForItsPhotosAgain()
+ {
+ long free = 1L << 50;
+ await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free });
+ var id = await UploadedAsync(fx, 2);
+ var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
+ var jobId = Guid.CreateVersion7();
+ var left = new Dictionary();
+ var callbacks = Callbacks(done: _ =>
+ {
+ foreach (var file in new[] { ConfirmedPhotos.PathFor(fx.Paths, jobId), ConvertJournal.PinsPathFor(fx.Paths, jobId), ConvertJournal.StagedPathFor(fx.Paths, jobId) })
+ left[file] = File.ReadAllBytes(file);
+ });
+ var copy = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single().NewSequenceId;
+ foreach (var (file, bytes) in left) File.WriteAllBytes(file, bytes);
+ free = 1_000;
+
+ var retried = (await RunHereAsync(fx, plan, jobId)).Sequences.Single();
+ Assert.Equal((ConvertOutcome.Created, copy), (retried.Outcome, retried.NewSequenceId));
+ Assert.Count(2, await FramesAsync(fx, copy!.Value));
+ }
+
+ ///
+ /// A conversion into new sequences retried after another conversion, confirmed later, copied the same source in the same
+ /// format: it makes no second copy, and is done with that sequence.
+ ///
+ [Test]
+ public async Task ARetriedConversionMakesNoSecondCopyOfASourceAnotherConversionCopiedSince()
+ {
+ await using var fx = await FixtureAsync();
+ var id = await UploadedAsync(fx, 2);
+ var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle;
+ var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
+ // Job A finds the sequence busy (a render works on it), and fails.
+ var a = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-10));
+ var render = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, TargetIds = [id], State = JobState.Running, Created = DateTimeOffset.UtcNow };
+ await fx.Db.Jobs.EnqueueAsync(render, fx.Ct);
+ Assert.Equal(ConvertOutcome.Blocked, (await RunHereAsync(fx, plan, a.Id, confirmed: a.Created)).Sequences.Single().Outcome);
+ await fx.Db.Jobs.SetStateAsync(render.Id, JobState.Done, null, fx.Ct);
+ await fx.Db.Jobs.SetStateAsync(a.Id, JobState.Failed, "kept", fx.Ct);
+ // The user plans again: job B makes the copy. A JPEG copy made meanwhile is another format, which does not count.
+ var again = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
+ var b = await QueuedAsync(fx, again, DateTimeOffset.UtcNow.AddMinutes(-5));
+ var made = (await RunHereAsync(fx, again, b.Id, confirmed: b.Created)).Sequences.Single();
+ Assert.Equal(ConvertOutcome.Created, made.Outcome);
+ await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Done, null, fx.Ct);
+ var sequences = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count;
+
+ // Retry on A: done with the sequence, no second copy.
+ var lines = new List();
+ var retried = (await RunHereAsync(fx, plan, a.Id, Callbacks() with { Log = lines.Add }, a.Created)).Sequences.Single();
+ Assert.Equal((ConvertOutcome.Superseded, true), (retried.Outcome, retried.Final));
+ Assert.SequenceEqual([Loc.L("Archive.Log.ConvertSuperseded", title)], lines);
+ Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(a.Id, id)));
+ Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
+ Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, a.Id));
+
+ // A conversion confirmed after that copy was made is the user asking again: it makes its copy.
+ var latest = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
+ var later = await QueuedAsync(fx, latest, DateTimeOffset.UtcNow);
+ Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, latest, later.Id, confirmed: later.Created)).Sequences.Single().Outcome);
+
+ // A copy in another format is no copy of this one: a TIFF conversion confirmed before a DNG copy was made makes its own.
+ var other = await UploadedAsync(fx, 2, "upload-2", seed: 300);
+ var tiff = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, other);
+ var confirmedFirst = DateTimeOffset.UtcNow.AddMinutes(-1);
+ var dng = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, other);
+ Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, dng, Guid.CreateVersion7(), confirmed: DateTimeOffset.UtcNow)).Sequences.Single().Outcome);
+ Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, tiff, Guid.CreateVersion7(), confirmed: confirmedFirst)).Sequences.Single().Outcome);
}
}
diff --git a/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs b/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs
index fec31f7c..d07851da 100644
--- a/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs
+++ b/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs
@@ -103,6 +103,56 @@ public abstract class JobRepositoryTests(TestContext context) : RepositoryTest(c
return job;
}
+ [Test]
+ public async Task HoldsNameTheTargetsOfRunningAndPausedJobsAndOnlyAVerifysParameters()
+ {
+ // What a dry run counts as busy, read at every one and at every sequence a conversion comes to: never the parameters
+ // a conversion or an import keeps, nor the row of the job asking.
+ var db = await OpenChronosAsync();
+ Guid a = Guid.CreateVersion7(), b = Guid.CreateVersion7(), archive = Guid.CreateVersion7();
+ var convert = await EnqueueAsync(db, TestData.Job(JobKind.Convert) with { TargetIds = [a, b], ParametersJson = "{\"fingerprint\": \"f\"}", State = JobState.Running });
+ var verify = await EnqueueAsync(db, TestData.Job(JobKind.Verify) with { TargetIds = [archive], ParametersJson = "{\"sequenceIds\": []}", State = JobState.Paused });
+ await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [a] });
+ await EnqueueAsync(db, TestData.Job(JobKind.Scan) with { TargetIds = [archive], State = JobState.Done, Finished = TestData.Now });
+
+ var holds = await db.Jobs.ListHoldsAsync(cancellationToken: Ct);
+ Assert.SequenceEqual(new[] { convert.Id, verify.Id }.Order(), holds.Select(h => h.Id).Order());
+ var running = holds.Single(h => h.Id == convert.Id);
+ Assert.Equal((JobKind.Convert, (string?)null), (running.Kind, running.ParametersJson));
+ Assert.SequenceEqual([a, b], running.TargetIds);
+ var paused = holds.Single(h => h.Id == verify.Id);
+ Assert.Equal((JobKind.Verify, verify.ParametersJson), (paused.Kind, paused.ParametersJson));
+ Assert.SequenceEqual([verify.Id], (await db.Jobs.ListHoldsAsync(convert.Id, Ct)).Select(h => h.Id));
+
+ Assert.Equal(convert.Created, await db.Jobs.GetCreatedAsync(convert.Id, Ct));
+ Assert.Null(await db.Jobs.GetCreatedAsync(Guid.CreateVersion7(), Ct));
+ }
+
+ [Test]
+ public async Task BriefListsReadEveryFieldButTheParameters()
+ {
+ var db = await OpenChronosAsync();
+ var job = TestData.Job(JobKind.Convert, JobPriority.User) with
+ {
+ TargetIds = [Guid.CreateVersion7(), Guid.CreateVersion7()],
+ TargetLabel = "Harbour sunset",
+ ParametersJson = "{\"fingerprint\": \"f\"}",
+ LogTail = "line 1",
+ CreatedBy = Guid.CreateVersion7(),
+ ScheduleId = Guid.CreateVersion7(),
+ };
+ await db.Jobs.EnqueueAsync(job, Ct);
+
+ var brief = Assert.NotNull((await db.Jobs.ListBriefAsync(JobState.Queued, JobKind.Convert, 10, null, Ct)).Items.SingleOrDefault());
+ Assert.SequenceEqual(job.TargetIds, brief.TargetIds);
+ Assert.Equal(job with { TargetIds = brief.TargetIds, ParametersJson = null }, brief);
+ Assert.Equal(job.ParametersJson, (await db.Jobs.ListAsync(JobState.Queued, JobKind.Convert, 10, null, Ct)).Items.Single().ParametersJson);
+ Assert.Empty((await db.Jobs.ListBriefAsync(JobState.Running, null, 10, null, Ct)).Items);
+ var next = Assert.NotNull((await db.Jobs.ListQueuedBriefAsync([JobKind.Convert], 10, Chronos.Data.Jobs.JobPriorityQueue.Off, Ct)).SingleOrDefault());
+ Assert.Equal((job.Id, (string?)null), (next.Id, next.ParametersJson));
+ Assert.Empty(await db.Jobs.ListQueuedBriefAsync([], 10, Chronos.Data.Jobs.JobPriorityQueue.Off, Ct));
+ }
+
[Test]
public async Task ThePriorityQueueClaimsTheHigherKindsFirst()
{