diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index 39be0d17..bba3d8b3 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -224,6 +224,7 @@ Left as it was: {0}. {1}long: job log line; {0} sequence title, {1} what failed and why (sentences) {0}: photos not converted, left in the source alone: {1}long: job log line after a conversion into a new sequence; {0} sequence title, {1} photos Nothing to convert: {0}long: job log line; {0} sequence title + {0}: another conversion made its copy in this format since this one was confirmed; no second copy is made.long: job log line of a conversion into new sequences; {0} sequence title {0}: photos replaced by their conversion: {1}long: job log line; {0} sequence title, {1} photos {0}: photos converted into a new sequence: {1}long: job log line; {0} sequence title, {1} photos Dry run: nothing was changed.long: job log line diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 91523868..cc064662 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -218,6 +218,7 @@ Lasciata com'era: {0}. {1} {0}: foto non convertite, rimaste solo nella sorgente: {1} Niente da convertire: {0} + {0}: dopo la conferma di questa conversione, un'altra ne ha già creato la copia in questo formato; non ne viene creata una seconda. {0}: foto sostituite dalla loro conversione: {1} {0}: foto convertite in una nuova sequenza: {1} Simulazione: nulla è stato modificato. diff --git a/src/Chronos.Data/Archiving/ArchiveService.cs b/src/Chronos.Data/Archiving/ArchiveService.cs index ed69f220..6fe247f5 100644 --- a/src/Chronos.Data/Archiving/ArchiveService.cs +++ b/src/Chronos.Data/Archiving/ArchiveService.cs @@ -697,29 +697,35 @@ public static class ArchiveJobs /// /// : applies the conversion () - /// as confirmed (owner review 2, R7): the sequences it converts, in its order with their frames versions, the photos each - /// converts and their digest (), are the job's targets, so jobs that would touch - /// them wait. The job converts those photos alone, and leaves as it is and reports a sequence where one of them no longer - /// holds the content confirmed when the job comes to it. Queue it through , which - /// refuses a second conversion of the same sequences. + /// as confirmed (owner review 2, R7): the sequences it converts, in its order with their frames versions and the digest + /// of the photos each converts (), are the job's targets, so jobs that would touch + /// them wait. The request names the job's id, under which (the way to queue it, + /// which refuses a second conversion of the same sequences) writes which photos those are beside the job's journal + /// (). The job converts those photos alone, and leaves as it is and reports a sequence where + /// one of them no longer holds the content confirmed when the job comes to it. /// /// The plan converts nothing. public static ArchiveJobRequest Convert(Archive archive, ConvertPlan plan) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(plan); - var sequences = plan.Converted - .Select(s => new ConvertSequence(s.SequenceId, s.FramesVersion, s.Digest(), ConvertSequence.EncodePhotos(s.Frames.Select(f => f.FrameId)))) - .ToList(); + var sequences = plan.Converted.Select(s => new ConvertSequence(s.SequenceId, s.FramesVersion, s.Digest())).ToList(); if (sequences.Count == 0) throw new ArgumentException("The plan converts nothing.", nameof(plan)); var label = sequences.Count == 1 ? plan.Converted.First().Title : archive.Name; return new(JobKind.Convert, [.. sequences.Select(s => s.Id)], label, Json(new ConvertJobParameters(archive.Id, plan.Options.Target, plan.Options.Quality, plan.Mode, sequences, plan.Fingerprint()), - ArchiveJobJson.Default.ConvertJobParameters), JobPriority.User); + ArchiveJobJson.Default.ConvertJobParameters), JobPriority.User, Guid.CreateVersion7()); } - public static Task EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) => - engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken); + /// Queues , under its own id when it names one (). + public static Task EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(engine); + ArgumentNullException.ThrowIfNull(request); + return request.Id is { } id + ? engine.EnqueueAsync(id, request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken) + : engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken); + } private static string Json(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo typeInfo) => JsonSerializer.Serialize(value, typeInfo); } diff --git a/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs b/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs index cd34d322..8a467a4a 100644 --- a/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs +++ b/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs @@ -56,55 +56,29 @@ public sealed record MoveSequence(Guid Id); /// (owner review 2, R7): targets = the sequences converted, so the jobs that would touch them /// wait; the archive, the target format and JPEG quality, the mode chosen for this run, the sequences of the confirmed plan /// in its order, each with what it converts, and the plan's fingerprint (). A sequence -/// whose photos to convert are no longer those confirmed is left as it is and reported. +/// whose photos to convert are no longer those confirmed is left as it is and reported. Which photos those are is not +/// here but in the job's own file beside its journal (), so that the row stays as small as the +/// number of sequences, whatever the number of photos, for everything that reads jobs. /// public sealed record ConvertJobParameters(Guid ArchiveId, ConvertTarget Target, int Quality, ConvertMode Mode, IReadOnlyList Sequences, string Fingerprint); /// -/// A sequence of a : the frames version it was planned at, the digest of the photos it -/// converts, each with its content (), and which photos those are (). -/// Under the sequence's lease, before it converts anything, the job takes those photos from its plan made again and goes -/// on only while they have the digest confirmed: photos that came in since (an upload while the job waited its turn) stay -/// as they are, and a photo the plan did not list (one another program held while it was made) is never converted. Null -/// in a job an earlier build queued, which checks the frames version only (no digest), or the whole plan's digest (no photos). +/// A sequence of a : the frames version it was planned at, and the digest of the photos it +/// converts, each with its content (). Under the sequence's lease, before it +/// converts anything, the job takes the photos confirmed () from its plan made again and goes +/// on only while they have that digest: photos that came in since (an upload while the job waited its turn) stay as they +/// are, and a photo the plan did not list (one another program held while it was made) is never converted. Without the +/// confirmed photos, the whole plan made again must have the digest. Null in a job an earlier build queued, which checks +/// the frames version only. /// -/// The frame ids of the photos converted, in the plan's order, as writes them. -public sealed record ConvertSequence(Guid Id, int FramesVersion, string? Frames = null, string? Photos = null) -{ - /// - /// Frame ids as a job's parameters hold them: their 16 bytes each, one after the other, in base64 (a whole-archive - /// conversion lists every photo it converts, so the job row stays about half the size of a list of ids). - /// - public static string EncodePhotos(IEnumerable ids) - { - ArgumentNullException.ThrowIfNull(ids); - IReadOnlyCollection frames = ids as IReadOnlyCollection ?? [.. ids]; - var bytes = new byte[frames.Count * 16]; - var at = 0; - foreach (var frame in frames) - { - frame.TryWriteBytes(bytes.AsSpan(at, 16)); - at += 16; - } - return Convert.ToBase64String(bytes); - } +public sealed record ConvertSequence(Guid Id, int FramesVersion, string? Frames = null); - /// The frame ids holds, in order; null when it holds none (a job an earlier build queued). - /// The text is not ids as writes them. - public IReadOnlyList? PhotoIds() - { - if (Photos is null) return null; - var bytes = Convert.FromBase64String(Photos); - if (bytes.Length % 16 != 0) throw new FormatException($"The photos of sequence {Id} are not frame ids."); - var ids = new Guid[bytes.Length / 16]; - for (var i = 0; i < ids.Length; i++) ids[i] = new Guid(bytes.AsSpan(i * 16, 16)); - return ids; - } -} - -/// A job ready to enqueue. -public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets, string? Label, string? ParametersJson, JobPriority Priority); +/// +/// A job ready to enqueue; , when set, is the id it is queued under (chosen first, for what the job +/// reads beside its journal: ), otherwise the engine gives it one. +/// +public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets, string? Label, string? ParametersJson, JobPriority Priority, Guid? Id = null); [JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)] [JsonSerializable(typeof(ScanParameters))] @@ -120,6 +94,7 @@ public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets [JsonSerializable(typeof(ConvertJournal))] [JsonSerializable(typeof(StagedLine))] [JsonSerializable(typeof(PinnedSequence))] +[JsonSerializable(typeof(ConfirmedLine))] [JsonSerializable(typeof(CardProfile))] internal sealed partial class ArchiveJobJson : JsonSerializerContext { diff --git a/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs b/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs index 6dc02824..c7026207 100644 --- a/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs +++ b/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs @@ -17,7 +17,9 @@ namespace Chronos.Data.Archiving.Jobs; /// one when the source had it, thumbnails and proxies) are queued as follow-ups, as soon as it is done. Sequences left as /// they were (changed after the plan, busy, a photo another program holds, a photo that failed) are logged and fail the /// job at the end, after the others are converted; a retry tries again those busy, held or failed, with what they staged, -/// and leaves one that changed after the plan as it is (its log line says to plan again). +/// and leaves one that changed after the plan as it is (its log line says to plan again). A retry of a New sequence +/// conversion finishes the copies it made and makes none of a source another conversion copied in the same format since +/// this job was queued (its creation time: when the plan was confirmed). /// internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter converter, Func? sequenceChanged) : IJobHandler { @@ -39,7 +41,8 @@ internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter conver var archive = await db.Archives.GetAsync(parameters.ArchiveId, context.Cancellation).ConfigureAwait(false) ?? throw new InvalidOperationException($"Convert job {context.Job.Id} does not name a known archive."); var name = context.Job.CreatedBy is { } user ? (await db.Users.GetAsync(user, context.Cancellation).ConfigureAwait(false))?.Username : null; - var result = await converter.RunAsync(archive, parameters, context.Job.Id, new ConvertRequester(name, context.Job.CreatedBy), new ConvertCallbacks( + var by = new ConvertRequester(name, context.Job.CreatedBy, context.Job.Created); + var result = await converter.RunAsync(archive, parameters, context.Job.Id, by, new ConvertCallbacks( (done, total) => context.ReportAsync(done, total), (done, total) => context.CheckpointAsync(Checkpoint, done, total), context.Log, TellAsync, requests => QueueAsync(context, requests)), context.Cancellation).ConfigureAwait(false); diff --git a/src/Chronos.Data/Archiving/MassConverter.cs b/src/Chronos.Data/Archiving/MassConverter.cs index 3a7f1ce1..1c1bb014 100644 --- a/src/Chronos.Data/Archiving/MassConverter.cs +++ b/src/Chronos.Data/Archiving/MassConverter.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Security.Cryptography; using System.Text; using System.Text.Json; @@ -29,6 +30,12 @@ public enum ConvertOutcome /// It had nothing to convert. Nothing, + /// + /// Done with, no copy made (): another conversion made its copy in this format since + /// this one was confirmed (a retry of an older conversion, or a duplicate queued beside it), so a second copy is not made. + /// + Superseded, + /// /// Left as it was: a photo it converts no longer holds the content confirmed (it changed, is gone, or no longer converts), /// or it is no longer in the archive. A retry finds the same: the user plans again. @@ -52,8 +59,12 @@ public enum ConvertOutcome public sealed record ConvertSequenceResult(Guid SequenceId, string Title, ConvertOutcome Outcome, int Frames, Guid? NewSequenceId = null, string? Error = null, int LeftOut = 0) { - /// Whether the sequence is done with: converted, made, or with nothing to convert. - public bool Final => Outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing; + /// Whether the sequence is done with: converted, made, with nothing to convert, or copied by another conversion since. + public bool Final => IsFinal(Outcome); + + /// Whether leaves a sequence done with (). + internal static bool IsFinal(ConvertOutcome? outcome) => + outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing or ConvertOutcome.Superseded; } /// What a conversion did, and the passes the sequences it changed or made need (queued as follow-ups as each one ended). @@ -75,8 +86,12 @@ public sealed record ConvertResult(IReadOnlyList Sequence internal sealed record ConvertCallbacks(Func Progress, Func Checkpoint, Action Log, Func? Done = null, Func, ValueTask>? Queue = null); -/// Who asked for a conversion: the user's name, kept in the trash records of a Replace, and id, the owner of a new sequence when its source has none. -internal sealed record ConvertRequester(string? Name, Guid? Id); +/// +/// Who asked for a conversion: the user's name, kept in the trash records of a Replace, and id, the owner of a new sequence +/// when its source has none; and when the plan was confirmed (the job queued): a copy another conversion made of a source +/// since then supersedes this one's (). Null: no copy made by another counts. +/// +internal sealed record ConvertRequester(string? Name, Guid? Id, DateTimeOffset? Confirmed = null); /// /// A conversion of sequences a confirmed plan converts is queued, running or paused already, so its job was not queued @@ -94,7 +109,7 @@ public sealed class ConversionQueuedException(IReadOnlyList sequenceIds) : /// the conversion writes, ), and a job applies the /// confirmed plan (queued by , which refuses a second conversion of the same sequences) one sequence /// at a time. Under the sequence's lease, its sidecar settled where the index has it then (), -/// the job plans it again and takes the photos confirmed (), going on only while each is +/// the job plans it again and takes the photos confirmed (), going on only while each is /// still planned with its content confirmed (); it pins them then (), /// so a photo that came in, was verified or could be opened again since the plan stays as it is, on a retry too. Stage A, /// without the lease: each photo is hashed against its record and converted (, several at once @@ -107,14 +122,19 @@ public sealed class ConversionQueuedException(IReadOnlyList sequenceIds) : /// files the converted copies into a new sequence through the archive ingest (session /// key convert:<job>:<source>, so a resume finds it again), titled "title (FORMAT)", owned as the source is, /// with the source's tags, rating, notes and project, the source left as it is (the photos the plan did not convert stay -/// there alone, as the job log says). A sequence whose photos changed, that is busy, or has a photo that fails is left as it -/// was and reported. +/// there alone, as the job log says). A retry or a resume finishes the copy it made already rather than make another, and +/// makes none of a source that another conversion copied in the same format since this one was confirmed +/// (). A sequence whose photos changed, that is busy, or has a photo that fails is +/// left as it was and reported. /// public sealed class MassConverter { /// What the frames of a sequence no longer have once their files were replaced (). private const SequenceStatus Uncovered = SequenceStatus.AnalyzedLight | SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt; + /// How the session key of a new sequence a conversion makes begins (). + private const string KeyPrefix = "convert:"; + /// Held while a confirmed conversion is checked against those pending and queued (): one at a time in this process. private static readonly SemaphoreSlim Queueing = new(1, 1); @@ -186,8 +206,8 @@ public sealed class MassConverter var busy = new HashSet(await _trash.BusyAsync(job, cancellationToken).ConfigureAwait(false)); if (job is not { } self) busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken).ConfigureAwait(false)); - else if (await _db.Jobs.GetAsync(self, cancellationToken).ConfigureAwait(false) is { } running) - busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, running.Created, cancellationToken).ConfigureAwait(false)); + else if (await _db.Jobs.GetCreatedAsync(self, cancellationToken).ConfigureAwait(false) is { } queued) + busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, queued, cancellationToken).ConfigureAwait(false)); return busy; } @@ -195,8 +215,11 @@ public sealed class MassConverter /// Queues the job of the confirmed () through , /// unless a conversion of a sequence it converts is queued, running or paused already: another dialog or tab, an answer /// lost and Apply pressed again, a dry run made before that one was queued or none at all. Checked and queued under one - /// lock of this process, so two confirmations at once queue one job. runs in the caller's - /// context (a desktop's tells the user on its UI thread). + /// lock of this process, so two confirmations at once queue one job. First, which photos it converts are written under + /// the job's id beside the journal it starts (); queues the + /// request under that id (), and runs in the caller's context (a desktop's tells the + /// user on its UI thread). When they cannot be written (the archive went away since its dry run), the job is queued all + /// the same and holds each sequence to its whole plan. /// /// A conversion of one of its sequences waits or runs; nothing was queued. /// The plan converts nothing. @@ -204,13 +227,25 @@ public sealed class MassConverter { ArgumentNullException.ThrowIfNull(enqueue); var request = ArchiveJobs.Convert(archive, plan); + var paths = new ArchivePaths(archive.RootPath); + var job = request.Id!.Value; // No ConfigureAwait(false) here: each wait comes back to the caller's context, where enqueue is called. await Queueing.WaitAsync(cancellationToken); try { var pending = await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken); if (request.Targets.Where(pending.Contains).ToList() is { Count: > 0 } queued) throw new ConversionQueuedException(queued); - await enqueue(request); + var written = await Task.Run(() => ConfirmedPhotos.TryWrite(paths, job, plan), cancellationToken); + try + { + await enqueue(request); + } + catch + { + // Not queued: no job will read them. + if (written) ConfirmedPhotos.Delete(paths, job); + throw; + } } finally { @@ -260,6 +295,7 @@ public sealed class MassConverter var steps = parameters.Sequences.DistinctBy(s => s.Id).ToList(); var staged = journal.Staged(paths); var pins = journal.Pins(paths); + var confirmed = ConfirmedPhotos.Read(paths, jobId); // What is still to do: every sequence not final in an earlier attempt that is still in the archive, its sidecar settled // first when a writer a crash interrupted left a journal in its folder (this job's replacement among them). var rows = new Dictionary(); @@ -277,11 +313,15 @@ public sealed class MassConverter var unstaged = new Dictionary(); foreach (var p in plan.Sequences.Where(p => p.Blocked.Count == 0)) { - if (Confirmed(byId[p.SequenceId], p, pins.Of(p.SequenceId)) is not { Frames.Count: > 0 } converts) continue; + if (Confirmed(byId[p.SequenceId], confirmed.Of(p.SequenceId), p, pins.Of(p.SequenceId)) is not { Frames.Count: > 0 } converts) continue; progress.Count(converts.SequenceId, converts.Frames.Count, staged.Of(converts.SequenceId, converts.FramesVersion).Count); - unstaged[converts.SequenceId] = Unstaged(paths, jobId, converts, staged); + // What a new sequence this job made holds already takes no space again. + var filed = parameters.Mode == ConvertMode.NewSequence + ? await FiledAsync(archive, SessionKey(jobId, converts.SequenceId), cancellationToken).ConfigureAwait(false) + : null; + unstaged[converts.SequenceId] = Unstaged(paths, jobId, converts, staged, filed); } - var space = new SpaceWatch(this, paths); + var space = new SpaceWatch(() => FreeSpace(paths)); EnsureSpace(unstaged.Values.Sum(), space); await callbacks.Progress(progress.Done, progress.Total).ConfigureAwait(false); @@ -309,8 +349,8 @@ public sealed class MassConverter // The space for what is left, measured again: the volume may have filled since, and the files staged so far tell // how far off the estimate is. EnsureSpace(unstaged.Values.Sum(), space); - done = await ConvertOneAsync(archive, paths, step, row, options, parameters.Mode, jobId, by, journal, pins, staged, progress, space, callbacks, - cancellationToken).ConfigureAwait(false); + done = await ConvertOneAsync(archive, paths, step, row, options, parameters.Mode, jobId, by, journal, confirmed, pins, staged, progress, space, + callbacks, cancellationToken).ConfigureAwait(false); } unstaged.Remove(step.Id); var result = done.Result; @@ -349,16 +389,37 @@ public sealed class MassConverter /// /// One sequence: under its lease, its sidecar settled, planned again and held to what was confirmed (the photos pinned the - /// first time); then stage A without the lease, and stage B. + /// first time); then stage A without the lease, and stage B. A New sequence that has its copy already is not converted + /// again (). /// private async Task ConvertOneAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options, - ConvertMode mode, Guid jobId, ConvertRequester by, ConvertJournal journal, PinLog pins, StagedLog staged, Progress progress, SpaceWatch space, - ConvertCallbacks callbacks, CancellationToken cancellationToken) + ConvertMode mode, Guid jobId, ConvertRequester by, ConvertJournal journal, ConfirmedPhotos confirmedPhotos, PinLog pins, StagedLog staged, + Progress progress, SpaceWatch space, ConvertCallbacks callbacks, CancellationToken cancellationToken) { // Another job holds it now (a Reorganize paused since the job started, a render, a conversion of it queued first): it // waits for the retry. var busy = await HeldAsync(jobId, cancellationToken).ConfigureAwait(false); if (busy.Contains(step.Id) || busy.Contains(archive.Id)) return new StepDone(Result(row, ConvertOutcome.Blocked)); + var key = SessionKey(jobId, row.Id); + if (mode == ConvertMode.NewSequence) + { + try + { + if (await CopiedAsync(archive, paths, step, row, options, jobId, by, journal, pins, staged, key, progress, cancellationToken).ConfigureAwait(false) + is { } copied) + return copied; + } + catch (ArchiveOfflineException) + { + throw; + } + catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException) + { + Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the copy this conversion made of it could not be finished yet", ex); + return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex))); + } + } + var photos = confirmedPhotos.Of(step.Id); SequencePaths folder; SequenceManifest manifest; ConvertSequencePlan planned; @@ -379,7 +440,7 @@ public sealed class MassConverter manifest = SequenceWriter.ReadLocked(folder, row.Id); FrameTrash.EnsureReplaced(folder); var pin = pins.Of(step.Id); - if (mode == ConvertMode.Replace && ReplacedEarlier(manifest, step, pin, staged.Of(step.Id, step.FramesVersion), options) is { } frames) + if (mode == ConvertMode.Replace && ReplacedEarlier(manifest, step, photos, pin, staged.Of(step.Id, step.FramesVersion), options) is { } frames) { // Replaced by this job already: a crash after its stage B, before the journal said so (the settle may just have // completed it). @@ -394,11 +455,11 @@ public sealed class MassConverter IReadOnlySet files = new HashSet(FileRenames.ListPresent(at, sidecar), StringComparer.OrdinalIgnoreCase); return (ConvertPlanner.Plan(sidecar, at, files, options, decoder, title), files); }, cancellationToken).ConfigureAwait(false); - if (Confirmed(step, fresh, pin) is not { } confirmed) + if (Confirmed(step, photos, fresh, pin) is not { } confirmed) { // A photo it converts that another program holds now cannot be told from one that changed: the retry, once it // is free, can. - if (await Task.Run(() => Held(step, pin, sidecar, at, present, fresh, options, decoder), cancellationToken).ConfigureAwait(false)) + if (await Task.Run(() => Held(photos, pin, sidecar, at, present, fresh, options, decoder), cancellationToken).ConfigureAwait(false)) { Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: a photo it converts cannot be opened now; it is not converted"); return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.PhotoHeld"))); @@ -426,7 +487,6 @@ public sealed class MassConverter return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex))); } var staging = StagingOf(paths, jobId, row.Id); - var key = SessionKey(jobId, row.Id); // A new sequence a resumed job finds again holds the photos it filed before it stopped. var filed = mode == ConvertMode.NewSequence ? await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false) @@ -443,7 +503,7 @@ public sealed class MassConverter { return mode == ConvertMode.Replace ? await ReplaceAsync(archive, paths, row, planned, ready, staging, options, jobId, by.Name, cancellationToken).ConfigureAwait(false) - : await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, cancellationToken).ConfigureAwait(false); + : await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, journal, cancellationToken).ConfigureAwait(false); } catch (LeaseUnavailableException ex) { @@ -479,15 +539,97 @@ public sealed class MassConverter }; /// - /// What the job converts of a sequence whose plan is now : the photos it pinned when it first came to - /// it, each still planned with the content it had then; the first time, the photos confirmed () - /// while each is still planned with the content confirmed (the step's digest). Photos that came in, were verified or could - /// be opened again since stay as they are, so an upload while the job waits its turn neither changes what it converts - /// nor holds it back. A job an earlier build queued: the whole plan while it has the step's digest, or its frames - /// version. Null when a photo confirmed no longer converts as it was: the sequence changed after the plan, and the user - /// plans again. + /// What a conversion does with a source that has its copy already when the job comes + /// to it (a retry, a resume): the copy this job made (its journal names it, or its session key when a crash came before + /// the journal did) is finished, whatever changed in the source since, once it holds every photo the job pinned, and one + /// gone since is not made again without a new plan; a copy in this format that another conversion made since this one was + /// confirmed () supersedes this one's. Null when neither: the sequence converts, + /// and a copy holding only some of its photos (a crash while they were filed) gets the rest. /// - internal static ConvertSequencePlan? Confirmed(ConvertSequence step, ConvertSequencePlan fresh, PinnedSequence? pin) + private async Task CopiedAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options, Guid jobId, + ConvertRequester by, ConvertJournal journal, PinLog pins, StagedLog staged, string key, Progress progress, CancellationToken cancellationToken) + { + if ((journal.Of(step.Id)?.NewSequenceId ?? await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, cancellationToken).ConfigureAwait(false)) is { } made) + { + if (await _db.Sequences.GetAsync(made, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id) + { + Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it is gone; no other is made without a new plan"); + return new StepDone(Result(row, ConvertOutcome.Changed)); + } + return pins.Of(step.Id) is { } pin + ? await FinishCopyAsync(archive, paths, row, pin, staged, key, made, progress, cancellationToken).ConfigureAwait(false) + : null; + } + if (by.Confirmed is not { } confirmed + || await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } other) + return null; + Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed; no second copy is made"); + progress.Count(row.Id, 0, 0); + return new StepDone(Result(row, ConvertOutcome.Superseded)); + } + + /// + /// The copy this job made of a source, finished when it holds the conversion of every photo pinned (a crash came after + /// they were filed, before the journal said so): it takes the source's tags, rating, notes and project as stage B gives + /// them, whatever became of the photos of the source since. Null while a photo pinned is not in it, or when the source + /// is no longer where this index has it: the job converts the sequence, which files the rest. + /// + private async Task FinishCopyAsync(Archive archive, ArchivePaths paths, Sequence row, PinnedSequence pin, StagedLog staged, string key, Guid made, + Progress progress, CancellationToken cancellationToken) + { + var conversions = staged.Of(row.Id, pin.FramesVersion).ToDictionary(c => c.FrameId); + var filed = await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false); + var ready = new Dictionary(); + foreach (var photo in pin.Frames) + { + if (!conversions.TryGetValue(photo.Id, out var conversion) || !string.Equals(conversion.Source, photo.Sha256, StringComparison.OrdinalIgnoreCase) + || !filed.Contains(conversion.Sha256)) + return null; + ready[photo.Id] = conversion; + } + SequenceManifest manifest; + string? project; + await using (await _writer.LockAsync(paths, row.Id, cancellationToken).ConfigureAwait(false)) + { + if (await _db.Sequences.GetAsync(row.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id) return null; + row = now; + var folder = paths.Sequence(row.Folder); + if (!SequenceFolders.Holds(folder, row.Id)) return null; + manifest = SequenceWriter.ReadLocked(folder, row.Id); + project = row.SettingsJson ?? (File.Exists(folder.ProjectPath) ? AtomicFile.ReadAllText(folder.ProjectPath) : null); + } + Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it holds every photo already; it is finished, no other is made"); + progress.Count(row.Id, pin.Frames.Count, pin.Frames.Count); + return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, made, pin.Frames.Count, cancellationToken).ConfigureAwait(false); + } + + /// + /// A copy of in this conversion's format that another conversion made since : + /// a sequence filed under another job's session key for that source () whose sidecar says it was + /// made since then and holds photos of that format. Null when there is none. + /// + private async Task LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed, + CancellationToken cancellationToken) + { + foreach (var (key, id) in await _db.Sequences.ListSessionKeysAsync(archive.Id, $"{KeyPrefix}%:{source:N}", cancellationToken).ConfigureAwait(false)) + { + if (JobOfKey(key) is not { } other || other == jobId) continue; + if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id) continue; + if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format)) return id; + } + return null; + } + + /// + /// What the job converts of a sequence whose plan is now : the photos it pinned when it first came to + /// it, each still planned with the content it had then; the first time, the photos confirmed (, + /// ) while each is still planned with the content confirmed (the step's digest). Photos that + /// came in, were verified or could be opened again since stay as they are, so an upload while the job waits its turn + /// neither changes what it converts nor holds it back. Without the photos confirmed (they could not be written, or a job + /// an earlier build queued): the whole plan while it has the step's digest, or its frames version. Null when a photo + /// confirmed no longer converts as it was: the sequence changed after the plan, and the user plans again. + /// + internal static ConvertSequencePlan? Confirmed(ConvertSequence step, IReadOnlyList? photos, ConvertSequencePlan fresh, PinnedSequence? pin) { var now = fresh.Frames.ToDictionary(f => f.FrameId); if (pin is not null) @@ -500,7 +642,7 @@ public sealed class MassConverter } return frames.Count > 0 ? fresh with { FramesVersion = pin.FramesVersion, Frames = frames } : null; } - if (step is { Frames: { } confirmedDigest } && step.PhotoIds() is { Count: > 0 } photos) + if (step is { Frames: { } confirmedDigest } && photos is { Count: > 0 }) { var frames = new List(photos.Count); foreach (var id in photos) @@ -520,15 +662,15 @@ public sealed class MassConverter /// /// Whether the plan made now () leaves out a photo the job converts of the sequence (one it pinned, - /// else one confirmed) only because its file could not be opened (: another program - /// holds it): the sequence then waits for the retry rather than counting as changed. Photos the job does not convert do - /// not count, nor does a confirmed photo that changed meanwhile. A job an earlier build queued, which names no photos: - /// any photo of the sequence that could not be opened. + /// else one of , those confirmed) only because its file could not be opened + /// (: another program holds it): the sequence then waits for the retry rather than + /// counting as changed. Photos the job does not convert do not count, nor does a confirmed photo that changed meanwhile. + /// Without the photos confirmed: any photo of the sequence that could not be opened. /// - private static bool Held(ConvertSequence step, PinnedSequence? pin, SequenceManifest manifest, SequencePaths folder, IReadOnlySet present, + private static bool Held(IReadOnlyList? photos, PinnedSequence? pin, SequenceManifest manifest, SequencePaths folder, IReadOnlySet present, ConvertSequencePlan fresh, ConvertOptions options, ImageDecoder decoder) { - IReadOnlyList? converts = pin is not null ? [.. pin.Frames.Select(f => f.Id)] : step.PhotoIds(); + IReadOnlyList? converts = pin is not null ? [.. pin.Frames.Select(f => f.Id)] : photos; if (converts is null) return fresh.Skipped.Any(s => s.Reason == ConvertSkip.Inaccessible); var planned = fresh.Frames.Select(f => f.FrameId).ToHashSet(); var frames = manifest.Frames.ToDictionary(f => f.Id); @@ -579,12 +721,13 @@ public sealed class MassConverter /// /// Stage A: converts the planned frames not staged yet, several at once, each staged file appended to the journal as it - /// is consumed (in frame order). A photo starts only while the volume has the space it and the photos after it that may - /// convert at once with it take (as many as the host converts at once, never photos that are not there to start), with - /// the reserve: short of it the job stops (Archive.Error.ConvertSpace) before the volume fills. The space the - /// photos converting already take is in the volume's free space as they write it, so it is not counted twice. Returns - /// the first failure (the file and why, localised; the exception goes to the diagnostics log); the frames converted - /// meanwhile stay staged for a retry. + /// is consumed (in frame order). A photo starts only while the volume has the space for it and for every photo handed to + /// the work before it whose file has not landed yet (: converting or waiting for its turn, + /// up to twice as many as convert at once, at the end of a sequence as anywhere), with the margin and the reserve: short + /// of it the job stops (Archive.Error.ConvertSpace) before the volume fills. A converted file is encoded in memory + /// and lands whole, by a rename, before it is verified: what those photos will write is not in the free space measured + /// yet, and what landed is, so each counts until its file lands, never twice. Returns the first failure (the file and why, + /// localised; the exception goes to the diagnostics log); the frames converted meanwhile stay staged for a retry. /// /// The volume lacks the space for the next photo; what was staged stays. private async Task StageAsync(Sequence row, SequenceManifest manifest, SequencePaths folder, ConvertSequencePlan planned, @@ -622,13 +765,26 @@ public sealed class MassConverter }, cancellationToken); for (var i = 0; i < todo.Count && failure is null; i++) { - if (space.Short(Ahead(todo, i, atOnce)) is { } message) + var frame = frames[todo[i].FrameId]; + var target = Path.Combine(staging, frame.Id.ToString("N") + options.Extension); + if (space.Start(todo[i].EstimatedBytes, target) is { } message) { full = message; break; } - var frame = frames[todo[i].FrameId]; - await work.AddAsync(token => Stage(frame, manifest, folder, staging, options, decoder, threads, token)).ConfigureAwait(false); + // A photo that never starts (the job stops: cancelled, or a write of the journal failed) is never counted off, + // and the watch goes with the job. + await work.AddAsync(token => + { + try + { + return Stage(frame, manifest, folder, target, options, decoder, threads, token); + } + finally + { + space.Ended(target); + } + }).ConfigureAwait(false); } await work.CompleteAsync().ConfigureAwait(false); if (full is not null) @@ -640,20 +796,8 @@ public sealed class MassConverter return failure; } - /// - /// The bytes photo of and the photos after it that may convert at once with - /// it are estimated at: of them, fewer near the end (the space a photo of stage A starts with). - /// - internal static long Ahead(IReadOnlyList todo, int index, int atOnce) - { - ArgumentNullException.ThrowIfNull(todo); - long bytes = 0; - for (var i = index; i < Math.Min(todo.Count, index + Math.Max(1, atOnce)); i++) bytes += todo[i].EstimatedBytes; - return bytes; - } - - /// One photo of stage A: hashed against its record, converted into the staging folder and verified. - private static Staged Stage(SidecarFrame frame, SequenceManifest manifest, SequencePaths folder, string staging, ConvertOptions options, ImageDecoder decoder, + /// One photo of stage A: hashed against its record, converted into (in the staging folder) and verified. + private static Staged Stage(SidecarFrame frame, SequenceManifest manifest, SequencePaths folder, string target, ConvertOptions options, ImageDecoder decoder, int threads, CancellationToken cancellationToken) { var source = folder.File(frame.File); @@ -666,7 +810,6 @@ public sealed class MassConverter if (!string.Equals(sha, frame.Sha256, StringComparison.OrdinalIgnoreCase)) throw new ConversionException(ConversionFailure.Changed, $"{frame.File} no longer matches its SHA-256; verify the sequence"); cancellationToken.ThrowIfCancellationRequested(); - var target = Path.Combine(staging, frame.Id.ToString("N") + options.Extension); var file = FrameConverter.Convert(new ConvertSource(source, frame, manifest.Camera, manifest.Lens), options, decoder, target, threads, cancellationToken); return new Staged(frame.File, new StagedConversion(frame.Id, sha, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height), null); } @@ -734,9 +877,12 @@ public sealed class MassConverter /// source's capture facts; then the source's tags, rating, notes and project (its positions translated to the new frames). /// Every conversion not filed yet must still be staged whole: one gone since (a clean-up, a quarantine) fails the sequence /// rather than leave its photo out. The photos the plan did not convert are not in the new sequence (). + /// Once its photos are filed, the journal names the new sequence (), so that a retry or a + /// resume finishes it and never makes a second one. /// private async Task CreateAsync(Archive archive, ArchivePaths paths, Sequence row, ConvertSequencePlan planned, - Dictionary ready, string staging, ConvertOptions options, string key, Guid? requestedBy, CancellationToken cancellationToken) + Dictionary ready, string staging, ConvertOptions options, string key, Guid? requestedBy, ConvertJournal journal, + CancellationToken cancellationToken) { SequenceManifest manifest; string? project; @@ -775,13 +921,26 @@ public sealed class MassConverter await _ingest.CommitSessionAsync(archive, new SequenceTarget(key, title, null, owner), pending, CancellationToken.None).ConfigureAwait(false); var id = await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, CancellationToken.None).ConfigureAwait(false) ?? throw new IOException($"No converted photo of {row.DisplayTitle} could be filed into a new sequence."); + journal.Made(row.Id, id); + journal.Write(paths); + return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, id, planned.Frames.Count, cancellationToken).ConfigureAwait(false); + } + + /// + /// The end of stage B of , once the new sequence holds the + /// photos converted: the source's tags, rating, notes and project given to it (), + /// and the passes it needs, from what the source has. The photos of the source it does not hold are counted as left out. + /// + private async Task CompleteCopyAsync(Archive archive, ArchivePaths paths, Sequence row, SequenceManifest manifest, string? project, + Dictionary ready, Guid id, int frames, CancellationToken cancellationToken) + { await CopyAsync(paths, row, manifest, project, ready, id, cancellationToken).ConfigureAwait(false); var made = await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false) ?? throw new InvalidOperationException($"The new sequence {id} is not indexed."); - var leftOut = manifest.Frames.Count - planned.Frames.Count; - Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: {planned.Frames.Count} photo(s) converted into the new sequence {made.DisplayTitle}, {leftOut} left in the source alone"); + var leftOut = manifest.Frames.Count(f => !ready.ContainsKey(f.Id)); + Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: {frames} photo(s) converted into the new sequence {made.DisplayTitle}, {leftOut} left in the source alone"); var had = row.Status & (SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt); - return new StepDone(new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Created, planned.Frames.Count, id, LeftOut: leftOut), had, + return new StepDone(new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Created, frames, id, LeftOut: leftOut), had, _frames.FollowUps(archive, made, had)); } @@ -855,13 +1014,13 @@ public sealed class MassConverter /// /// The photos converted, when this job replaced the sequence's frames already (a crash after its stage B, before the /// journal said so): every photo it pinned is a frame holding a file of the target's format converted from the content - /// pinned. Without a pin (its journal lost), every photo confirmed () is a frame - /// holding a conversion to the target whose sources have the step's digest; a job an earlier build queued: the frames - /// holding a conversion to the target are, by their sources, the photos of the step's digest, or every staged conversion - /// is a frame holding the converted content. Null otherwise. + /// pinned. Without a pin (its journal lost), every photo confirmed () is a frame holding a + /// conversion to the target whose sources have the step's digest; without those either: the frames holding a conversion + /// to the target are, by their sources, the photos of the step's digest, or (a job an earlier build queued) every staged + /// conversion is a frame holding the converted content. Null otherwise. /// - private static int? ReplacedEarlier(SequenceManifest manifest, ConvertSequence step, PinnedSequence? pin, IReadOnlyList noted, - ConvertOptions options) + private static int? ReplacedEarlier(SequenceManifest manifest, ConvertSequence step, IReadOnlyList? photos, PinnedSequence? pin, + IReadOnlyList noted, ConvertOptions options) { var frames = manifest.Frames.ToDictionary(f => f.Id); if (pin is not null) @@ -872,7 +1031,7 @@ public sealed class MassConverter ? pin.Frames.Count : null; } - if (step is { Frames: { } confirmed } && step.PhotoIds() is { Count: > 0 } photos) + if (step is { Frames: { } confirmed } && photos is { Count: > 0 }) { var sources = new List<(Guid, string?)>(photos.Count); foreach (var id in photos) @@ -987,11 +1146,16 @@ public sealed class MassConverter return null; } - /// The bytes the photos converts are estimated at, but for those whose staged file is there already. - private static long Unstaged(ArchivePaths paths, Guid jobId, ConvertSequencePlan plan, StagedLog staged) + /// + /// The bytes the photos converts are estimated at, but for those whose staged file is there already, + /// or which the new sequence this job made holds (, the SHA-256 of its photos). + /// + private static long Unstaged(ArchivePaths paths, Guid jobId, ConvertSequencePlan plan, StagedLog staged, IReadOnlySet? filed) { var folder = StagingOf(paths, jobId, plan.SequenceId); - var there = staged.Of(plan.SequenceId, plan.FramesVersion).Where(c => Present(Path.Combine(folder, c.File), c)).Select(c => c.FrameId).ToHashSet(); + var there = staged.Of(plan.SequenceId, plan.FramesVersion) + .Where(c => filed?.Contains(c.Sha256) == true || Present(Path.Combine(folder, c.File), c)) + .Select(c => c.FrameId).ToHashSet(); return plan.Frames.Where(f => !there.Contains(f.FrameId)).Sum(f => f.EstimatedBytes); } @@ -1042,6 +1206,7 @@ public sealed class MassConverter ConvertOutcome.Converted => Loc.L("Archive.Log.Converted", result.Title, result.Frames), ConvertOutcome.Created => Loc.L("Archive.Log.ConvertedNew", result.Title, result.Frames), ConvertOutcome.Nothing => Loc.L("Archive.Log.ConvertNothing", result.Title), + ConvertOutcome.Superseded => Loc.L("Archive.Log.ConvertSuperseded", result.Title), ConvertOutcome.Changed => Loc.L("Archive.Log.ConvertChanged", result.Title), // Busy, or why the retry may do better (a photo another program holds, a folder another index moved), in one sentence. ConvertOutcome.Blocked when result.Error is null => Loc.L("Archive.Log.ConvertBlocked", result.Title), @@ -1067,7 +1232,11 @@ public sealed class MassConverter : Loc.L("Common.Size.MB", (bytes / 1e6).ToString(digits, Loc.Culture)); /// The session key of the new sequence a conversion makes from . - internal static string SessionKey(Guid job, Guid source) => $"convert:{job:N}:{source:N}"; + internal static string SessionKey(Guid job, Guid source) => $"{KeyPrefix}{job:N}:{source:N}"; + + /// The conversion job a session key of names; null for any other key. + private static Guid? JobOfKey(string key) => + key.Split(':') is ["convert", var job, _] && Guid.TryParseExact(job, "N", out var id) ? id : null; /// The folder under .chronos/staging a conversion job stages into, one subfolder per sequence. internal static string StagingFolder(Guid job) => ConvertJournal.Prefix + job.ToString("N"); @@ -1112,34 +1281,68 @@ public sealed class MassConverter } /// - /// The free space a running conversion watches: measured where it writes, against estimates scaled by how far the files it - /// staged came out above theirs (photos of a 14-bit RAW carry more noise than the estimate's bits per pixel allow for). + /// The free space a running conversion watches: measured where it writes (), against estimates scaled + /// by how far the files it staged came out above theirs (photos of a 14-bit RAW carry more noise than the estimate's bits + /// per pixel allow for), and, in stage A, with the photos started whose converted file is not there yet (): + /// a converted file is encoded in memory and lands whole, by a rename, before its photo's work verifies it, so the free + /// space measured holds every file landed, and none of what the others will write. /// - private sealed class SpaceWatch(MassConverter converter, ArchivePaths paths) + internal sealed class SpaceWatch(Func free) { + private readonly ConcurrentDictionary _started = new(StringComparer.Ordinal); private long _actual; private long _estimated; /// The bytes staged over their estimates, at least 1. public double Ratio => _estimated > 0 ? Math.Max(1, (double)_actual / _estimated) : 1; + /// + /// The estimated bytes of the photos started () whose converted file has not landed yet: the file is + /// not at its path, or is still the one that was there when the photo started (one a crash left, which it replaces). + /// + public long Writing => _started.Sum(s => Landed(s.Key) == s.Value.Before ? s.Value.Estimate : 0); + public void Staged(long actual, long estimated) { _actual += actual; _estimated += Math.Max(1, estimated); } + /// + /// Why a photo estimated at bytes, converted into , cannot start now: + /// the volume lacks the space for it and for the photos started before it whose file has not landed yet + /// (, ), localized. Null when it can: it counts among those until its file lands + /// or its work ends (). + /// + public string? Start(long estimated, string target) + { + var before = Landed(target); + if (Short(Math.Max(0, estimated) + Writing) is { } message) return message; + _started[target] = new Starting(Math.Max(0, estimated), before); + return null; + } + + /// The work of the photo converted into ended: its file landed, or never will. + public void Ended(string target) => _started.TryRemove(target, out _); + /// /// Why the volume cannot take bytes more (scaled, with the margin and the reserve), localized; /// null when it can, or when its free space cannot be measured. /// public string? Short(long estimated) { - var space = new ConvertSpace(ConvertPlanner.Required((long)Math.Ceiling(estimated * Ratio)), converter.FreeSpace(paths)); + var space = new ConvertSpace(ConvertPlanner.Required((long)Math.Ceiling(estimated * Ratio)), free()); if (space.Enough) return null; - var (required, free) = Sizes(space.Required, space.Free ?? 0); - return Loc.L("Archive.Error.ConvertSpace", required, free); + var (required, left) = Sizes(space.Required, space.Free ?? 0); + return Loc.L("Archive.Error.ConvertSpace", required, left); } + + /// The file at as a landing changes it, its size and time of writing; null when there is none. + private static (long Size, long Written)? Landed(string path) => + new FileInfo(path) is { Exists: true } file ? (file.Length, file.LastWriteTimeUtc.Ticks) : null; + + /// A photo started: its estimate, and the file at its path then. + private sealed record Starting(long Estimate, (long Size, long Written)? Before); } /// What a sequence came to: its result, the statuses its follow-ups follow from, and those follow-ups (null: none to queue). @@ -1171,12 +1374,13 @@ internal sealed record PinnedFrame(Guid Id, string Sha256); /// /// The journal of a conversion job (.chronos/journal/convert-<job>.json, owner review 2, R7): what each sequence -/// came to, and whether the passes it needs were queued, written after each one; with it, the pin log -/// (convert-<job>.pins, ), the photos the job converts of each sequence, and the staged log -/// (convert-<job>.staged), one line per photo staged, both appended and flushed as they grow, so a job of thousands -/// of photos journals each once. A resumed or retried job keeps the sequences that are final, the photos pinned, and the -/// staged files whose hash still verifies. All of it goes when every sequence is final, or once the job is gone -/// (). +/// came to, the new sequence a New sequence conversion made of it as soon as its photos are filed, and whether the passes +/// it needs were queued, written after each one; with it, the photos confirmed (convert-<job>.photos, +/// , written when the job is queued), the pin log (convert-<job>.pins, +/// ), the photos the job converts of each sequence, and the staged log (convert-<job>.staged), +/// one line per photo staged, both appended and flushed as they grow, so a job of thousands of photos journals each once. A +/// resumed or retried job keeps the sequences that are final, the copies made, the photos pinned, and the staged files +/// whose hash still verifies. All of it goes when every sequence is final, or once the job is gone (). /// internal sealed class ConvertJournal { @@ -1198,7 +1402,10 @@ internal sealed class ConvertJournal noted.Had = had; } - /// Notes what a sequence came to and, when it was converted or made, the statuses its follow-ups follow from. + /// + /// Notes what a sequence came to and, when it was converted or made, the statuses its follow-ups follow from. The new + /// sequence a New sequence conversion made of it stays named whatever it came to since (). + /// public void Note(ConvertSequenceResult result, SequenceStatus? had = null) { var noted = Of(result.SequenceId); @@ -1206,10 +1413,21 @@ internal sealed class ConvertJournal noted.Title = result.Title; noted.Outcome = result.Outcome; noted.Frames = result.Frames; - noted.NewSequenceId = result.NewSequenceId; + noted.NewSequenceId = result.NewSequenceId ?? noted.NewSequenceId; if (had is { } status) noted.Had = status; } + /// + /// Notes the new sequence a New sequence conversion filed the photos of into, before anything else is + /// done with it: a retry or a resume finishes that one and makes no other. + /// + public void Made(Guid id, Guid newSequenceId) + { + var noted = Of(id); + if (noted is null) Sequences.Add(noted = new ConvertJournalSequence { Id = id }); + noted.NewSequenceId = newSequenceId; + } + public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.json"); public static string StagedPathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.staged"); @@ -1256,8 +1474,9 @@ internal sealed class ConvertJournal /// The pin log of this job, read now. public PinLog Pins(ArchivePaths paths) => PinLog.Open(PinsPathFor(paths, JobId)); - /// The journal files of , whichever exist. - public static IEnumerable Files(ArchivePaths paths, Guid job) => [StagedPathFor(paths, job), PinsPathFor(paths, job), PathFor(paths, job)]; + /// The journal files of , whichever exist: the photos confirmed among them (). + public static IEnumerable Files(ArchivePaths paths, Guid job) => + [StagedPathFor(paths, job), PinsPathFor(paths, job), ConfirmedPhotos.PathFor(paths, job), PathFor(paths, job)]; public static void Delete(ArchivePaths paths, Guid job) { @@ -1291,7 +1510,7 @@ internal sealed class ConvertJournalSequence public bool Queued { get; set; } [JsonIgnore] - public bool Final => Outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing; + public bool Final => ConvertSequenceResult.IsFinal(Outcome); } /// The staged log of a conversion job (): read once, appended to line by line. @@ -1380,6 +1599,114 @@ internal sealed class PinLog } } +/// A line of : a sequence, and the frame ids of the photos converted of it, 16 bytes each, in base64. +internal sealed record ConfirmedLine(Guid Sequence, string Photos); + +/// +/// The photos a conversion job converts of each sequence, as its plan was confirmed (.chronos/journal/convert-<job>.photos, +/// owner review 2, R7): written whole under the job's id before it is queued (), one +/// line per sequence with the frame ids in the plan's order. The job's row keeps each sequence's digest of those photos +/// with their content (), never the ids: it stays as small as the number of sequences +/// whatever the number of photos, for every list and check that reads jobs. Under each sequence's lease the job takes these +/// photos from its plan made again and goes on only while they have that digest; a sequence this file does not name (it +/// could not be written, or a job an earlier build queued) is held to its whole plan. It goes with the journal +/// (). +/// +internal sealed class ConfirmedPhotos +{ + private readonly Dictionary> _photos = []; + + public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{ConvertJournal.Prefix}{job:N}.photos"); + + /// The photos confirmed of , in the plan's order; null when the file does not name it. + public IReadOnlyList? Of(Guid sequence) => _photos.GetValueOrDefault(sequence); + + /// The photos confirmed for : none when the file is not there; a line that cannot be read names nothing. + public static ConfirmedPhotos Read(ArchivePaths paths, Guid job) + { + var read = new ConfirmedPhotos(); + foreach (var line in JournalLines.Read(PathFor(paths, job))) + { + try + { + if (JsonSerializer.Deserialize(line, ArchiveJobJson.Default.ConfirmedLine) is { } confirmed) read._photos[confirmed.Sequence] = Decode(confirmed.Photos); + } + catch (Exception ex) when (ex is JsonException or FormatException) + { + Core.Diagnostics.Log.Warning("archive", $"A line of the photos confirmed for conversion job {job} cannot be read; that sequence is held to its whole plan", ex); + } + } + return read; + } + + /// + /// Writes the photos converts of each sequence it converts, for the job , whole + /// (a reader finds all of it or none); false, logged, when it cannot be written. + /// + public static bool TryWrite(ArchivePaths paths, Guid job, ConvertPlan plan) + { + ArgumentNullException.ThrowIfNull(plan); + try + { + using var lines = new MemoryStream(); + foreach (var sequence in plan.Converted) + { + lines.Write(JsonSerializer.SerializeToUtf8Bytes(new ConfirmedLine(sequence.SequenceId, Encode(sequence.Frames.Select(f => f.FrameId))), + ArchiveJobJson.Default.ConfirmedLine)); + lines.WriteByte((byte)'\n'); + } + System.IO.Directory.CreateDirectory(paths.JournalDirectory); + AtomicFile.WriteAllBytes(PathFor(paths, job), lines.GetBuffer().AsSpan(0, (int)lines.Length)); + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be written; it holds each sequence to its whole plan", ex); + return false; + } + } + + /// Deletes the file of ; a failure is logged, and maintenance removes it later. + public static void Delete(ArchivePaths paths, Guid job) + { + try + { + File.Delete(PathFor(paths, job)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be deleted; maintenance removes them later", ex); + } + } + + /// Frame ids as a line holds them: their 16 bytes each, one after the other, in base64. + public static string Encode(IEnumerable ids) + { + ArgumentNullException.ThrowIfNull(ids); + IReadOnlyCollection frames = ids as IReadOnlyCollection ?? [.. ids]; + var bytes = new byte[frames.Count * 16]; + var at = 0; + foreach (var frame in frames) + { + frame.TryWriteBytes(bytes.AsSpan(at, 16)); + at += 16; + } + return Convert.ToBase64String(bytes); + } + + /// The frame ids wrote, in order. + /// The text is not frame ids as writes them. + public static IReadOnlyList Decode(string text) + { + ArgumentNullException.ThrowIfNull(text); + var bytes = Convert.FromBase64String(text); + if (bytes.Length % 16 != 0) throw new FormatException("The text is not frame ids."); + var ids = new Guid[bytes.Length / 16]; + for (var i = 0; i < ids.Length; i++) ids[i] = new Guid(bytes.AsSpan(i * 16, 16)); + return ids; + } +} + /// Files of JSON lines a job appends to as it goes: each line written whole and flushed to disk, read back skipping blank ones. internal static class JournalLines { diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs index 1c8b3cc5..7a9312ee 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.cs @@ -6,6 +6,7 @@ using Chronos.Core.IO; using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Repositories; namespace Chronos.Data.Archiving; @@ -115,7 +116,6 @@ public sealed partial class SequenceTrash public const string FolderName = "trash"; private const string RecordExtension = ".json"; - private const int JobPage = 200; /// Bytes of a sidecar read to find its id (written second, after the schema version). private const int SidecarHead = 4096; @@ -493,30 +493,23 @@ public sealed partial class SequenceTrash /// internal Task> BusyAsync(CancellationToken cancellationToken) => BusyAsync(null, cancellationToken); - /// What running and paused jobs hold, but the job (the one asking). + /// + /// What running and paused jobs hold, but the job (the one asking), read from their targets + /// alone (): nothing reads the parameters a conversion or an import keeps. + /// internal async Task> BusyAsync(Guid? except, CancellationToken cancellationToken) { var busy = new HashSet(); - foreach (var state in new[] { JobState.Running, JobState.Paused }) - { - string? cursor = null; - do - { - var page = await _db.Jobs.ListAsync(state, JobPage, cursor, cancellationToken).ConfigureAwait(false); - foreach (var job in page.Items.Where(j => j.Id != except)) busy.UnionWith(Holds(job)); - cursor = page.NextCursor; - } - while (cursor is not null); - } + foreach (var job in await _db.Jobs.ListHoldsAsync(except, cancellationToken).ConfigureAwait(false)) busy.UnionWith(Holds(job)); return busy; } - private static IReadOnlyList Holds(Job job) + private static IReadOnlyList Holds(JobHold job) { - if (job.Kind != JobKind.Verify) return job.TargetIds; + if (job is not { Kind: JobKind.Verify, ParametersJson: { Length: > 0 } json }) return job.TargetIds; try { - return ArchiveJobJson.Read(job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null)).SequenceIds ?? job.TargetIds; + return JsonSerializer.Deserialize(json, ArchiveJobJson.Default.VerifyParameters)?.SequenceIds ?? job.TargetIds; } catch (JsonException) { diff --git a/src/Chronos.Data/Jobs/JobEngine.cs b/src/Chronos.Data/Jobs/JobEngine.cs index e84eac82..e739aad8 100644 --- a/src/Chronos.Data/Jobs/JobEngine.cs +++ b/src/Chronos.Data/Jobs/JobEngine.cs @@ -122,6 +122,14 @@ public sealed class JobEngine : IAsyncDisposable Guid? createdBy, Guid? scheduleId, CancellationToken cancellationToken = default) => EnqueueAsync(Guid.CreateVersion7(), kind, targets, targetLabel, parametersJson, priority, createdBy, scheduleId, cancellationToken); + /// + /// Queues a job under the id , chosen before (what it reads beside its journal is written under that + /// id first, as a conversion's confirmed photos are), and wakes a worker. + /// + public Task EnqueueAsync(Guid id, JobKind kind, IReadOnlyList targets, string? targetLabel, string? parametersJson, JobPriority priority, + Guid? createdBy, CancellationToken cancellationToken = default) => + EnqueueAsync(id, kind, targets, targetLabel, parametersJson, priority, createdBy, null, cancellationToken); + /// /// True when this engine runs jobs of : it has a handler for it and /// does not leave it out. Known before , so a host that does not start the engine still answers. @@ -377,7 +385,7 @@ public sealed class JobEngine : IAsyncDisposable { var candidate = await AcceptedAsync(queue, null, cancellationToken).ConfigureAwait(false); if (candidate is null) return _others.Count > 0 ? await _db.Jobs.ClaimNextAsync(workerId, _others, queue, cancellationToken).ConfigureAwait(false) : null; - if (_others.Count > 0 && (await _db.Jobs.ListQueuedAsync(_others, 1, queue, cancellationToken).ConfigureAwait(false)).FirstOrDefault() is { } head + if (_others.Count > 0 && (await _db.Jobs.ListQueuedBriefAsync(_others, 1, queue, cancellationToken).ConfigureAwait(false)).FirstOrDefault() is { } head && queue.ClaimsBefore(head, candidate)) { if (await _db.Jobs.ClaimNextAsync(workerId, _others, queue, cancellationToken).ConfigureAwait(false) is { } next) return next; @@ -397,7 +405,8 @@ public sealed class JobEngine : IAsyncDisposable Job? head = null; if (_others.Count > 0) { - foreach (var queued in await _db.Jobs.ListQueuedAsync(_others, ReviewWindow, queue, cancellationToken).ConfigureAwait(false)) + // Only who comes first counts here: the job that yields to it reads its kind, and a claim reads a job whole. + foreach (var queued in await _db.Jobs.ListQueuedBriefAsync(_others, ReviewWindow, queue, cancellationToken).ConfigureAwait(false)) { if (running.FollowUps.ContainsKey(queued.Id)) continue; head = queued; diff --git a/src/Chronos.Data/Repositories/JobRepository.cs b/src/Chronos.Data/Repositories/JobRepository.cs index 8f29cffc..356a66d0 100644 --- a/src/Chronos.Data/Repositories/JobRepository.cs +++ b/src/Chronos.Data/Repositories/JobRepository.cs @@ -7,6 +7,12 @@ namespace Chronos.Data.Repositories; /// Jobs per state; and count those finished since a given time. public sealed record JobCounts(int Queued, int Running, int Paused, int DoneSince, int FailedSince); +/// +/// What a running or paused job works on (): its targets and, for a verify, its +/// parameters, which name the sequences it holds; null for every other kind, whose parameters are not read. +/// +public sealed record JobHold(Guid Id, JobKind Kind, IReadOnlyList TargetIds, string? ParametersJson); + /// /// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically, /// write checkpoints while running, and on startup requeue what a crash left running. @@ -17,6 +23,14 @@ public sealed class JobRepository "id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " + "eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id"; + /// + /// as lists read them (): the parameters left out, null. A conversion's or + /// an import's name every sequence or file they work on; the job reads them, and so do the screens that act on that one job. + /// + private const string BriefColumns = + "id, kind, target_ids, target_label, CAST(NULL AS TEXT), state, priority, progress_done, progress_total, items_per_second, " + + "eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id"; + /// The states of a job that has not ended: queued, running, paused. private static readonly string PendingStates = string.Join(", ", new[] { JobState.Queued, JobState.Running, JobState.Paused }.Select(s => ((int)s).ToString(CultureInfo.InvariantCulture))); @@ -37,6 +51,11 @@ public sealed class JobRepository public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => _db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken); + /// When the job was queued; null when there is no such job. Reads that column alone. + public async Task GetCreatedAsync(Guid id, CancellationToken cancellationToken = default) => + (await _db.ListAsync("SELECT created FROM jobs WHERE id = $1", [id], r => r.GetDateTimeOffset(0), cancellationToken).ConfigureAwait(false)) + is [var created] ? created : null; + /// /// Marks the next queued job (highest priority, then oldest) of the given kinds as running for /// and returns it, or null when none waits. Concurrent claimers never get the same @@ -93,7 +112,17 @@ public sealed class JobRepository ListAsync(state, kind: null, limit, cursor, cancellationToken); /// Jobs, newest first, optionally of one state and one kind. - public Task> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) + public Task> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) => + PageAsync(Columns, state, kind, limit, cursor, cancellationToken); + + /// + /// as lists show jobs: every field but + /// , which stays null ( reads a job whole). + /// + public Task> ListBriefAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) => + PageAsync(BriefColumns, state, kind, limit, cursor, cancellationToken); + + private Task> PageAsync(string columns, JobState? state, JobKind? kind, int limit, string? cursor, CancellationToken cancellationToken) { var p = new SqlParameters(); var where = new List(); @@ -105,7 +134,7 @@ public sealed class JobRepository where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add)); } var take = Cursor.ClampLimit(limit); - var sql = $"SELECT {Columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " + + var sql = $"SELECT {columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " + $"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}"; return _db.RunAsync(async session => { @@ -158,11 +187,22 @@ public sealed class JobRepository ListQueuedAsync(kinds, limit, JobPriorityQueue.Off, cancellationToken); /// Queued jobs of the given kinds in the claim order of . - public Task> ListQueuedAsync(IReadOnlyCollection kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) + public Task> ListQueuedAsync(IReadOnlyCollection kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) => + QueuedAsync(Columns, kinds, limit, queue, cancellationToken); + + /// + /// without the + /// parameters (null), for a look at what would be claimed next: claiming a job reads it whole. + /// + public Task> ListQueuedBriefAsync(IReadOnlyCollection kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) => + QueuedAsync(BriefColumns, kinds, limit, queue, cancellationToken); + + private Task> QueuedAsync(string columns, IReadOnlyCollection kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(kinds); ArgumentNullException.ThrowIfNull(queue); return kinds.Count == 0 ? Task.FromResult>([]) : _db.ListAsync( - $"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1", + $"SELECT {columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1", [Cursor.ClampLimit(limit)], Map, cancellationToken); } @@ -195,6 +235,18 @@ public sealed class JobRepository return await _db.ScalarAsync(sql, p.ToArray(), cancellationToken).ConfigureAwait(false) > 0; } + /// + /// What the running and paused jobs hold, but the job (the one asking): each one's id, kind and + /// targets, and only a verify's parameters (). Neither the other parameters nor the targets of + /// are read: a conversion asks at every sequence it comes to, and its own row names them all. + /// + public Task> ListHoldsAsync(Guid? except = null, CancellationToken cancellationToken = default) => + _db.ListAsync( + $"SELECT id, kind, target_ids, CASE WHEN kind = {(int)JobKind.Verify} THEN parameters_json END FROM jobs " + + $"WHERE state IN ({(int)JobState.Running}, {(int)JobState.Paused}){(except is null ? string.Empty : " AND id <> $1")} ORDER BY created, id", + except is { } self ? [self] : [], + r => new JobHold(r.GetGuid(0), (JobKind)r.GetInt32(1), Targets(r.GetString(2)), r.GetStringOrNull(3)), cancellationToken); + /// The targets of the queued, running and paused jobs of (the newest 5000 such jobs). public Task> ListPendingTargetsAsync(JobKind kind, CancellationToken cancellationToken = default) => PendingTargetsAsync(kind, null, cancellationToken); @@ -306,11 +358,14 @@ public sealed class JobRepository private static string KindList(IEnumerable kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture))); + /// Target ids as the table stores them: comma-joined, empty for none. + private static Guid[] Targets(string ids) => ids.Length > 0 ? Array.ConvertAll(ids.Split(','), Guid.Parse) : []; + private static Job Map(DbRow r) => new() { Id = r.GetGuid(0), Kind = (JobKind)r.GetInt32(1), - TargetIds = r.GetString(2) is { Length: > 0 } ids ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [], + TargetIds = Targets(r.GetString(2)), TargetLabel = r.GetStringOrNull(3), ParametersJson = r.GetStringOrNull(4), State = (JobState)r.GetInt32(5), diff --git a/src/Chronos.Data/Repositories/SequenceRepository.cs b/src/Chronos.Data/Repositories/SequenceRepository.cs index d2d94ef6..a3338f62 100644 --- a/src/Chronos.Data/Repositories/SequenceRepository.cs +++ b/src/Chronos.Data/Repositories/SequenceRepository.cs @@ -565,6 +565,14 @@ public sealed class SequenceRepository _db.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2", [archiveId, sessionKey], cancellationToken); + /// + /// The session keys of an archive that match (a LIKE pattern, \ its escape), each + /// with the sequence it resolves to, by key: the new sequences conversions made of one source, say. + /// + public Task> ListSessionKeysAsync(Guid archiveId, string pattern, CancellationToken cancellationToken = default) => + _db.ListAsync("SELECT session_key, sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key LIKE $2 ESCAPE '\\' ORDER BY session_key", + [archiveId, pattern], r => (r.GetString(0), r.GetGuid(1)), cancellationToken); + /// Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to. public Task MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => diff --git a/src/Chronos.Server/Api/AdminEndpoints.cs b/src/Chronos.Server/Api/AdminEndpoints.cs index f59ca983..3e504403 100644 --- a/src/Chronos.Server/Api/AdminEndpoints.cs +++ b/src/Chronos.Server/Api/AdminEndpoints.cs @@ -48,8 +48,8 @@ public static class AdminEndpoints var users = await db.Users.ListAsync(ct); var names = users.ToDictionary(u => u.Id, u => u.Username); var counts = await db.Jobs.CountAsync(today, ct); - var running = await db.Jobs.ListAsync(JobState.Running, null, 10, null, ct); - var queued = await db.Jobs.ListAsync(JobState.Queued, null, 10, null, ct); + var running = await db.Jobs.ListBriefAsync(JobState.Running, null, 10, null, ct); + var queued = await db.Jobs.ListBriefAsync(JobState.Queued, null, 10, null, ct); var nodes = await db.RenderNodes.ListAsync(ct); var uploads = await db.Uploads.CommittedPerDayAsync(today.AddDays(-29), ct); var activity = await db.Audit.ListAsync(null, null, 12, null, ct); diff --git a/src/Chronos.Server/Api/JobEndpoints.cs b/src/Chronos.Server/Api/JobEndpoints.cs index bf219330..6817fe88 100644 --- a/src/Chronos.Server/Api/JobEndpoints.cs +++ b/src/Chronos.Server/Api/JobEndpoints.cs @@ -56,7 +56,8 @@ public static class JobEndpoints if (!JobWire.TryParseKind(kind, out var k)) return ApiErrors.BadRequest(context, "invalid_filter", "kind"); kindFilter = k; } - var page = await db.Jobs.ListAsync(stateFilter, kindFilter, limit ?? 50, string.IsNullOrEmpty(cursor) ? null : cursor, context.RequestAborted); + // A job's answer holds no parameters (a conversion's or an import's can name every sequence or file): the list reads none. + var page = await db.Jobs.ListBriefAsync(stateFilter, kindFilter, limit ?? 50, string.IsNullOrEmpty(cursor) ? null : cursor, context.RequestAborted); return Results.Json(new Page([.. page.Items.Select(Dto.Job)], page.NextCursor, null), ProtocolJson.Default.PageJobDto); } diff --git a/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs b/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs index 4395d681..cea35108 100644 --- a/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs @@ -76,18 +76,30 @@ public sealed class ConvertTests(TestContext context) return ValueTask.CompletedTask; }); - /// The conversion job of , run here as its handler runs it. - private static Task RunHereAsync(ArchiveFixture fx, ConvertPlan plan, Guid jobId, ConvertCallbacks? callbacks = null) => - fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), jobId, new ConvertRequester(null, null), callbacks ?? Callbacks(), fx.Ct); + /// + /// The conversion job of , run here as its handler runs it, with the photos + /// confirmed written beside its journal as writes them (once: a run again of the + /// same job finds them), and as the time it was queued. + /// + private static Task RunHereAsync(ArchiveFixture fx, ConvertPlan plan, Guid jobId, ConvertCallbacks? callbacks = null, DateTimeOffset? confirmed = null) + { + if (!File.Exists(ConfirmedPhotos.PathFor(fx.Paths, jobId))) Assert.True(ConfirmedPhotos.TryWrite(fx.Paths, jobId, plan)); + return fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), jobId, new ConvertRequester(null, null, confirmed), callbacks ?? Callbacks(), fx.Ct); + } private static string StagingOf(ArchiveFixture fx, Guid job, Guid sequence) => Path.Combine(Staging(fx), $"convert-{job:N}", $"{sequence:N}"); - private static async Task RunAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, JobState state = JobState.Done) + /// The job of , queued as the Library, the Archive manager and the panel queue it (). + private static async Task QueueAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, Guid? createdBy = null) { - var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct); - return await fx.WaitForAsync(job.Id, state, (int)TestTiming.Budget(60)); + Job? job = null; + await fx.Service.Converter.QueueAsync(fx.Archive, plan, async r => job = await engine.EnqueueAsync(r, createdBy, fx.Ct), fx.Ct); + return Assert.NotNull(job); } + private static async Task RunAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, JobState state = JobState.Done) => + await fx.WaitForAsync((await QueueAsync(fx, engine, plan)).Id, state, (int)TestTiming.Budget(60)); + private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath); private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path))); @@ -721,9 +733,16 @@ public sealed class ConvertTests(TestContext context) File.AppendAllText(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId), "{\"sequence\":"); var pinned = Assert.NotNull(read.Pins(fx.Paths).Of(first)); Assert.Equal((5, photo, "bb"), (pinned.FramesVersion, pinned.Frames.Single().Id, pinned.Frames.Single().Sha256)); - // Its files and its staging folder are known by their names. - Assert.SequenceEqual([journal.JobId, journal.JobId, journal.JobId], + // Its files (the photos confirmed among them) and its staging folder are known by their names. + Assert.SequenceEqual([journal.JobId, journal.JobId, journal.JobId, journal.JobId], ConvertJournal.Files(fx.Paths, journal.JobId).Select(p => ConvertJournal.JobOf(Path.GetFileName(p)))); + Assert.Contains(ConfirmedPhotos.PathFor(fx.Paths, journal.JobId), ConvertJournal.Files(fx.Paths, journal.JobId)); + // The copy a New sequence conversion made stays named whatever the sequence comes to after it. + var copy = Guid.NewGuid(); + journal.Made(harbour, copy); + journal.Note(new ConvertSequenceResult(harbour, "Harbour", ConvertOutcome.Failed, 0, Error: "no")); + journal.Write(fx.Paths); + Assert.Equal(copy, ConvertJournal.Read(fx.Paths, journal.JobId)!.Of(harbour)!.NewSequenceId); Assert.Equal(journal.JobId, ConvertJournal.JobOf(MassConverter.StagingFolder(journal.JobId))); Assert.Null(ConvertJournal.JobOf("convert-test")); Assert.Null(ConvertJournal.JobOf("rename-" + journal.JobId.ToString("N") + ".json")); @@ -1257,16 +1276,85 @@ public sealed class ConvertTests(TestContext context) } /// - /// Before each photo of stage A, the space is checked for it and the photos after it that may convert at once with it: - /// fewer near the end, so a job whose dry run had the space never stops short for photos that are not there. + /// Before each photo of stage A, the space is checked for it and for every photo handed to the work before it whose file + /// has not landed yet: a converted file lands whole, by a rename, before its photo's work verifies it, so the free space + /// measured holds the files landed and none of what the photos still converting will write. Near the end of a sequence + /// those are as many as anywhere else, never fewer; photos that are not there to start are never counted. /// [Test] - public async Task TheLastPhotosToStartNeedOnlyTheirOwnSpace() + public void APhotoStartsOnlyWithTheSpaceForItAndThePhotosStillWriting() { - ConvertFramePlan[] todo = [.. new long[] { 10, 20, 30, 40 }.Select((bytes, i) => new ConvertFramePlan(Guid.NewGuid(), i, $"originals/{i}.JPG", ImageFileFormat.Jpeg, 1, bytes))]; - Assert.SequenceEqual([60L, 90, 70, 40], Enumerable.Range(0, todo.Length).Select(i => MassConverter.Ahead(todo, i, 3))); - Assert.SequenceEqual([10L, 20, 30, 40], Enumerable.Range(0, todo.Length).Select(i => MassConverter.Ahead(todo, i, 1))); + const long photo = 100_000_000; + var staging = context.CreateTempDirectory("staging"); + string Target(int i) => Path.Combine(staging, $"{i:00}.tif"); + // The volume: what it had, less the files landed so far (those still converting are not on it yet). + var landed = 0L; + var volume = ConvertPlanner.Required(3 * photo); + var space = new MassConverter.SpaceWatch(() => volume - landed); + void Land(int i) + { + File.WriteAllBytes(Target(i), [1]); + landed += photo; + } + // The last photos of a sequence, four converting at once: none of the three started has landed, so the fourth, the + // last one, needs the space of all four, not its own alone. + for (var i = 0; i < 3; i++) Assert.Null(space.Start(photo, Target(i))); + Assert.Equal(3 * photo, space.Writing); + var refused = Assert.NotNull(space.Start(photo, Target(3))); + Assert.True(refused.StartsWith("The conversion needs", StringComparison.Ordinal), refused); + Assert.Equal(3 * photo, space.Writing); + + // One lands, and its work verifies it: the volume holds it, so the watch counts it no more (never twice). The last + // photo still waits: the two others and it need more than what is left. + Land(0); + Assert.Equal(2 * photo, space.Writing); + Assert.NotNull(space.Start(photo, Target(3))); + space.Ended(Target(0)); + Land(1); + Land(2); + Assert.Equal(0, space.Writing); + space.Ended(Target(1)); + space.Ended(Target(2)); + // All three landed: what is left, with the margin and the reserve, cannot take a fourth; the reserve is whole. + Assert.NotNull(space.Start(photo, Target(3))); + Assert.True(volume - landed >= ConvertPlanner.SpaceReserve); + + // Room for one more: it starts, and a photo whose work ends without a file (it failed) counts no more either. + volume = landed + ConvertPlanner.Required(photo); + Assert.Null(space.Start(photo, Target(3))); + Assert.NotNull(space.Start(photo, Target(4))); + space.Ended(Target(3)); + Assert.Null(space.Start(photo, Target(4))); + space.Ended(Target(4)); + + // A file a crash left where a photo converts is not its file: the photo counts until the new one replaces it. + File.WriteAllBytes(Target(5), [1, 2, 3]); + File.SetLastWriteTimeUtc(Target(5), new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc)); + Assert.Null(space.Start(photo, Target(5))); + Assert.Equal(photo, space.Writing); + File.WriteAllBytes(Target(5), new byte[10]); + Assert.Equal(0, space.Writing); + + // Files staged over their estimate scale what is still writing too. + var scaled = new MassConverter.SpaceWatch(() => ConvertPlanner.Required(3 * photo)); + scaled.Staged(2 * photo, photo); + Assert.Equal(2, scaled.Ratio); + Assert.Null(scaled.Start(photo, Target(6))); + Assert.NotNull(scaled.Start(photo, Target(7))); + scaled.Ended(Target(6)); + Assert.Null(scaled.Start(photo, Target(7))); + // A volume that cannot be measured stops nothing. + Assert.Null(new MassConverter.SpaceWatch(() => null).Start(long.MaxValue / 4, Target(8))); + } + + /// + /// A job whose dry run had the space never stops short at the end of a sequence: the last photo to start counts only + /// the photos that started and have not written yet, none that are not there. + /// + [Test] + public async Task TheLastPhotosToStartNeedOnlyTheSpaceOfThePhotosStillWriting() + { long free = 1L << 50; await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free, Parallelism = () => 4 }); var id = await UploadedAsync(fx, 1); @@ -1280,6 +1368,41 @@ public sealed class ConvertTests(TestContext context) Assert.Equal((ConvertOutcome.Converted, 1), (done.Outcome, done.Frames)); } + /// + /// A sequence that fits converts to its last photo, four at once, while its converted files take the volume as they land: + /// at the end the photos still writing are counted, those landed (verified meanwhile) are in the free space measured, and + /// none is counted twice. + /// + [Test] + public async Task ASequenceThatFitsConvertsToItsLastPhotoWhileItsFilesFillTheVolume() + { + long volume = 1L << 50; + string? staging = null; + // The volume as the converted files land on it; an atomic write's temporary file is left out, being there only while + // its photo still counts. + long Landed() => staging is not null && Directory.Exists(staging) + ? Directory.EnumerateFiles(staging).Where(f => !Path.GetFileName(f).StartsWith('.')).Sum(f => new FileInfo(f).Length) + : 0; + await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => volume - Landed(), Parallelism = () => 4 }); + var id = await UploadedAsync(fx, 12); + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); + // What the twelve converted files take, each converted beforehand as the job converts it. + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var manifest = Sidecar(fx, sequence); + var sizes = context.CreateTempDirectory("sizes"); + var converted = manifest.Frames.Sum(f => FrameConverter.Convert(new ConvertSource(folder.File(f.File), f, manifest.Camera, manifest.Lens), + new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), Path.Combine(sizes, f.Id.ToString("N") + ".tif"), cancellationToken: fx.Ct).Size); + // Just the space they need with the margin and the reserve, and never less than the dry run asks. + volume = ConvertPlanner.Required(Math.Max(converted, plan.BytesAfter)); + var jobId = Guid.CreateVersion7(); + staging = StagingOf(fx, jobId, id); + + var done = (await RunHereAsync(fx, plan, jobId)).Sequences.Single(); + Assert.Equal((ConvertOutcome.Converted, 12), (done.Outcome, done.Frames)); + Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff)); + } + [Test] public async Task ASecondConversionOfTheSameSequencesIsRefusedWithOrWithoutADryRun() { @@ -1333,26 +1456,239 @@ public sealed class ConvertTests(TestContext context) var first = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-2)); var second = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-1)); - var held = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null), Callbacks(), fx.Ct)) + var held = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null, second.Created), Callbacks(), fx.Ct)) .Sequences.Single(); Assert.Equal(ConvertOutcome.Blocked, held.Outcome); Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(second.Id, id))); // The one queued first is not held back by the one after it. - var made = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), first.Id, new ConvertRequester(null, null), Callbacks(), fx.Ct)) + var made = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), first.Id, new ConvertRequester(null, null, first.Created), Callbacks(), fx.Ct)) .Sequences.Single(); Assert.Equal((ConvertOutcome.Created, 2), (made.Outcome, made.Frames)); + await fx.Db.Jobs.SetStateAsync(first.Id, JobState.Done, null, fx.Ct); + // The duplicate, tried again once the first is done, finds the copy made since it was queued: it makes no second one. + var again = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null, second.Created), Callbacks(), fx.Ct)) + .Sequences.Single(); + Assert.Equal(ConvertOutcome.Superseded, again.Outcome); + Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(second.Id, id))); + } + + /// + /// A job's row names no photo, so it stays as small as the number of sequences however many photos convert: the photos + /// confirmed are written whole beside the journal, under the job's id, before it is queued, and go with the journal. + /// + [Test] + public async Task TheJobRowNamesNoPhotoAndThePhotosConfirmedWaitBesideItsJournal() + { + await using var fx = await FixtureAsync(); + var id = await UploadedAsync(fx, 3); + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); + var frames = plan.Sequences.Single().Frames.Select(f => f.FrameId).ToList(); + // Queued and not run yet: an engine that is not started. + await using var idle = new JobEngine(fx.Db, []); + Job? job = null; + await fx.Service.Converter.QueueAsync(fx.Archive, plan, async r => + { + Assert.FileExists(ConfirmedPhotos.PathFor(fx.Paths, r.Id!.Value)); + job = await idle.EnqueueAsync(r, cancellationToken: fx.Ct); + }, fx.Ct); + var queued = Assert.NotNull(await fx.Db.Jobs.GetAsync(job!.Id, fx.Ct)); + Assert.SequenceEqual(frames, ConfirmedPhotos.Read(fx.Paths, queued.Id).Of(id)!); + Assert.Null(ConfirmedPhotos.Read(fx.Paths, queued.Id).Of(Guid.NewGuid())); + foreach (var frame in frames) + { + Assert.False(queued.ParametersJson!.Contains(frame.ToString(), StringComparison.OrdinalIgnoreCase), queued.ParametersJson); + Assert.False(queued.ParametersJson!.Contains(ConfirmedPhotos.Encode([frame])[..20], StringComparison.Ordinal), queued.ParametersJson); + } + Assert.Equal(plan.Sequences.Single().Digest(), Parameters(fx, plan).Sequences.Single().Frames); + // Lists of jobs read none of their parameters. + Assert.Null((await fx.Db.Jobs.ListBriefAsync(JobState.Queued, JobKind.Convert, 10, null, fx.Ct)).Items.Single().ParametersJson); + + // A confirmation the queue refuses leaves nothing behind. + var other = await UploadedAsync(fx, 1, "upload-2", seed: 300); + var refused = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, other); + Guid? named = null; + await Assert.ThrowsAsync(() => fx.Service.Converter.QueueAsync(fx.Archive, refused, r => + { + named = r.Id; + throw new InvalidOperationException("the index is away"); + }, fx.Ct)); + Assert.FileMissing(ConfirmedPhotos.PathFor(fx.Paths, named!.Value)); + + // The job runs, and once done with every sequence, its photos go with its journal. + var engine = await fx.StartEngineAsync(); + engine.Wake(); + await fx.WaitForAsync(queued.Id, JobState.Done, (int)TestTiming.Budget(60)); + Assert.FileMissing(ConfirmedPhotos.PathFor(fx.Paths, queued.Id)); + Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff)); } [Test] - public void AJobsParametersNameThePhotosItConverts() + public void ThePhotosConfirmedReadBackAsWrittenAndALineThatCannotBeReadNamesNothing() { Guid[] ids = [Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.NewGuid()]; - var step = new ConvertSequence(Guid.NewGuid(), 3, "digest", ConvertSequence.EncodePhotos(ids)); - Assert.SequenceEqual(ids, step.PhotoIds()!); - var json = JsonSerializer.Serialize(step, ArchiveJobJson.Default.ConvertSequence); - Assert.SequenceEqual(ids, JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ConvertSequence)!.PhotoIds()!); - // A job an earlier build queued names none. - Assert.Null(new ConvertSequence(Guid.NewGuid(), 3, "digest").PhotoIds()); - Assert.Empty(new ConvertSequence(Guid.NewGuid(), 3, "digest", ConvertSequence.EncodePhotos([])).PhotoIds()!); + Assert.SequenceEqual(ids, ConfirmedPhotos.Decode(ConfirmedPhotos.Encode(ids))); + Assert.Empty(ConfirmedPhotos.Decode(ConfirmedPhotos.Encode([]))); + Assert.Throws(() => ConfirmedPhotos.Decode(Convert.ToBase64String(new byte[15]))); + + var paths = new ArchivePaths(context.CreateTempDirectory("archive")); + var job = Guid.CreateVersion7(); + var (good, bad) = (Guid.NewGuid(), Guid.NewGuid()); + Directory.CreateDirectory(paths.JournalDirectory); + File.WriteAllLines(ConfirmedPhotos.PathFor(paths, job), + [ + JsonSerializer.Serialize(new ConfirmedLine(good, ConfirmedPhotos.Encode(ids)), ArchiveJobJson.Default.ConfirmedLine), + JsonSerializer.Serialize(new ConfirmedLine(bad, "not ids"), ArchiveJobJson.Default.ConfirmedLine), + "{\"sequence\":", + ]); + var read = ConfirmedPhotos.Read(paths, job); + Assert.SequenceEqual(ids, read.Of(good)!); + Assert.Null(read.Of(bad)); + // None at all where nothing was written: every sequence is then held to its whole plan. + Assert.Null(ConfirmedPhotos.Read(paths, Guid.CreateVersion7()).Of(good)); + } + + /// + /// Without the photos confirmed (they could not be written when the job was queued, or an earlier build queued it), a + /// sequence converts only while its whole plan made again is the one confirmed: an upload since leaves it as it is. + /// + [Test] + public async Task WithoutThePhotosConfirmedASequenceIsHeldToItsWholePlan() + { + await using var fx = await FixtureAsync(); + var still = await UploadedAsync(fx, 2); + var grown = await UploadedAsync(fx, 2, "upload-2", seed: 300); + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, still, grown); + var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, grown, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct); + var before = await FramesAsync(fx, grown); + + var result = await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), Guid.CreateVersion7(), new ConvertRequester(null, null), Callbacks(), fx.Ct); + Assert.SequenceEqual([ConvertOutcome.Converted, ConvertOutcome.Changed], result.Sequences.Select(s => s.Outcome)); + Assert.SequenceEqual(before.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, grown)).Select(f => (f.Id, f.Sha256))); + } + + /// + /// A New sequence job that stopped once its copy held the photos, before its journal said the sequence was done (a crash, + /// a stop): retried after the source changed, it finishes that copy and makes no other; the copy is reported as made. + /// + [Test] + public async Task ARetryAfterACrashOnceTheCopyWasFiledFinishesThatCopyAndMakesNoOther() + { + await using var fx = await FixtureAsync(); + var id = await UploadedAsync(fx, 3); + await fx.Service.SetSequenceInfoAsync(id, "Roof", ["sunset"], 4, "From the roof", fx.Ct); + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); + var jobId = Guid.CreateVersion7(); + // What the attempt that stopped there left: the photos confirmed, pinned and staged, and the copy filed, the journal + // not saying more (it is written once the copy is finished). + var left = new Dictionary(); + var callbacks = Callbacks(done: _ => + { + foreach (var file in new[] { ConfirmedPhotos.PathFor(fx.Paths, jobId), ConvertJournal.PinsPathFor(fx.Paths, jobId), ConvertJournal.StagedPathFor(fx.Paths, jobId) }) + left[file] = File.ReadAllBytes(file); + }); + var first = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single(); + Assert.Equal(ConvertOutcome.Created, first.Outcome); + var copy = first.NewSequenceId!.Value; + await fx.Db.Sequences.EditAsync(copy, new SequenceEdit { Tags = ["other"], Rating = 1, Notes = "Scrambled" }, null, fx.Ct); + foreach (var (file, bytes) in left) File.WriteAllBytes(file, bytes); + // The source changed since: a photo of it deleted. + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [(await FramesAsync(fx, id))[1].Id], sequence.FramesVersion, fx.Ct); + await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct); + var sequences = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count; + + var retried = (await RunHereAsync(fx, plan, jobId)).Sequences.Single(); + Assert.Equal((ConvertOutcome.Created, 3, (Guid?)copy, 0), (retried.Outcome, retried.Frames, retried.NewSequenceId, retried.LeftOut)); + Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count); + Assert.Count(3, await FramesAsync(fx, copy)); + // Finished as stage B finishes a copy: the source's tags, rating and notes. + var finished = (await fx.Db.Sequences.GetAsync(copy))!; + Assert.Equal((4, "From the roof"), (finished.Rating, finished.Notes)); + Assert.SequenceEqual(["sunset"], finished.Tags); + Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId)); + + // Its journal names the copy as soon as the copy holds its photos: a copy deleted since is not made again. + var again = Guid.CreateVersion7(); + var journal = new ConvertJournal { JobId = again }; + journal.Made(id, Guid.CreateVersion7()); + journal.Write(fx.Paths); + var gone = (await RunHereAsync(fx, plan, again)).Sequences.Single(); + Assert.Equal(ConvertOutcome.Changed, gone.Outcome); + Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(again, id))); + Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count); + } + + /// The photos a New sequence job filed into its copy before it stopped take no space again: its retry finishes on a full volume. + [Test] + public async Task ARetryOnceTheCopyWasFiledNeedsNoSpaceForItsPhotosAgain() + { + long free = 1L << 50; + await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free }); + var id = await UploadedAsync(fx, 2); + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); + var jobId = Guid.CreateVersion7(); + var left = new Dictionary(); + var callbacks = Callbacks(done: _ => + { + foreach (var file in new[] { ConfirmedPhotos.PathFor(fx.Paths, jobId), ConvertJournal.PinsPathFor(fx.Paths, jobId), ConvertJournal.StagedPathFor(fx.Paths, jobId) }) + left[file] = File.ReadAllBytes(file); + }); + var copy = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single().NewSequenceId; + foreach (var (file, bytes) in left) File.WriteAllBytes(file, bytes); + free = 1_000; + + var retried = (await RunHereAsync(fx, plan, jobId)).Sequences.Single(); + Assert.Equal((ConvertOutcome.Created, copy), (retried.Outcome, retried.NewSequenceId)); + Assert.Count(2, await FramesAsync(fx, copy!.Value)); + } + + /// + /// A conversion into new sequences retried after another conversion, confirmed later, copied the same source in the same + /// format: it makes no second copy, and is done with that sequence. + /// + [Test] + public async Task ARetriedConversionMakesNoSecondCopyOfASourceAnotherConversionCopiedSince() + { + await using var fx = await FixtureAsync(); + var id = await UploadedAsync(fx, 2); + var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle; + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); + // Job A finds the sequence busy (a render works on it), and fails. + var a = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-10)); + var render = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, TargetIds = [id], State = JobState.Running, Created = DateTimeOffset.UtcNow }; + await fx.Db.Jobs.EnqueueAsync(render, fx.Ct); + Assert.Equal(ConvertOutcome.Blocked, (await RunHereAsync(fx, plan, a.Id, confirmed: a.Created)).Sequences.Single().Outcome); + await fx.Db.Jobs.SetStateAsync(render.Id, JobState.Done, null, fx.Ct); + await fx.Db.Jobs.SetStateAsync(a.Id, JobState.Failed, "kept", fx.Ct); + // The user plans again: job B makes the copy. A JPEG copy made meanwhile is another format, which does not count. + var again = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); + var b = await QueuedAsync(fx, again, DateTimeOffset.UtcNow.AddMinutes(-5)); + var made = (await RunHereAsync(fx, again, b.Id, confirmed: b.Created)).Sequences.Single(); + Assert.Equal(ConvertOutcome.Created, made.Outcome); + await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Done, null, fx.Ct); + var sequences = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count; + + // Retry on A: done with the sequence, no second copy. + var lines = new List(); + var retried = (await RunHereAsync(fx, plan, a.Id, Callbacks() with { Log = lines.Add }, a.Created)).Sequences.Single(); + Assert.Equal((ConvertOutcome.Superseded, true), (retried.Outcome, retried.Final)); + Assert.SequenceEqual([Loc.L("Archive.Log.ConvertSuperseded", title)], lines); + Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(a.Id, id))); + Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count); + Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, a.Id)); + + // A conversion confirmed after that copy was made is the user asking again: it makes its copy. + var latest = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); + var later = await QueuedAsync(fx, latest, DateTimeOffset.UtcNow); + Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, latest, later.Id, confirmed: later.Created)).Sequences.Single().Outcome); + + // A copy in another format is no copy of this one: a TIFF conversion confirmed before a DNG copy was made makes its own. + var other = await UploadedAsync(fx, 2, "upload-2", seed: 300); + var tiff = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, other); + var confirmedFirst = DateTimeOffset.UtcNow.AddMinutes(-1); + var dng = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, other); + Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, dng, Guid.CreateVersion7(), confirmed: DateTimeOffset.UtcNow)).Sequences.Single().Outcome); + Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, tiff, Guid.CreateVersion7(), confirmed: confirmedFirst)).Sequences.Single().Outcome); } } diff --git a/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs b/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs index fec31f7c..d07851da 100644 --- a/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs +++ b/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs @@ -103,6 +103,56 @@ public abstract class JobRepositoryTests(TestContext context) : RepositoryTest(c return job; } + [Test] + public async Task HoldsNameTheTargetsOfRunningAndPausedJobsAndOnlyAVerifysParameters() + { + // What a dry run counts as busy, read at every one and at every sequence a conversion comes to: never the parameters + // a conversion or an import keeps, nor the row of the job asking. + var db = await OpenChronosAsync(); + Guid a = Guid.CreateVersion7(), b = Guid.CreateVersion7(), archive = Guid.CreateVersion7(); + var convert = await EnqueueAsync(db, TestData.Job(JobKind.Convert) with { TargetIds = [a, b], ParametersJson = "{\"fingerprint\": \"f\"}", State = JobState.Running }); + var verify = await EnqueueAsync(db, TestData.Job(JobKind.Verify) with { TargetIds = [archive], ParametersJson = "{\"sequenceIds\": []}", State = JobState.Paused }); + await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [a] }); + await EnqueueAsync(db, TestData.Job(JobKind.Scan) with { TargetIds = [archive], State = JobState.Done, Finished = TestData.Now }); + + var holds = await db.Jobs.ListHoldsAsync(cancellationToken: Ct); + Assert.SequenceEqual(new[] { convert.Id, verify.Id }.Order(), holds.Select(h => h.Id).Order()); + var running = holds.Single(h => h.Id == convert.Id); + Assert.Equal((JobKind.Convert, (string?)null), (running.Kind, running.ParametersJson)); + Assert.SequenceEqual([a, b], running.TargetIds); + var paused = holds.Single(h => h.Id == verify.Id); + Assert.Equal((JobKind.Verify, verify.ParametersJson), (paused.Kind, paused.ParametersJson)); + Assert.SequenceEqual([verify.Id], (await db.Jobs.ListHoldsAsync(convert.Id, Ct)).Select(h => h.Id)); + + Assert.Equal(convert.Created, await db.Jobs.GetCreatedAsync(convert.Id, Ct)); + Assert.Null(await db.Jobs.GetCreatedAsync(Guid.CreateVersion7(), Ct)); + } + + [Test] + public async Task BriefListsReadEveryFieldButTheParameters() + { + var db = await OpenChronosAsync(); + var job = TestData.Job(JobKind.Convert, JobPriority.User) with + { + TargetIds = [Guid.CreateVersion7(), Guid.CreateVersion7()], + TargetLabel = "Harbour sunset", + ParametersJson = "{\"fingerprint\": \"f\"}", + LogTail = "line 1", + CreatedBy = Guid.CreateVersion7(), + ScheduleId = Guid.CreateVersion7(), + }; + await db.Jobs.EnqueueAsync(job, Ct); + + var brief = Assert.NotNull((await db.Jobs.ListBriefAsync(JobState.Queued, JobKind.Convert, 10, null, Ct)).Items.SingleOrDefault()); + Assert.SequenceEqual(job.TargetIds, brief.TargetIds); + Assert.Equal(job with { TargetIds = brief.TargetIds, ParametersJson = null }, brief); + Assert.Equal(job.ParametersJson, (await db.Jobs.ListAsync(JobState.Queued, JobKind.Convert, 10, null, Ct)).Items.Single().ParametersJson); + Assert.Empty((await db.Jobs.ListBriefAsync(JobState.Running, null, 10, null, Ct)).Items); + var next = Assert.NotNull((await db.Jobs.ListQueuedBriefAsync([JobKind.Convert], 10, Chronos.Data.Jobs.JobPriorityQueue.Off, Ct)).SingleOrDefault()); + Assert.Equal((job.Id, (string?)null), (next.Id, next.ParametersJson)); + Assert.Empty(await db.Jobs.ListQueuedBriefAsync([], 10, Chronos.Data.Jobs.JobPriorityQueue.Off, Ct)); + } + [Test] public async Task ThePriorityQueueClaimsTheHigherKindsFirst() {