|
|
|
@@ -1,3 +1,4 @@
|
|
|
|
|
using System.Collections.Concurrent;
|
|
|
|
|
using System.Security.Cryptography;
|
|
|
|
|
using System.Text;
|
|
|
|
|
using System.Text.Json;
|
|
|
|
@@ -29,6 +30,12 @@ public enum ConvertOutcome
|
|
|
|
|
/// <summary>It had nothing to convert.</summary>
|
|
|
|
|
Nothing,
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Done with, no copy made (<see cref="ConvertMode.NewSequence"/>): another conversion made its copy in this format since
|
|
|
|
|
/// this one was confirmed (a retry of an older conversion, or a duplicate queued beside it), so a second copy is not made.
|
|
|
|
|
/// </summary>
|
|
|
|
|
Superseded,
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Left as it was: a photo it converts no longer holds the content confirmed (it changed, is gone, or no longer converts),
|
|
|
|
|
/// or it is no longer in the archive. A retry finds the same: the user plans again.
|
|
|
|
@@ -52,8 +59,12 @@ public enum ConvertOutcome
|
|
|
|
|
public sealed record ConvertSequenceResult(Guid SequenceId, string Title, ConvertOutcome Outcome, int Frames, Guid? NewSequenceId = null, string? Error = null,
|
|
|
|
|
int LeftOut = 0)
|
|
|
|
|
{
|
|
|
|
|
/// <summary>Whether the sequence is done with: converted, made, or with nothing to convert.</summary>
|
|
|
|
|
public bool Final => Outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing;
|
|
|
|
|
/// <summary>Whether the sequence is done with: converted, made, with nothing to convert, or copied by another conversion since.</summary>
|
|
|
|
|
public bool Final => IsFinal(Outcome);
|
|
|
|
|
|
|
|
|
|
/// <summary>Whether <paramref name="outcome"/> leaves a sequence done with (<see cref="Final"/>).</summary>
|
|
|
|
|
internal static bool IsFinal(ConvertOutcome? outcome) =>
|
|
|
|
|
outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing or ConvertOutcome.Superseded;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>What a conversion did, and the passes the sequences it changed or made need (queued as follow-ups as each one ended).</summary>
|
|
|
|
@@ -75,8 +86,12 @@ public sealed record ConvertResult(IReadOnlyList<ConvertSequenceResult> Sequence
|
|
|
|
|
internal sealed record ConvertCallbacks(Func<long, long, ValueTask> Progress, Func<long, long, ValueTask> Checkpoint, Action<string> Log,
|
|
|
|
|
Func<ConvertSequenceResult, ValueTask>? Done = null, Func<IReadOnlyList<ArchiveJobRequest>, ValueTask>? Queue = null);
|
|
|
|
|
|
|
|
|
|
/// <summary>Who asked for a conversion: the user's name, kept in the trash records of a Replace, and id, the owner of a new sequence when its source has none.</summary>
|
|
|
|
|
internal sealed record ConvertRequester(string? Name, Guid? Id);
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Who asked for a conversion: the user's name, kept in the trash records of a Replace, and id, the owner of a new sequence
|
|
|
|
|
/// when its source has none; and when the plan was confirmed (the job queued): a copy another conversion made of a source
|
|
|
|
|
/// since then supersedes this one's (<see cref="ConvertOutcome.Superseded"/>). Null: no copy made by another counts.
|
|
|
|
|
/// </summary>
|
|
|
|
|
internal sealed record ConvertRequester(string? Name, Guid? Id, DateTimeOffset? Confirmed = null);
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// A conversion of sequences a confirmed plan converts is queued, running or paused already, so its job was not queued
|
|
|
|
@@ -94,7 +109,7 @@ public sealed class ConversionQueuedException(IReadOnlyList<Guid> sequenceIds) :
|
|
|
|
|
/// the conversion writes, <see cref="ArchiveServiceOptions.FreeSpace"/>), and a <see cref="JobKind.Convert"/> job applies the
|
|
|
|
|
/// confirmed plan (queued by <see cref="QueueAsync"/>, which refuses a second conversion of the same sequences) one sequence
|
|
|
|
|
/// at a time. Under the sequence's lease, its sidecar settled where the index has it then (<see cref="SequenceWriter.ReadLocked"/>),
|
|
|
|
|
/// the job plans it again and takes the photos confirmed (<see cref="ConvertSequence.Photos"/>), going on only while each is
|
|
|
|
|
/// the job plans it again and takes the photos confirmed (<see cref="ConfirmedPhotos"/>), going on only while each is
|
|
|
|
|
/// still planned with its content confirmed (<see cref="ConvertSequence.Frames"/>); it pins them then (<see cref="PinLog"/>),
|
|
|
|
|
/// so a photo that came in, was verified or could be opened again since the plan stays as it is, on a retry too. Stage A,
|
|
|
|
|
/// without the lease: each photo is hashed against its record and converted (<see cref="FrameConverter"/>, several at once
|
|
|
|
@@ -107,14 +122,19 @@ public sealed class ConversionQueuedException(IReadOnlyList<Guid> sequenceIds) :
|
|
|
|
|
/// <see cref="ConvertMode.NewSequence"/> files the converted copies into a new sequence through the archive ingest (session
|
|
|
|
|
/// key <c>convert:<job>:<source></c>, so a resume finds it again), titled "title (FORMAT)", owned as the source is,
|
|
|
|
|
/// with the source's tags, rating, notes and project, the source left as it is (the photos the plan did not convert stay
|
|
|
|
|
/// there alone, as the job log says). A sequence whose photos changed, that is busy, or has a photo that fails is left as it
|
|
|
|
|
/// was and reported.
|
|
|
|
|
/// there alone, as the job log says). A retry or a resume finishes the copy it made already rather than make another, and
|
|
|
|
|
/// makes none of a source that another conversion copied in the same format since this one was confirmed
|
|
|
|
|
/// (<see cref="ConvertOutcome.Superseded"/>). A sequence whose photos changed, that is busy, or has a photo that fails is
|
|
|
|
|
/// left as it was and reported.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public sealed class MassConverter
|
|
|
|
|
{
|
|
|
|
|
/// <summary>What the frames of a sequence no longer have once their files were replaced (<see cref="FrameTrash.ReplaceAsync"/>).</summary>
|
|
|
|
|
private const SequenceStatus Uncovered = SequenceStatus.AnalyzedLight | SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
|
|
|
|
|
|
|
|
|
|
/// <summary>How the session key of a new sequence a conversion makes begins (<see cref="SessionKey"/>).</summary>
|
|
|
|
|
private const string KeyPrefix = "convert:";
|
|
|
|
|
|
|
|
|
|
/// <summary>Held while a confirmed conversion is checked against those pending and queued (<see cref="QueueAsync"/>): one at a time in this process.</summary>
|
|
|
|
|
private static readonly SemaphoreSlim Queueing = new(1, 1);
|
|
|
|
|
|
|
|
|
@@ -186,8 +206,8 @@ public sealed class MassConverter
|
|
|
|
|
var busy = new HashSet<Guid>(await _trash.BusyAsync(job, cancellationToken).ConfigureAwait(false));
|
|
|
|
|
if (job is not { } self)
|
|
|
|
|
busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken).ConfigureAwait(false));
|
|
|
|
|
else if (await _db.Jobs.GetAsync(self, cancellationToken).ConfigureAwait(false) is { } running)
|
|
|
|
|
busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, running.Created, cancellationToken).ConfigureAwait(false));
|
|
|
|
|
else if (await _db.Jobs.GetCreatedAsync(self, cancellationToken).ConfigureAwait(false) is { } queued)
|
|
|
|
|
busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, queued, cancellationToken).ConfigureAwait(false));
|
|
|
|
|
return busy;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -195,8 +215,11 @@ public sealed class MassConverter
|
|
|
|
|
/// Queues the job of the confirmed <paramref name="plan"/> (<see cref="ArchiveJobs.Convert"/>) through <paramref name="enqueue"/>,
|
|
|
|
|
/// unless a conversion of a sequence it converts is queued, running or paused already: another dialog or tab, an answer
|
|
|
|
|
/// lost and Apply pressed again, a dry run made before that one was queued or none at all. Checked and queued under one
|
|
|
|
|
/// lock of this process, so two confirmations at once queue one job. <paramref name="enqueue"/> runs in the caller's
|
|
|
|
|
/// context (a desktop's tells the user on its UI thread).
|
|
|
|
|
/// lock of this process, so two confirmations at once queue one job. First, which photos it converts are written under
|
|
|
|
|
/// the job's id beside the journal it starts (<see cref="ConfirmedPhotos"/>); <paramref name="enqueue"/> queues the
|
|
|
|
|
/// request under that id (<see cref="ArchiveJobRequest.Id"/>), and runs in the caller's context (a desktop's tells the
|
|
|
|
|
/// user on its UI thread). When they cannot be written (the archive went away since its dry run), the job is queued all
|
|
|
|
|
/// the same and holds each sequence to its whole plan.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <exception cref="ConversionQueuedException">A conversion of one of its sequences waits or runs; nothing was queued.</exception>
|
|
|
|
|
/// <exception cref="ArgumentException">The plan converts nothing.</exception>
|
|
|
|
@@ -204,13 +227,25 @@ public sealed class MassConverter
|
|
|
|
|
{
|
|
|
|
|
ArgumentNullException.ThrowIfNull(enqueue);
|
|
|
|
|
var request = ArchiveJobs.Convert(archive, plan);
|
|
|
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
|
|
|
var job = request.Id!.Value;
|
|
|
|
|
// No ConfigureAwait(false) here: each wait comes back to the caller's context, where enqueue is called.
|
|
|
|
|
await Queueing.WaitAsync(cancellationToken);
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
var pending = await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken);
|
|
|
|
|
if (request.Targets.Where(pending.Contains).ToList() is { Count: > 0 } queued) throw new ConversionQueuedException(queued);
|
|
|
|
|
await enqueue(request);
|
|
|
|
|
var written = await Task.Run(() => ConfirmedPhotos.TryWrite(paths, job, plan), cancellationToken);
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
await enqueue(request);
|
|
|
|
|
}
|
|
|
|
|
catch
|
|
|
|
|
{
|
|
|
|
|
// Not queued: no job will read them.
|
|
|
|
|
if (written) ConfirmedPhotos.Delete(paths, job);
|
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
finally
|
|
|
|
|
{
|
|
|
|
@@ -260,6 +295,7 @@ public sealed class MassConverter
|
|
|
|
|
var steps = parameters.Sequences.DistinctBy(s => s.Id).ToList();
|
|
|
|
|
var staged = journal.Staged(paths);
|
|
|
|
|
var pins = journal.Pins(paths);
|
|
|
|
|
var confirmed = ConfirmedPhotos.Read(paths, jobId);
|
|
|
|
|
// What is still to do: every sequence not final in an earlier attempt that is still in the archive, its sidecar settled
|
|
|
|
|
// first when a writer a crash interrupted left a journal in its folder (this job's replacement among them).
|
|
|
|
|
var rows = new Dictionary<Guid, Sequence>();
|
|
|
|
@@ -277,11 +313,15 @@ public sealed class MassConverter
|
|
|
|
|
var unstaged = new Dictionary<Guid, long>();
|
|
|
|
|
foreach (var p in plan.Sequences.Where(p => p.Blocked.Count == 0))
|
|
|
|
|
{
|
|
|
|
|
if (Confirmed(byId[p.SequenceId], p, pins.Of(p.SequenceId)) is not { Frames.Count: > 0 } converts) continue;
|
|
|
|
|
if (Confirmed(byId[p.SequenceId], confirmed.Of(p.SequenceId), p, pins.Of(p.SequenceId)) is not { Frames.Count: > 0 } converts) continue;
|
|
|
|
|
progress.Count(converts.SequenceId, converts.Frames.Count, staged.Of(converts.SequenceId, converts.FramesVersion).Count);
|
|
|
|
|
unstaged[converts.SequenceId] = Unstaged(paths, jobId, converts, staged);
|
|
|
|
|
// What a new sequence this job made holds already takes no space again.
|
|
|
|
|
var filed = parameters.Mode == ConvertMode.NewSequence
|
|
|
|
|
? await FiledAsync(archive, SessionKey(jobId, converts.SequenceId), cancellationToken).ConfigureAwait(false)
|
|
|
|
|
: null;
|
|
|
|
|
unstaged[converts.SequenceId] = Unstaged(paths, jobId, converts, staged, filed);
|
|
|
|
|
}
|
|
|
|
|
var space = new SpaceWatch(this, paths);
|
|
|
|
|
var space = new SpaceWatch(() => FreeSpace(paths));
|
|
|
|
|
EnsureSpace(unstaged.Values.Sum(), space);
|
|
|
|
|
await callbacks.Progress(progress.Done, progress.Total).ConfigureAwait(false);
|
|
|
|
|
|
|
|
|
@@ -309,8 +349,8 @@ public sealed class MassConverter
|
|
|
|
|
// The space for what is left, measured again: the volume may have filled since, and the files staged so far tell
|
|
|
|
|
// how far off the estimate is.
|
|
|
|
|
EnsureSpace(unstaged.Values.Sum(), space);
|
|
|
|
|
done = await ConvertOneAsync(archive, paths, step, row, options, parameters.Mode, jobId, by, journal, pins, staged, progress, space, callbacks,
|
|
|
|
|
cancellationToken).ConfigureAwait(false);
|
|
|
|
|
done = await ConvertOneAsync(archive, paths, step, row, options, parameters.Mode, jobId, by, journal, confirmed, pins, staged, progress, space,
|
|
|
|
|
callbacks, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
unstaged.Remove(step.Id);
|
|
|
|
|
var result = done.Result;
|
|
|
|
@@ -349,16 +389,37 @@ public sealed class MassConverter
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// One sequence: under its lease, its sidecar settled, planned again and held to what was confirmed (the photos pinned the
|
|
|
|
|
/// first time); then stage A without the lease, and stage B.
|
|
|
|
|
/// first time); then stage A without the lease, and stage B. A New sequence that has its copy already is not converted
|
|
|
|
|
/// again (<see cref="CopiedAsync"/>).
|
|
|
|
|
/// </summary>
|
|
|
|
|
private async Task<StepDone> ConvertOneAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options,
|
|
|
|
|
ConvertMode mode, Guid jobId, ConvertRequester by, ConvertJournal journal, PinLog pins, StagedLog staged, Progress progress, SpaceWatch space,
|
|
|
|
|
ConvertCallbacks callbacks, CancellationToken cancellationToken)
|
|
|
|
|
ConvertMode mode, Guid jobId, ConvertRequester by, ConvertJournal journal, ConfirmedPhotos confirmedPhotos, PinLog pins, StagedLog staged,
|
|
|
|
|
Progress progress, SpaceWatch space, ConvertCallbacks callbacks, CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
// Another job holds it now (a Reorganize paused since the job started, a render, a conversion of it queued first): it
|
|
|
|
|
// waits for the retry.
|
|
|
|
|
var busy = await HeldAsync(jobId, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
if (busy.Contains(step.Id) || busy.Contains(archive.Id)) return new StepDone(Result(row, ConvertOutcome.Blocked));
|
|
|
|
|
var key = SessionKey(jobId, row.Id);
|
|
|
|
|
if (mode == ConvertMode.NewSequence)
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
if (await CopiedAsync(archive, paths, step, row, options, jobId, by, journal, pins, staged, key, progress, cancellationToken).ConfigureAwait(false)
|
|
|
|
|
is { } copied)
|
|
|
|
|
return copied;
|
|
|
|
|
}
|
|
|
|
|
catch (ArchiveOfflineException)
|
|
|
|
|
{
|
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
|
|
|
|
|
{
|
|
|
|
|
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the copy this conversion made of it could not be finished yet", ex);
|
|
|
|
|
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex)));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
var photos = confirmedPhotos.Of(step.Id);
|
|
|
|
|
SequencePaths folder;
|
|
|
|
|
SequenceManifest manifest;
|
|
|
|
|
ConvertSequencePlan planned;
|
|
|
|
@@ -379,7 +440,7 @@ public sealed class MassConverter
|
|
|
|
|
manifest = SequenceWriter.ReadLocked(folder, row.Id);
|
|
|
|
|
FrameTrash.EnsureReplaced(folder);
|
|
|
|
|
var pin = pins.Of(step.Id);
|
|
|
|
|
if (mode == ConvertMode.Replace && ReplacedEarlier(manifest, step, pin, staged.Of(step.Id, step.FramesVersion), options) is { } frames)
|
|
|
|
|
if (mode == ConvertMode.Replace && ReplacedEarlier(manifest, step, photos, pin, staged.Of(step.Id, step.FramesVersion), options) is { } frames)
|
|
|
|
|
{
|
|
|
|
|
// Replaced by this job already: a crash after its stage B, before the journal said so (the settle may just have
|
|
|
|
|
// completed it).
|
|
|
|
@@ -394,11 +455,11 @@ public sealed class MassConverter
|
|
|
|
|
IReadOnlySet<string> files = new HashSet<string>(FileRenames.ListPresent(at, sidecar), StringComparer.OrdinalIgnoreCase);
|
|
|
|
|
return (ConvertPlanner.Plan(sidecar, at, files, options, decoder, title), files);
|
|
|
|
|
}, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
if (Confirmed(step, fresh, pin) is not { } confirmed)
|
|
|
|
|
if (Confirmed(step, photos, fresh, pin) is not { } confirmed)
|
|
|
|
|
{
|
|
|
|
|
// A photo it converts that another program holds now cannot be told from one that changed: the retry, once it
|
|
|
|
|
// is free, can.
|
|
|
|
|
if (await Task.Run(() => Held(step, pin, sidecar, at, present, fresh, options, decoder), cancellationToken).ConfigureAwait(false))
|
|
|
|
|
if (await Task.Run(() => Held(photos, pin, sidecar, at, present, fresh, options, decoder), cancellationToken).ConfigureAwait(false))
|
|
|
|
|
{
|
|
|
|
|
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: a photo it converts cannot be opened now; it is not converted");
|
|
|
|
|
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.PhotoHeld")));
|
|
|
|
@@ -426,7 +487,6 @@ public sealed class MassConverter
|
|
|
|
|
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex)));
|
|
|
|
|
}
|
|
|
|
|
var staging = StagingOf(paths, jobId, row.Id);
|
|
|
|
|
var key = SessionKey(jobId, row.Id);
|
|
|
|
|
// A new sequence a resumed job finds again holds the photos it filed before it stopped.
|
|
|
|
|
var filed = mode == ConvertMode.NewSequence
|
|
|
|
|
? await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false)
|
|
|
|
@@ -443,7 +503,7 @@ public sealed class MassConverter
|
|
|
|
|
{
|
|
|
|
|
return mode == ConvertMode.Replace
|
|
|
|
|
? await ReplaceAsync(archive, paths, row, planned, ready, staging, options, jobId, by.Name, cancellationToken).ConfigureAwait(false)
|
|
|
|
|
: await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
: await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, journal, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
catch (LeaseUnavailableException ex)
|
|
|
|
|
{
|
|
|
|
@@ -479,15 +539,97 @@ public sealed class MassConverter
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// What the job converts of a sequence whose plan is now <paramref name="fresh"/>: the photos it pinned when it first came to
|
|
|
|
|
/// it, each still planned with the content it had then; the first time, the photos confirmed (<see cref="ConvertSequence.Photos"/>)
|
|
|
|
|
/// while each is still planned with the content confirmed (the step's digest). Photos that came in, were verified or could
|
|
|
|
|
/// be opened again since stay as they are, so an upload while the job waits its turn neither changes what it converts
|
|
|
|
|
/// nor holds it back. A job an earlier build queued: the whole plan while it has the step's digest, or its frames
|
|
|
|
|
/// version. Null when a photo confirmed no longer converts as it was: the sequence changed after the plan, and the user
|
|
|
|
|
/// plans again.
|
|
|
|
|
/// What a <see cref="ConvertMode.NewSequence"/> conversion does with a source that has its copy already when the job comes
|
|
|
|
|
/// to it (a retry, a resume): the copy this job made (its journal names it, or its session key when a crash came before
|
|
|
|
|
/// the journal did) is finished, whatever changed in the source since, once it holds every photo the job pinned, and one
|
|
|
|
|
/// gone since is not made again without a new plan; a copy in this format that another conversion made since this one was
|
|
|
|
|
/// confirmed (<see cref="ConvertRequester.Confirmed"/>) supersedes this one's. Null when neither: the sequence converts,
|
|
|
|
|
/// and a copy holding only some of its photos (a crash while they were filed) gets the rest.
|
|
|
|
|
/// </summary>
|
|
|
|
|
internal static ConvertSequencePlan? Confirmed(ConvertSequence step, ConvertSequencePlan fresh, PinnedSequence? pin)
|
|
|
|
|
private async Task<StepDone?> CopiedAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options, Guid jobId,
|
|
|
|
|
ConvertRequester by, ConvertJournal journal, PinLog pins, StagedLog staged, string key, Progress progress, CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
if ((journal.Of(step.Id)?.NewSequenceId ?? await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, cancellationToken).ConfigureAwait(false)) is { } made)
|
|
|
|
|
{
|
|
|
|
|
if (await _db.Sequences.GetAsync(made, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id)
|
|
|
|
|
{
|
|
|
|
|
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it is gone; no other is made without a new plan");
|
|
|
|
|
return new StepDone(Result(row, ConvertOutcome.Changed));
|
|
|
|
|
}
|
|
|
|
|
return pins.Of(step.Id) is { } pin
|
|
|
|
|
? await FinishCopyAsync(archive, paths, row, pin, staged, key, made, progress, cancellationToken).ConfigureAwait(false)
|
|
|
|
|
: null;
|
|
|
|
|
}
|
|
|
|
|
if (by.Confirmed is not { } confirmed
|
|
|
|
|
|| await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } other)
|
|
|
|
|
return null;
|
|
|
|
|
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed; no second copy is made");
|
|
|
|
|
progress.Count(row.Id, 0, 0);
|
|
|
|
|
return new StepDone(Result(row, ConvertOutcome.Superseded));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The copy this job made of a source, finished when it holds the conversion of every photo pinned (a crash came after
|
|
|
|
|
/// they were filed, before the journal said so): it takes the source's tags, rating, notes and project as stage B gives
|
|
|
|
|
/// them, whatever became of the photos of the source since. Null while a photo pinned is not in it, or when the source
|
|
|
|
|
/// is no longer where this index has it: the job converts the sequence, which files the rest.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private async Task<StepDone?> FinishCopyAsync(Archive archive, ArchivePaths paths, Sequence row, PinnedSequence pin, StagedLog staged, string key, Guid made,
|
|
|
|
|
Progress progress, CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
var conversions = staged.Of(row.Id, pin.FramesVersion).ToDictionary(c => c.FrameId);
|
|
|
|
|
var filed = await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
var ready = new Dictionary<Guid, StagedConversion>();
|
|
|
|
|
foreach (var photo in pin.Frames)
|
|
|
|
|
{
|
|
|
|
|
if (!conversions.TryGetValue(photo.Id, out var conversion) || !string.Equals(conversion.Source, photo.Sha256, StringComparison.OrdinalIgnoreCase)
|
|
|
|
|
|| !filed.Contains(conversion.Sha256))
|
|
|
|
|
return null;
|
|
|
|
|
ready[photo.Id] = conversion;
|
|
|
|
|
}
|
|
|
|
|
SequenceManifest manifest;
|
|
|
|
|
string? project;
|
|
|
|
|
await using (await _writer.LockAsync(paths, row.Id, cancellationToken).ConfigureAwait(false))
|
|
|
|
|
{
|
|
|
|
|
if (await _db.Sequences.GetAsync(row.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id) return null;
|
|
|
|
|
row = now;
|
|
|
|
|
var folder = paths.Sequence(row.Folder);
|
|
|
|
|
if (!SequenceFolders.Holds(folder, row.Id)) return null;
|
|
|
|
|
manifest = SequenceWriter.ReadLocked(folder, row.Id);
|
|
|
|
|
project = row.SettingsJson ?? (File.Exists(folder.ProjectPath) ? AtomicFile.ReadAllText(folder.ProjectPath) : null);
|
|
|
|
|
}
|
|
|
|
|
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it holds every photo already; it is finished, no other is made");
|
|
|
|
|
progress.Count(row.Id, pin.Frames.Count, pin.Frames.Count);
|
|
|
|
|
return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, made, pin.Frames.Count, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// A copy of <paramref name="source"/> in this conversion's format that another conversion made since <paramref name="confirmed"/>:
|
|
|
|
|
/// a sequence filed under another job's session key for that source (<see cref="SessionKey"/>) whose sidecar says it was
|
|
|
|
|
/// made since then and holds photos of that format. Null when there is none.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private async Task<Guid?> LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed,
|
|
|
|
|
CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
foreach (var (key, id) in await _db.Sequences.ListSessionKeysAsync(archive.Id, $"{KeyPrefix}%:{source:N}", cancellationToken).ConfigureAwait(false))
|
|
|
|
|
{
|
|
|
|
|
if (JobOfKey(key) is not { } other || other == jobId) continue;
|
|
|
|
|
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id) continue;
|
|
|
|
|
if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format)) return id;
|
|
|
|
|
}
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// What the job converts of a sequence whose plan is now <paramref name="fresh"/>: the photos it pinned when it first came to
|
|
|
|
|
/// it, each still planned with the content it had then; the first time, the photos confirmed (<paramref name="photos"/>,
|
|
|
|
|
/// <see cref="ConfirmedPhotos"/>) while each is still planned with the content confirmed (the step's digest). Photos that
|
|
|
|
|
/// came in, were verified or could be opened again since stay as they are, so an upload while the job waits its turn
|
|
|
|
|
/// neither changes what it converts nor holds it back. Without the photos confirmed (they could not be written, or a job
|
|
|
|
|
/// an earlier build queued): the whole plan while it has the step's digest, or its frames version. Null when a photo
|
|
|
|
|
/// confirmed no longer converts as it was: the sequence changed after the plan, and the user plans again.
|
|
|
|
|
/// </summary>
|
|
|
|
|
internal static ConvertSequencePlan? Confirmed(ConvertSequence step, IReadOnlyList<Guid>? photos, ConvertSequencePlan fresh, PinnedSequence? pin)
|
|
|
|
|
{
|
|
|
|
|
var now = fresh.Frames.ToDictionary(f => f.FrameId);
|
|
|
|
|
if (pin is not null)
|
|
|
|
@@ -500,7 +642,7 @@ public sealed class MassConverter
|
|
|
|
|
}
|
|
|
|
|
return frames.Count > 0 ? fresh with { FramesVersion = pin.FramesVersion, Frames = frames } : null;
|
|
|
|
|
}
|
|
|
|
|
if (step is { Frames: { } confirmedDigest } && step.PhotoIds() is { Count: > 0 } photos)
|
|
|
|
|
if (step is { Frames: { } confirmedDigest } && photos is { Count: > 0 })
|
|
|
|
|
{
|
|
|
|
|
var frames = new List<ConvertFramePlan>(photos.Count);
|
|
|
|
|
foreach (var id in photos)
|
|
|
|
@@ -520,15 +662,15 @@ public sealed class MassConverter
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Whether the plan made now (<paramref name="fresh"/>) leaves out a photo the job converts of the sequence (one it pinned,
|
|
|
|
|
/// else one confirmed) only because its file could not be opened (<see cref="ConvertSkip.Inaccessible"/>: another program
|
|
|
|
|
/// holds it): the sequence then waits for the retry rather than counting as changed. Photos the job does not convert do
|
|
|
|
|
/// not count, nor does a confirmed photo that changed meanwhile. A job an earlier build queued, which names no photos:
|
|
|
|
|
/// any photo of the sequence that could not be opened.
|
|
|
|
|
/// else one of <paramref name="photos"/>, those confirmed) only because its file could not be opened
|
|
|
|
|
/// (<see cref="ConvertSkip.Inaccessible"/>: another program holds it): the sequence then waits for the retry rather than
|
|
|
|
|
/// counting as changed. Photos the job does not convert do not count, nor does a confirmed photo that changed meanwhile.
|
|
|
|
|
/// Without the photos confirmed: any photo of the sequence that could not be opened.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private static bool Held(ConvertSequence step, PinnedSequence? pin, SequenceManifest manifest, SequencePaths folder, IReadOnlySet<string> present,
|
|
|
|
|
private static bool Held(IReadOnlyList<Guid>? photos, PinnedSequence? pin, SequenceManifest manifest, SequencePaths folder, IReadOnlySet<string> present,
|
|
|
|
|
ConvertSequencePlan fresh, ConvertOptions options, ImageDecoder decoder)
|
|
|
|
|
{
|
|
|
|
|
IReadOnlyList<Guid>? converts = pin is not null ? [.. pin.Frames.Select(f => f.Id)] : step.PhotoIds();
|
|
|
|
|
IReadOnlyList<Guid>? converts = pin is not null ? [.. pin.Frames.Select(f => f.Id)] : photos;
|
|
|
|
|
if (converts is null) return fresh.Skipped.Any(s => s.Reason == ConvertSkip.Inaccessible);
|
|
|
|
|
var planned = fresh.Frames.Select(f => f.FrameId).ToHashSet();
|
|
|
|
|
var frames = manifest.Frames.ToDictionary(f => f.Id);
|
|
|
|
@@ -579,12 +721,13 @@ public sealed class MassConverter
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Stage A: converts the planned frames not staged yet, several at once, each staged file appended to the journal as it
|
|
|
|
|
/// is consumed (in frame order). A photo starts only while the volume has the space it and the photos after it that may
|
|
|
|
|
/// convert at once with it take (as many as the host converts at once, never photos that are not there to start), with
|
|
|
|
|
/// the reserve: short of it the job stops (<c>Archive.Error.ConvertSpace</c>) before the volume fills. The space the
|
|
|
|
|
/// photos converting already take is in the volume's free space as they write it, so it is not counted twice. Returns
|
|
|
|
|
/// the first failure (the file and why, localised; the exception goes to the diagnostics log); the frames converted
|
|
|
|
|
/// meanwhile stay staged for a retry.
|
|
|
|
|
/// is consumed (in frame order). A photo starts only while the volume has the space for it and for every photo handed to
|
|
|
|
|
/// the work before it whose file has not landed yet (<see cref="SpaceWatch.Start"/>: converting or waiting for its turn,
|
|
|
|
|
/// up to twice as many as convert at once, at the end of a sequence as anywhere), with the margin and the reserve: short
|
|
|
|
|
/// of it the job stops (<c>Archive.Error.ConvertSpace</c>) before the volume fills. A converted file is encoded in memory
|
|
|
|
|
/// and lands whole, by a rename, before it is verified: what those photos will write is not in the free space measured
|
|
|
|
|
/// yet, and what landed is, so each counts until its file lands, never twice. Returns the first failure (the file and why,
|
|
|
|
|
/// localised; the exception goes to the diagnostics log); the frames converted meanwhile stay staged for a retry.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <exception cref="IOException">The volume lacks the space for the next photo; what was staged stays.</exception>
|
|
|
|
|
private async Task<string?> StageAsync(Sequence row, SequenceManifest manifest, SequencePaths folder, ConvertSequencePlan planned,
|
|
|
|
@@ -622,13 +765,26 @@ public sealed class MassConverter
|
|
|
|
|
}, cancellationToken);
|
|
|
|
|
for (var i = 0; i < todo.Count && failure is null; i++)
|
|
|
|
|
{
|
|
|
|
|
if (space.Short(Ahead(todo, i, atOnce)) is { } message)
|
|
|
|
|
var frame = frames[todo[i].FrameId];
|
|
|
|
|
var target = Path.Combine(staging, frame.Id.ToString("N") + options.Extension);
|
|
|
|
|
if (space.Start(todo[i].EstimatedBytes, target) is { } message)
|
|
|
|
|
{
|
|
|
|
|
full = message;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
var frame = frames[todo[i].FrameId];
|
|
|
|
|
await work.AddAsync(token => Stage(frame, manifest, folder, staging, options, decoder, threads, token)).ConfigureAwait(false);
|
|
|
|
|
// A photo that never starts (the job stops: cancelled, or a write of the journal failed) is never counted off,
|
|
|
|
|
// and the watch goes with the job.
|
|
|
|
|
await work.AddAsync(token =>
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
return Stage(frame, manifest, folder, target, options, decoder, threads, token);
|
|
|
|
|
}
|
|
|
|
|
finally
|
|
|
|
|
{
|
|
|
|
|
space.Ended(target);
|
|
|
|
|
}
|
|
|
|
|
}).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
await work.CompleteAsync().ConfigureAwait(false);
|
|
|
|
|
if (full is not null)
|
|
|
|
@@ -640,20 +796,8 @@ public sealed class MassConverter
|
|
|
|
|
return failure;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The bytes photo <paramref name="index"/> of <paramref name="todo"/> and the photos after it that may convert at once with
|
|
|
|
|
/// it are estimated at: <paramref name="atOnce"/> of them, fewer near the end (the space a photo of stage A starts with).
|
|
|
|
|
/// </summary>
|
|
|
|
|
internal static long Ahead(IReadOnlyList<ConvertFramePlan> todo, int index, int atOnce)
|
|
|
|
|
{
|
|
|
|
|
ArgumentNullException.ThrowIfNull(todo);
|
|
|
|
|
long bytes = 0;
|
|
|
|
|
for (var i = index; i < Math.Min(todo.Count, index + Math.Max(1, atOnce)); i++) bytes += todo[i].EstimatedBytes;
|
|
|
|
|
return bytes;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>One photo of stage A: hashed against its record, converted into the staging folder and verified.</summary>
|
|
|
|
|
private static Staged Stage(SidecarFrame frame, SequenceManifest manifest, SequencePaths folder, string staging, ConvertOptions options, ImageDecoder decoder,
|
|
|
|
|
/// <summary>One photo of stage A: hashed against its record, converted into <paramref name="target"/> (in the staging folder) and verified.</summary>
|
|
|
|
|
private static Staged Stage(SidecarFrame frame, SequenceManifest manifest, SequencePaths folder, string target, ConvertOptions options, ImageDecoder decoder,
|
|
|
|
|
int threads, CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
var source = folder.File(frame.File);
|
|
|
|
@@ -666,7 +810,6 @@ public sealed class MassConverter
|
|
|
|
|
if (!string.Equals(sha, frame.Sha256, StringComparison.OrdinalIgnoreCase))
|
|
|
|
|
throw new ConversionException(ConversionFailure.Changed, $"{frame.File} no longer matches its SHA-256; verify the sequence");
|
|
|
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
|
|
|
var target = Path.Combine(staging, frame.Id.ToString("N") + options.Extension);
|
|
|
|
|
var file = FrameConverter.Convert(new ConvertSource(source, frame, manifest.Camera, manifest.Lens), options, decoder, target, threads, cancellationToken);
|
|
|
|
|
return new Staged(frame.File, new StagedConversion(frame.Id, sha, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height), null);
|
|
|
|
|
}
|
|
|
|
@@ -734,9 +877,12 @@ public sealed class MassConverter
|
|
|
|
|
/// source's capture facts; then the source's tags, rating, notes and project (its positions translated to the new frames).
|
|
|
|
|
/// Every conversion not filed yet must still be staged whole: one gone since (a clean-up, a quarantine) fails the sequence
|
|
|
|
|
/// rather than leave its photo out. The photos the plan did not convert are not in the new sequence (<see cref="ConvertSequenceResult.LeftOut"/>).
|
|
|
|
|
/// Once its photos are filed, the journal names the new sequence (<see cref="ConvertJournal.Made"/>), so that a retry or a
|
|
|
|
|
/// resume finishes it and never makes a second one.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private async Task<StepDone> CreateAsync(Archive archive, ArchivePaths paths, Sequence row, ConvertSequencePlan planned,
|
|
|
|
|
Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options, string key, Guid? requestedBy, CancellationToken cancellationToken)
|
|
|
|
|
Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options, string key, Guid? requestedBy, ConvertJournal journal,
|
|
|
|
|
CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
SequenceManifest manifest;
|
|
|
|
|
string? project;
|
|
|
|
@@ -775,13 +921,26 @@ public sealed class MassConverter
|
|
|
|
|
await _ingest.CommitSessionAsync(archive, new SequenceTarget(key, title, null, owner), pending, CancellationToken.None).ConfigureAwait(false);
|
|
|
|
|
var id = await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, CancellationToken.None).ConfigureAwait(false)
|
|
|
|
|
?? throw new IOException($"No converted photo of {row.DisplayTitle} could be filed into a new sequence.");
|
|
|
|
|
journal.Made(row.Id, id);
|
|
|
|
|
journal.Write(paths);
|
|
|
|
|
return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, id, planned.Frames.Count, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The end of stage B of <see cref="ConvertMode.NewSequence"/>, once the new sequence <paramref name="id"/> holds the
|
|
|
|
|
/// <paramref name="frames"/> photos converted: the source's tags, rating, notes and project given to it (<see cref="CopyAsync"/>),
|
|
|
|
|
/// and the passes it needs, from what the source has. The photos of the source it does not hold are counted as left out.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private async Task<StepDone> CompleteCopyAsync(Archive archive, ArchivePaths paths, Sequence row, SequenceManifest manifest, string? project,
|
|
|
|
|
Dictionary<Guid, StagedConversion> ready, Guid id, int frames, CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
await CopyAsync(paths, row, manifest, project, ready, id, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
var made = await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false)
|
|
|
|
|
?? throw new InvalidOperationException($"The new sequence {id} is not indexed.");
|
|
|
|
|
var leftOut = manifest.Frames.Count - planned.Frames.Count;
|
|
|
|
|
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: {planned.Frames.Count} photo(s) converted into the new sequence {made.DisplayTitle}, {leftOut} left in the source alone");
|
|
|
|
|
var leftOut = manifest.Frames.Count(f => !ready.ContainsKey(f.Id));
|
|
|
|
|
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: {frames} photo(s) converted into the new sequence {made.DisplayTitle}, {leftOut} left in the source alone");
|
|
|
|
|
var had = row.Status & (SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt);
|
|
|
|
|
return new StepDone(new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Created, planned.Frames.Count, id, LeftOut: leftOut), had,
|
|
|
|
|
return new StepDone(new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Created, frames, id, LeftOut: leftOut), had,
|
|
|
|
|
_frames.FollowUps(archive, made, had));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -855,13 +1014,13 @@ public sealed class MassConverter
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The photos converted, when this job replaced the sequence's frames already (a crash after its stage B, before the
|
|
|
|
|
/// journal said so): every photo it pinned is a frame holding a file of the target's format converted from the content
|
|
|
|
|
/// pinned. Without a pin (its journal lost), every photo confirmed (<see cref="ConvertSequence.Photos"/>) is a frame
|
|
|
|
|
/// holding a conversion to the target whose sources have the step's digest; a job an earlier build queued: the frames
|
|
|
|
|
/// holding a conversion to the target are, by their sources, the photos of the step's digest, or every staged conversion
|
|
|
|
|
/// is a frame holding the converted content. Null otherwise.
|
|
|
|
|
/// pinned. Without a pin (its journal lost), every photo confirmed (<paramref name="photos"/>) is a frame holding a
|
|
|
|
|
/// conversion to the target whose sources have the step's digest; without those either: the frames holding a conversion
|
|
|
|
|
/// to the target are, by their sources, the photos of the step's digest, or (a job an earlier build queued) every staged
|
|
|
|
|
/// conversion is a frame holding the converted content. Null otherwise.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private static int? ReplacedEarlier(SequenceManifest manifest, ConvertSequence step, PinnedSequence? pin, IReadOnlyList<StagedConversion> noted,
|
|
|
|
|
ConvertOptions options)
|
|
|
|
|
private static int? ReplacedEarlier(SequenceManifest manifest, ConvertSequence step, IReadOnlyList<Guid>? photos, PinnedSequence? pin,
|
|
|
|
|
IReadOnlyList<StagedConversion> noted, ConvertOptions options)
|
|
|
|
|
{
|
|
|
|
|
var frames = manifest.Frames.ToDictionary(f => f.Id);
|
|
|
|
|
if (pin is not null)
|
|
|
|
@@ -872,7 +1031,7 @@ public sealed class MassConverter
|
|
|
|
|
? pin.Frames.Count
|
|
|
|
|
: null;
|
|
|
|
|
}
|
|
|
|
|
if (step is { Frames: { } confirmed } && step.PhotoIds() is { Count: > 0 } photos)
|
|
|
|
|
if (step is { Frames: { } confirmed } && photos is { Count: > 0 })
|
|
|
|
|
{
|
|
|
|
|
var sources = new List<(Guid, string?)>(photos.Count);
|
|
|
|
|
foreach (var id in photos)
|
|
|
|
@@ -987,11 +1146,16 @@ public sealed class MassConverter
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>The bytes the photos <paramref name="plan"/> converts are estimated at, but for those whose staged file is there already.</summary>
|
|
|
|
|
private static long Unstaged(ArchivePaths paths, Guid jobId, ConvertSequencePlan plan, StagedLog staged)
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The bytes the photos <paramref name="plan"/> converts are estimated at, but for those whose staged file is there already,
|
|
|
|
|
/// or which the new sequence this job made holds (<paramref name="filed"/>, the SHA-256 of its photos).
|
|
|
|
|
/// </summary>
|
|
|
|
|
private static long Unstaged(ArchivePaths paths, Guid jobId, ConvertSequencePlan plan, StagedLog staged, IReadOnlySet<string>? filed)
|
|
|
|
|
{
|
|
|
|
|
var folder = StagingOf(paths, jobId, plan.SequenceId);
|
|
|
|
|
var there = staged.Of(plan.SequenceId, plan.FramesVersion).Where(c => Present(Path.Combine(folder, c.File), c)).Select(c => c.FrameId).ToHashSet();
|
|
|
|
|
var there = staged.Of(plan.SequenceId, plan.FramesVersion)
|
|
|
|
|
.Where(c => filed?.Contains(c.Sha256) == true || Present(Path.Combine(folder, c.File), c))
|
|
|
|
|
.Select(c => c.FrameId).ToHashSet();
|
|
|
|
|
return plan.Frames.Where(f => !there.Contains(f.FrameId)).Sum(f => f.EstimatedBytes);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -1042,6 +1206,7 @@ public sealed class MassConverter
|
|
|
|
|
ConvertOutcome.Converted => Loc.L("Archive.Log.Converted", result.Title, result.Frames),
|
|
|
|
|
ConvertOutcome.Created => Loc.L("Archive.Log.ConvertedNew", result.Title, result.Frames),
|
|
|
|
|
ConvertOutcome.Nothing => Loc.L("Archive.Log.ConvertNothing", result.Title),
|
|
|
|
|
ConvertOutcome.Superseded => Loc.L("Archive.Log.ConvertSuperseded", result.Title),
|
|
|
|
|
ConvertOutcome.Changed => Loc.L("Archive.Log.ConvertChanged", result.Title),
|
|
|
|
|
// Busy, or why the retry may do better (a photo another program holds, a folder another index moved), in one sentence.
|
|
|
|
|
ConvertOutcome.Blocked when result.Error is null => Loc.L("Archive.Log.ConvertBlocked", result.Title),
|
|
|
|
@@ -1067,7 +1232,11 @@ public sealed class MassConverter
|
|
|
|
|
: Loc.L("Common.Size.MB", (bytes / 1e6).ToString(digits, Loc.Culture));
|
|
|
|
|
|
|
|
|
|
/// <summary>The session key of the new sequence a conversion makes from <paramref name="source"/>.</summary>
|
|
|
|
|
internal static string SessionKey(Guid job, Guid source) => $"convert:{job:N}:{source:N}";
|
|
|
|
|
internal static string SessionKey(Guid job, Guid source) => $"{KeyPrefix}{job:N}:{source:N}";
|
|
|
|
|
|
|
|
|
|
/// <summary>The conversion job a session key of <see cref="SessionKey"/> names; null for any other key.</summary>
|
|
|
|
|
private static Guid? JobOfKey(string key) =>
|
|
|
|
|
key.Split(':') is ["convert", var job, _] && Guid.TryParseExact(job, "N", out var id) ? id : null;
|
|
|
|
|
|
|
|
|
|
/// <summary>The folder under <c>.chronos/staging</c> a conversion job stages into, one subfolder per sequence.</summary>
|
|
|
|
|
internal static string StagingFolder(Guid job) => ConvertJournal.Prefix + job.ToString("N");
|
|
|
|
@@ -1112,34 +1281,68 @@ public sealed class MassConverter
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The free space a running conversion watches: measured where it writes, against estimates scaled by how far the files it
|
|
|
|
|
/// staged came out above theirs (photos of a 14-bit RAW carry more noise than the estimate's bits per pixel allow for).
|
|
|
|
|
/// The free space a running conversion watches: measured where it writes (<paramref name="free"/>), against estimates scaled
|
|
|
|
|
/// by how far the files it staged came out above theirs (photos of a 14-bit RAW carry more noise than the estimate's bits
|
|
|
|
|
/// per pixel allow for), and, in stage A, with the photos started whose converted file is not there yet (<see cref="Start"/>):
|
|
|
|
|
/// a converted file is encoded in memory and lands whole, by a rename, before its photo's work verifies it, so the free
|
|
|
|
|
/// space measured holds every file landed, and none of what the others will write.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private sealed class SpaceWatch(MassConverter converter, ArchivePaths paths)
|
|
|
|
|
internal sealed class SpaceWatch(Func<long?> free)
|
|
|
|
|
{
|
|
|
|
|
private readonly ConcurrentDictionary<string, Starting> _started = new(StringComparer.Ordinal);
|
|
|
|
|
private long _actual;
|
|
|
|
|
private long _estimated;
|
|
|
|
|
|
|
|
|
|
/// <summary>The bytes staged over their estimates, at least 1.</summary>
|
|
|
|
|
public double Ratio => _estimated > 0 ? Math.Max(1, (double)_actual / _estimated) : 1;
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The estimated bytes of the photos started (<see cref="Start"/>) whose converted file has not landed yet: the file is
|
|
|
|
|
/// not at its path, or is still the one that was there when the photo started (one a crash left, which it replaces).
|
|
|
|
|
/// </summary>
|
|
|
|
|
public long Writing => _started.Sum(s => Landed(s.Key) == s.Value.Before ? s.Value.Estimate : 0);
|
|
|
|
|
|
|
|
|
|
public void Staged(long actual, long estimated)
|
|
|
|
|
{
|
|
|
|
|
_actual += actual;
|
|
|
|
|
_estimated += Math.Max(1, estimated);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Why a photo estimated at <paramref name="estimated"/> bytes, converted into <paramref name="target"/>, cannot start now:
|
|
|
|
|
/// the volume lacks the space for it and for the photos started before it whose file has not landed yet
|
|
|
|
|
/// (<see cref="Writing"/>, <see cref="Short"/>), localized. Null when it can: it counts among those until its file lands
|
|
|
|
|
/// or its work ends (<see cref="Ended"/>).
|
|
|
|
|
/// </summary>
|
|
|
|
|
public string? Start(long estimated, string target)
|
|
|
|
|
{
|
|
|
|
|
var before = Landed(target);
|
|
|
|
|
if (Short(Math.Max(0, estimated) + Writing) is { } message) return message;
|
|
|
|
|
_started[target] = new Starting(Math.Max(0, estimated), before);
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>The work of the photo converted into <paramref name="target"/> ended: its file landed, or never will.</summary>
|
|
|
|
|
public void Ended(string target) => _started.TryRemove(target, out _);
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Why the volume cannot take <paramref name="estimated"/> bytes more (scaled, with the margin and the reserve), localized;
|
|
|
|
|
/// null when it can, or when its free space cannot be measured.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public string? Short(long estimated)
|
|
|
|
|
{
|
|
|
|
|
var space = new ConvertSpace(ConvertPlanner.Required((long)Math.Ceiling(estimated * Ratio)), converter.FreeSpace(paths));
|
|
|
|
|
var space = new ConvertSpace(ConvertPlanner.Required((long)Math.Ceiling(estimated * Ratio)), free());
|
|
|
|
|
if (space.Enough) return null;
|
|
|
|
|
var (required, free) = Sizes(space.Required, space.Free ?? 0);
|
|
|
|
|
return Loc.L("Archive.Error.ConvertSpace", required, free);
|
|
|
|
|
var (required, left) = Sizes(space.Required, space.Free ?? 0);
|
|
|
|
|
return Loc.L("Archive.Error.ConvertSpace", required, left);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>The file at <paramref name="path"/> as a landing changes it, its size and time of writing; null when there is none.</summary>
|
|
|
|
|
private static (long Size, long Written)? Landed(string path) =>
|
|
|
|
|
new FileInfo(path) is { Exists: true } file ? (file.Length, file.LastWriteTimeUtc.Ticks) : null;
|
|
|
|
|
|
|
|
|
|
/// <summary>A photo started: its estimate, and the file at its path then.</summary>
|
|
|
|
|
private sealed record Starting(long Estimate, (long Size, long Written)? Before);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>What a sequence came to: its result, the statuses its follow-ups follow from, and those follow-ups (null: none to queue).</summary>
|
|
|
|
@@ -1171,12 +1374,13 @@ internal sealed record PinnedFrame(Guid Id, string Sha256);
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The journal of a conversion job (<c>.chronos/journal/convert-<job>.json</c>, owner review 2, R7): what each sequence
|
|
|
|
|
/// came to, and whether the passes it needs were queued, written after each one; with it, the pin log
|
|
|
|
|
/// (<c>convert-<job>.pins</c>, <see cref="PinLog"/>), the photos the job converts of each sequence, and the staged log
|
|
|
|
|
/// (<c>convert-<job>.staged</c>), one line per photo staged, both appended and flushed as they grow, so a job of thousands
|
|
|
|
|
/// of photos journals each once. A resumed or retried job keeps the sequences that are final, the photos pinned, and the
|
|
|
|
|
/// staged files whose hash still verifies. All of it goes when every sequence is final, or once the job is gone
|
|
|
|
|
/// (<see cref="ArchiveMaintenance"/>).
|
|
|
|
|
/// came to, the new sequence a New sequence conversion made of it as soon as its photos are filed, and whether the passes
|
|
|
|
|
/// it needs were queued, written after each one; with it, the photos confirmed (<c>convert-<job>.photos</c>,
|
|
|
|
|
/// <see cref="ConfirmedPhotos"/>, written when the job is queued), the pin log (<c>convert-<job>.pins</c>,
|
|
|
|
|
/// <see cref="PinLog"/>), the photos the job converts of each sequence, and the staged log (<c>convert-<job>.staged</c>),
|
|
|
|
|
/// one line per photo staged, both appended and flushed as they grow, so a job of thousands of photos journals each once. A
|
|
|
|
|
/// resumed or retried job keeps the sequences that are final, the copies made, the photos pinned, and the staged files
|
|
|
|
|
/// whose hash still verifies. All of it goes when every sequence is final, or once the job is gone (<see cref="ArchiveMaintenance"/>).
|
|
|
|
|
/// </summary>
|
|
|
|
|
internal sealed class ConvertJournal
|
|
|
|
|
{
|
|
|
|
@@ -1198,7 +1402,10 @@ internal sealed class ConvertJournal
|
|
|
|
|
noted.Had = had;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Notes what a sequence came to and, when it was converted or made, the statuses its follow-ups follow from.</summary>
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Notes what a sequence came to and, when it was converted or made, the statuses its follow-ups follow from. The new
|
|
|
|
|
/// sequence a New sequence conversion made of it stays named whatever it came to since (<see cref="Made"/>).
|
|
|
|
|
/// </summary>
|
|
|
|
|
public void Note(ConvertSequenceResult result, SequenceStatus? had = null)
|
|
|
|
|
{
|
|
|
|
|
var noted = Of(result.SequenceId);
|
|
|
|
@@ -1206,10 +1413,21 @@ internal sealed class ConvertJournal
|
|
|
|
|
noted.Title = result.Title;
|
|
|
|
|
noted.Outcome = result.Outcome;
|
|
|
|
|
noted.Frames = result.Frames;
|
|
|
|
|
noted.NewSequenceId = result.NewSequenceId;
|
|
|
|
|
noted.NewSequenceId = result.NewSequenceId ?? noted.NewSequenceId;
|
|
|
|
|
if (had is { } status) noted.Had = status;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Notes the new sequence a New sequence conversion filed the photos of <paramref name="id"/> into, before anything else is
|
|
|
|
|
/// done with it: a retry or a resume finishes that one and makes no other.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public void Made(Guid id, Guid newSequenceId)
|
|
|
|
|
{
|
|
|
|
|
var noted = Of(id);
|
|
|
|
|
if (noted is null) Sequences.Add(noted = new ConvertJournalSequence { Id = id });
|
|
|
|
|
noted.NewSequenceId = newSequenceId;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.json");
|
|
|
|
|
|
|
|
|
|
public static string StagedPathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.staged");
|
|
|
|
@@ -1256,8 +1474,9 @@ internal sealed class ConvertJournal
|
|
|
|
|
/// <summary>The pin log of this job, read now.</summary>
|
|
|
|
|
public PinLog Pins(ArchivePaths paths) => PinLog.Open(PinsPathFor(paths, JobId));
|
|
|
|
|
|
|
|
|
|
/// <summary>The journal files of <paramref name="job"/>, whichever exist.</summary>
|
|
|
|
|
public static IEnumerable<string> Files(ArchivePaths paths, Guid job) => [StagedPathFor(paths, job), PinsPathFor(paths, job), PathFor(paths, job)];
|
|
|
|
|
/// <summary>The journal files of <paramref name="job"/>, whichever exist: the photos confirmed among them (<see cref="ConfirmedPhotos"/>).</summary>
|
|
|
|
|
public static IEnumerable<string> Files(ArchivePaths paths, Guid job) =>
|
|
|
|
|
[StagedPathFor(paths, job), PinsPathFor(paths, job), ConfirmedPhotos.PathFor(paths, job), PathFor(paths, job)];
|
|
|
|
|
|
|
|
|
|
public static void Delete(ArchivePaths paths, Guid job)
|
|
|
|
|
{
|
|
|
|
@@ -1291,7 +1510,7 @@ internal sealed class ConvertJournalSequence
|
|
|
|
|
public bool Queued { get; set; }
|
|
|
|
|
|
|
|
|
|
[JsonIgnore]
|
|
|
|
|
public bool Final => Outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing;
|
|
|
|
|
public bool Final => ConvertSequenceResult.IsFinal(Outcome);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>The staged log of a conversion job (<see cref="ConvertJournal"/>): read once, appended to line by line.</summary>
|
|
|
|
@@ -1380,6 +1599,114 @@ internal sealed class PinLog
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>A line of <see cref="ConfirmedPhotos"/>: a sequence, and the frame ids of the photos converted of it, 16 bytes each, in base64.</summary>
|
|
|
|
|
internal sealed record ConfirmedLine(Guid Sequence, string Photos);
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The photos a conversion job converts of each sequence, as its plan was confirmed (<c>.chronos/journal/convert-<job>.photos</c>,
|
|
|
|
|
/// owner review 2, R7): written whole under the job's id before it is queued (<see cref="MassConverter.QueueAsync"/>), one
|
|
|
|
|
/// line per sequence with the frame ids in the plan's order. The job's row keeps each sequence's digest of those photos
|
|
|
|
|
/// with their content (<see cref="ConvertSequence.Frames"/>), never the ids: it stays as small as the number of sequences
|
|
|
|
|
/// whatever the number of photos, for every list and check that reads jobs. Under each sequence's lease the job takes these
|
|
|
|
|
/// photos from its plan made again and goes on only while they have that digest; a sequence this file does not name (it
|
|
|
|
|
/// could not be written, or a job an earlier build queued) is held to its whole plan. It goes with the journal
|
|
|
|
|
/// (<see cref="ConvertJournal.Files"/>).
|
|
|
|
|
/// </summary>
|
|
|
|
|
internal sealed class ConfirmedPhotos
|
|
|
|
|
{
|
|
|
|
|
private readonly Dictionary<Guid, IReadOnlyList<Guid>> _photos = [];
|
|
|
|
|
|
|
|
|
|
public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{ConvertJournal.Prefix}{job:N}.photos");
|
|
|
|
|
|
|
|
|
|
/// <summary>The photos confirmed of <paramref name="sequence"/>, in the plan's order; null when the file does not name it.</summary>
|
|
|
|
|
public IReadOnlyList<Guid>? Of(Guid sequence) => _photos.GetValueOrDefault(sequence);
|
|
|
|
|
|
|
|
|
|
/// <summary>The photos confirmed for <paramref name="job"/>: none when the file is not there; a line that cannot be read names nothing.</summary>
|
|
|
|
|
public static ConfirmedPhotos Read(ArchivePaths paths, Guid job)
|
|
|
|
|
{
|
|
|
|
|
var read = new ConfirmedPhotos();
|
|
|
|
|
foreach (var line in JournalLines.Read(PathFor(paths, job)))
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
if (JsonSerializer.Deserialize(line, ArchiveJobJson.Default.ConfirmedLine) is { } confirmed) read._photos[confirmed.Sequence] = Decode(confirmed.Photos);
|
|
|
|
|
}
|
|
|
|
|
catch (Exception ex) when (ex is JsonException or FormatException)
|
|
|
|
|
{
|
|
|
|
|
Core.Diagnostics.Log.Warning("archive", $"A line of the photos confirmed for conversion job {job} cannot be read; that sequence is held to its whole plan", ex);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return read;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Writes the photos <paramref name="plan"/> converts of each sequence it converts, for the job <paramref name="job"/>, whole
|
|
|
|
|
/// (a reader finds all of it or none); false, logged, when it cannot be written.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public static bool TryWrite(ArchivePaths paths, Guid job, ConvertPlan plan)
|
|
|
|
|
{
|
|
|
|
|
ArgumentNullException.ThrowIfNull(plan);
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
using var lines = new MemoryStream();
|
|
|
|
|
foreach (var sequence in plan.Converted)
|
|
|
|
|
{
|
|
|
|
|
lines.Write(JsonSerializer.SerializeToUtf8Bytes(new ConfirmedLine(sequence.SequenceId, Encode(sequence.Frames.Select(f => f.FrameId))),
|
|
|
|
|
ArchiveJobJson.Default.ConfirmedLine));
|
|
|
|
|
lines.WriteByte((byte)'\n');
|
|
|
|
|
}
|
|
|
|
|
System.IO.Directory.CreateDirectory(paths.JournalDirectory);
|
|
|
|
|
AtomicFile.WriteAllBytes(PathFor(paths, job), lines.GetBuffer().AsSpan(0, (int)lines.Length));
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
|
|
|
{
|
|
|
|
|
Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be written; it holds each sequence to its whole plan", ex);
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Deletes the file of <paramref name="job"/>; a failure is logged, and maintenance removes it later.</summary>
|
|
|
|
|
public static void Delete(ArchivePaths paths, Guid job)
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
File.Delete(PathFor(paths, job));
|
|
|
|
|
}
|
|
|
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
|
|
|
{
|
|
|
|
|
Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be deleted; maintenance removes them later", ex);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Frame ids as a line holds them: their 16 bytes each, one after the other, in base64.</summary>
|
|
|
|
|
public static string Encode(IEnumerable<Guid> ids)
|
|
|
|
|
{
|
|
|
|
|
ArgumentNullException.ThrowIfNull(ids);
|
|
|
|
|
IReadOnlyCollection<Guid> frames = ids as IReadOnlyCollection<Guid> ?? [.. ids];
|
|
|
|
|
var bytes = new byte[frames.Count * 16];
|
|
|
|
|
var at = 0;
|
|
|
|
|
foreach (var frame in frames)
|
|
|
|
|
{
|
|
|
|
|
frame.TryWriteBytes(bytes.AsSpan(at, 16));
|
|
|
|
|
at += 16;
|
|
|
|
|
}
|
|
|
|
|
return Convert.ToBase64String(bytes);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>The frame ids <see cref="Encode"/> wrote, in order.</summary>
|
|
|
|
|
/// <exception cref="FormatException">The text is not frame ids as <see cref="Encode"/> writes them.</exception>
|
|
|
|
|
public static IReadOnlyList<Guid> Decode(string text)
|
|
|
|
|
{
|
|
|
|
|
ArgumentNullException.ThrowIfNull(text);
|
|
|
|
|
var bytes = Convert.FromBase64String(text);
|
|
|
|
|
if (bytes.Length % 16 != 0) throw new FormatException("The text is not frame ids.");
|
|
|
|
|
var ids = new Guid[bytes.Length / 16];
|
|
|
|
|
for (var i = 0; i < ids.Length; i++) ids[i] = new Guid(bytes.AsSpan(i * 16, 16));
|
|
|
|
|
return ids;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Files of JSON lines a job appends to as it goes: each line written whole and flushed to disk, read back skipping blank ones.</summary>
|
|
|
|
|
internal static class JournalLines
|
|
|
|
|
{
|
|
|
|
|