CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
1734 lines
102 KiB
C#
1734 lines
102 KiB
C#
using System.Collections.Concurrent;
|
||
using System.Security.Cryptography;
|
||
using System.Text;
|
||
using System.Text.Json;
|
||
using System.Text.Json.Serialization;
|
||
using Chronos.Core.Archiving;
|
||
using Chronos.Core.Conversion;
|
||
using Chronos.Core.Diagnostics;
|
||
using Chronos.Core.Imaging;
|
||
using Chronos.Core.IO;
|
||
using Chronos.Core.Localization;
|
||
using Chronos.Core.Metadata;
|
||
using Chronos.Core.Model;
|
||
using Chronos.Core.Naming;
|
||
using Chronos.Core.Project;
|
||
using Chronos.Data.Archiving.Jobs;
|
||
using Chronos.Data.Processing;
|
||
|
||
namespace Chronos.Data.Archiving;
|
||
|
||
/// <summary>What a conversion did with one sequence (owner review 2, R7).</summary>
|
||
public enum ConvertOutcome
|
||
{
|
||
/// <summary>Its frames were replaced by the converted files (<see cref="ConvertMode.Replace"/>).</summary>
|
||
Converted,
|
||
|
||
/// <summary>The converted files went into a new sequence (<see cref="ConvertMode.NewSequence"/>).</summary>
|
||
Created,
|
||
|
||
/// <summary>It had nothing to convert.</summary>
|
||
Nothing,
|
||
|
||
/// <summary>
|
||
/// Done with, no copy made (<see cref="ConvertMode.NewSequence"/>): another conversion made its copy in this format since
|
||
/// this one was confirmed (a retry of an older conversion, or a duplicate queued beside it), so a second copy is not made.
|
||
/// </summary>
|
||
Superseded,
|
||
|
||
/// <summary>
|
||
/// Left as it was: a photo it converts no longer holds the content confirmed (it changed, is gone, or no longer converts),
|
||
/// or it is no longer in the archive. A retry finds the same: the user plans again.
|
||
/// </summary>
|
||
Changed,
|
||
|
||
/// <summary>
|
||
/// Left as it was for now: it is busy, another index moved its folder, a photo it converts is held by another program, or
|
||
/// its sidecar cannot be read or settled; a retry converts it once that is over.
|
||
/// </summary>
|
||
Blocked,
|
||
|
||
/// <summary>Left as it was: a photo could not be converted (all or nothing), or the files could not move.</summary>
|
||
Failed,
|
||
}
|
||
|
||
/// <summary>
|
||
/// One sequence of a conversion: what happened, how many photos, the new sequence when one was made, the failure, and the
|
||
/// photos of the source a new sequence does not hold (those the plan did not convert stay in the source alone).
|
||
/// </summary>
|
||
public sealed record ConvertSequenceResult(Guid SequenceId, string Title, ConvertOutcome Outcome, int Frames, Guid? NewSequenceId = null, string? Error = null,
|
||
int LeftOut = 0)
|
||
{
|
||
/// <summary>Whether the sequence is done with: converted, made, with nothing to convert, or copied by another conversion since.</summary>
|
||
public bool Final => IsFinal(Outcome);
|
||
|
||
/// <summary>Whether <paramref name="outcome"/> leaves a sequence done with (<see cref="Final"/>).</summary>
|
||
internal static bool IsFinal(ConvertOutcome? outcome) =>
|
||
outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing or ConvertOutcome.Superseded;
|
||
}
|
||
|
||
/// <summary>What a conversion did, and the passes the sequences it changed or made need (queued as follow-ups as each one ended).</summary>
|
||
public sealed record ConvertResult(IReadOnlyList<ConvertSequenceResult> Sequences, IReadOnlyList<ArchiveJobRequest> FollowUps)
|
||
{
|
||
/// <summary>Photos converted, into their sequence or a new one.</summary>
|
||
public int Frames => Sequences.Where(s => s.Outcome is ConvertOutcome.Converted or ConvertOutcome.Created).Sum(s => s.Frames);
|
||
|
||
/// <summary>The sequences left as they were (changed, blocked, failed).</summary>
|
||
public IEnumerable<ConvertSequenceResult> Kept => Sequences.Where(s => !s.Final);
|
||
}
|
||
|
||
/// <summary>
|
||
/// How a running conversion reports: progress in photos, a checkpoint after each sequence, job log lines, each sequence it
|
||
/// came to an end with in this run once its outcome is journaled (to tell the sequences it changed or made while the rest
|
||
/// of the job runs, as a Reorganize tells each step), and the passes a sequence it converted or made needs, to queue then
|
||
/// too (the journal says whether they were, so a pause, a yield, a stop or a crash later loses none).
|
||
/// </summary>
|
||
internal sealed record ConvertCallbacks(Func<long, long, ValueTask> Progress, Func<long, long, ValueTask> Checkpoint, Action<string> Log,
|
||
Func<ConvertSequenceResult, ValueTask>? Done = null, Func<IReadOnlyList<ArchiveJobRequest>, ValueTask>? Queue = null);
|
||
|
||
/// <summary>
|
||
/// Who asked for a conversion: the user's name, kept in the trash records of a Replace, and id, the owner of a new sequence
|
||
/// when its source has none; and when the plan was confirmed (the job queued): a copy another conversion made of a source
|
||
/// since then supersedes this one's (<see cref="ConvertOutcome.Superseded"/>). Null: no copy made by another counts.
|
||
/// </summary>
|
||
internal sealed record ConvertRequester(string? Name, Guid? Id, DateTimeOffset? Confirmed = null);
|
||
|
||
/// <summary>
|
||
/// A conversion of sequences a confirmed plan converts is queued, running or paused already, so its job was not queued
|
||
/// (<see cref="MassConverter.QueueAsync"/>): a dry run made now shows them busy.
|
||
/// </summary>
|
||
public sealed class ConversionQueuedException(IReadOnlyList<Guid> sequenceIds) : InvalidOperationException(Loc.L("Archive.Convert.AlreadyQueued"))
|
||
{
|
||
/// <summary>The sequences another conversion holds.</summary>
|
||
public IReadOnlyList<Guid> SequenceIds { get; } = sequenceIds;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Mass conversion of existing sequences (owner review 2, R7; BRIEF §12.5): <see cref="PlanAsync(Archive, IReadOnlyList{Guid}, ConvertOptions, ConvertMode, CancellationToken)"/>
|
||
/// is the dry run (<see cref="ConvertPlanner"/> per sequence, what blocks it, the space needed against the free space where
|
||
/// the conversion writes, <see cref="ArchiveServiceOptions.FreeSpace"/>), and a <see cref="JobKind.Convert"/> job applies the
|
||
/// confirmed plan (queued by <see cref="QueueAsync"/>, which refuses a second conversion of the same sequences) one sequence
|
||
/// at a time. Under the sequence's lease, its sidecar settled where the index has it then (<see cref="SequenceWriter.ReadLocked"/>),
|
||
/// the job plans it again and takes the photos confirmed (<see cref="ConfirmedPhotos"/>), going on only while each is
|
||
/// still planned with its content confirmed (<see cref="ConvertSequence.Frames"/>); it pins them then (<see cref="PinLog"/>),
|
||
/// so a photo that came in, was verified or could be opened again since the plan stays as it is, on a retry too. Stage A,
|
||
/// without the lease: each photo is hashed against its record and converted (<see cref="FrameConverter"/>, several at once
|
||
/// as the host's parallelism and memory allow) into <c>.chronos/staging/convert-<job>/<sequence>/</c>, the free
|
||
/// space checked before each one, each staged file appended to the job's journal (<see cref="ConvertJournal"/>), so a resumed or retried job keeps the staged files
|
||
/// whose hash still verifies. Stage B, all or nothing, under the sequence's lease and while every photo pinned still holds
|
||
/// the content converted: <see cref="ConvertMode.Replace"/> swaps the frames' files (<see cref="FrameTrash.ReplaceAsync"/>:
|
||
/// the originals into one trash entry of kind <see cref="TrashKind.Replaced"/>, same ids and indices, tombstones naming the
|
||
/// replacements; the converted files named as filing names a photo of the sequence, <see cref="ConvertOptions.FileName"/>);
|
||
/// <see cref="ConvertMode.NewSequence"/> files the converted copies into a new sequence through the archive ingest (session
|
||
/// key <c>convert:<job>:<source></c>, so a resume finds it again), titled "title (FORMAT)", owned as the source is,
|
||
/// with the source's tags, rating, notes and project, the source left as it is (the photos the plan did not convert stay
|
||
/// there alone, as the job log says). A retry or a resume finishes the copy it made already rather than make another, and
|
||
/// makes none of a source that another conversion copied in the same format since this one was confirmed
|
||
/// (<see cref="ConvertOutcome.Superseded"/>). A sequence whose photos changed, that is busy, or has a photo that fails is
|
||
/// left as it was and reported.
|
||
/// </summary>
|
||
public sealed class MassConverter
|
||
{
|
||
/// <summary>What the frames of a sequence no longer have once their files were replaced (<see cref="FrameTrash.ReplaceAsync"/>).</summary>
|
||
private const SequenceStatus Uncovered = SequenceStatus.AnalyzedLight | SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
|
||
|
||
/// <summary>How the session key of a new sequence a conversion makes begins (<see cref="SessionKey"/>).</summary>
|
||
private const string KeyPrefix = "convert:";
|
||
|
||
/// <summary>Held while a confirmed conversion is checked against those pending and queued (<see cref="QueueAsync"/>): one at a time in this process.</summary>
|
||
private static readonly SemaphoreSlim Queueing = new(1, 1);
|
||
|
||
private readonly ChronosDatabase _db;
|
||
private readonly SequenceWriter _writer;
|
||
private readonly ArchiveIngest _ingest;
|
||
private readonly FrameTrash _frames;
|
||
private readonly SequenceTrash _trash;
|
||
private readonly ArchiveService _service;
|
||
|
||
internal MassConverter(ChronosDatabase db, SequenceWriter writer, ArchiveIngest ingest, FrameTrash frames, SequenceTrash trash, ArchiveService service)
|
||
{
|
||
_db = db;
|
||
_writer = writer;
|
||
_ingest = ingest;
|
||
_frames = frames;
|
||
_trash = trash;
|
||
_service = service;
|
||
}
|
||
|
||
/// <summary>
|
||
/// The dry run of converting <paramref name="sequenceIds"/> of <paramref name="archive"/>: per sequence the frames
|
||
/// converted and those skipped with why, the bytes before and after, what blocks it (busy, a conversion of it already
|
||
/// queued, offline, unreadable); the space needed and free; the fingerprint a confirmation sends back. Reads only.
|
||
/// </summary>
|
||
/// <exception cref="ArgumentException">The options are invalid, or a sequence is not in the archive.</exception>
|
||
public Task<ConvertPlan> PlanAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, ConvertOptions options, ConvertMode mode,
|
||
CancellationToken cancellationToken = default) => PlanAsync(archive, sequenceIds, options, mode, null, cancellationToken);
|
||
|
||
/// <param name="job">The job asking (a conversion planning again): what it holds does not make its sequences busy.</param>
|
||
internal async Task<ConvertPlan> PlanAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, ConvertOptions options, ConvertMode mode, Guid? job,
|
||
CancellationToken cancellationToken)
|
||
{
|
||
ArgumentNullException.ThrowIfNull(archive);
|
||
ArgumentNullException.ThrowIfNull(sequenceIds);
|
||
ArgumentNullException.ThrowIfNull(options);
|
||
if (!options.IsValid) throw new ArgumentException("Unknown target or a quality outside 1–100.", nameof(options));
|
||
if (!Enum.IsDefined(mode)) throw new ArgumentException($"Unknown mode {mode}.", nameof(mode));
|
||
var paths = new ArchivePaths(archive.RootPath);
|
||
var online = ArchiveVolume.Probe(paths) == ArchiveAvailability.Online;
|
||
var busy = await HeldAsync(job, cancellationToken).ConfigureAwait(false);
|
||
var plans = new List<ConvertSequencePlan>();
|
||
foreach (var id in sequenceIds.Distinct())
|
||
{
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
var row = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
|
||
if (row is null || row.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Name}.", nameof(sequenceIds));
|
||
plans.Add(await PlanOneAsync(paths, row, online, busy.Contains(id) || busy.Contains(archive.Id), options, cancellationToken).ConfigureAwait(false));
|
||
}
|
||
var estimated = plans.Where(p => p.Converts).Sum(p => p.BytesAfter);
|
||
return new ConvertPlan(archive.Id, options, mode, plans, new ConvertSpace(ConvertPlanner.Required(estimated), FreeSpace(paths)));
|
||
}
|
||
|
||
/// <summary>
|
||
/// The sequences and archives a dry run counts as busy (<see cref="ConvertBlock.Busy"/>): what running and paused jobs hold,
|
||
/// and the sequences of every conversion waiting in the queue, so the same conversion is not confirmed twice. Reads only.
|
||
/// </summary>
|
||
public async Task<IReadOnlySet<Guid>> BusyAsync(CancellationToken cancellationToken = default) =>
|
||
await HeldAsync(null, cancellationToken).ConfigureAwait(false);
|
||
|
||
/// <summary>
|
||
/// What makes a sequence busy for a dry run (<paramref name="job"/> null: <see cref="BusyAsync"/>), or for the conversion
|
||
/// job <paramref name="job"/> planning again: what other running and paused jobs hold, and the sequences of the conversions
|
||
/// queued before it, which come first (a duplicate queued after it must not hold it back, and one queued before it, by an
|
||
/// earlier build or another tab, must not make a second copy of the same sequence).
|
||
/// </summary>
|
||
private async Task<HashSet<Guid>> HeldAsync(Guid? job, CancellationToken cancellationToken)
|
||
{
|
||
var busy = new HashSet<Guid>(await _trash.BusyAsync(job, cancellationToken).ConfigureAwait(false));
|
||
if (job is not { } self)
|
||
busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken).ConfigureAwait(false));
|
||
else if (await _db.Jobs.GetCreatedAsync(self, cancellationToken).ConfigureAwait(false) is { } queued)
|
||
busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, queued, cancellationToken).ConfigureAwait(false));
|
||
return busy;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Queues the job of the confirmed <paramref name="plan"/> (<see cref="ArchiveJobs.Convert"/>) through <paramref name="enqueue"/>,
|
||
/// unless a conversion of a sequence it converts is queued, running or paused already: another dialog or tab, an answer
|
||
/// lost and Apply pressed again, a dry run made before that one was queued or none at all. Checked and queued under one
|
||
/// lock of this process, so two confirmations at once queue one job. First, which photos it converts are written under
|
||
/// the job's id beside the journal it starts (<see cref="ConfirmedPhotos"/>); <paramref name="enqueue"/> queues the
|
||
/// request under that id (<see cref="ArchiveJobRequest.Id"/>), and runs in the caller's context (a desktop's tells the
|
||
/// user on its UI thread). When they cannot be written (the archive went away since its dry run), the job is queued all
|
||
/// the same and holds each sequence to its whole plan.
|
||
/// </summary>
|
||
/// <exception cref="ConversionQueuedException">A conversion of one of its sequences waits or runs; nothing was queued.</exception>
|
||
/// <exception cref="ArgumentException">The plan converts nothing.</exception>
|
||
public async Task QueueAsync(Archive archive, ConvertPlan plan, Func<ArchiveJobRequest, Task> enqueue, CancellationToken cancellationToken = default)
|
||
{
|
||
ArgumentNullException.ThrowIfNull(enqueue);
|
||
var request = ArchiveJobs.Convert(archive, plan);
|
||
var paths = new ArchivePaths(archive.RootPath);
|
||
var job = request.Id!.Value;
|
||
// No ConfigureAwait(false) here: each wait comes back to the caller's context, where enqueue is called.
|
||
await Queueing.WaitAsync(cancellationToken);
|
||
try
|
||
{
|
||
var pending = await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken);
|
||
if (request.Targets.Where(pending.Contains).ToList() is { Count: > 0 } queued) throw new ConversionQueuedException(queued);
|
||
var written = await Task.Run(() => ConfirmedPhotos.TryWrite(paths, job, plan), cancellationToken);
|
||
try
|
||
{
|
||
await enqueue(request);
|
||
}
|
||
catch
|
||
{
|
||
// Not queued: no job will read them.
|
||
if (written) ConfirmedPhotos.Delete(paths, job);
|
||
throw;
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
Queueing.Release();
|
||
}
|
||
}
|
||
|
||
private async Task<ConvertSequencePlan> PlanOneAsync(ArchivePaths paths, Sequence row, bool online, bool busy, ConvertOptions options,
|
||
CancellationToken cancellationToken)
|
||
{
|
||
List<ConvertBlock> blocked = [];
|
||
if (!online) blocked.Add(ConvertBlock.Offline);
|
||
if (busy) blocked.Add(ConvertBlock.Busy);
|
||
var empty = new ConvertSequencePlan(row.Id, row.DisplayTitle, row.Folder, row.FramesVersion, [], [], blocked);
|
||
// Created without photos (no folder yet), or nothing to read while offline.
|
||
if (!online || row.FrameCount == 0 && ArchiveIngest.AwaitsFolder(row, paths)) return empty;
|
||
var folder = paths.Sequence(row.Folder);
|
||
if (Read(folder, row.Id) is not { } manifest) return empty with { Blocked = [.. blocked, ConvertBlock.Unreadable] };
|
||
var decoder = _writer.Options.Decoder;
|
||
var plan = await Task.Run(() => ConvertPlanner.Plan(manifest, folder, FileRenames.ListPresent(folder, manifest), options, decoder, row.DisplayTitle),
|
||
cancellationToken).ConfigureAwait(false);
|
||
return plan with { Blocked = blocked };
|
||
}
|
||
|
||
/// <summary>
|
||
/// Applies a confirmed conversion (<see cref="ConvertJobParameters"/>) as the job <paramref name="jobId"/>, resuming from its
|
||
/// journal: settles the sidecars a crash left half-way, refuses when the volume lacks the space the photos still to convert
|
||
/// need, then converts each sequence while what it converts is still what was confirmed (stage A, then stage B), the space
|
||
/// measured again before each. The passes a sequence converted or made needs are queued as soon as it is journaled.
|
||
/// Sequences left as they were are in the result; the journal and their staged files stay while there are any, so a retry
|
||
/// tries them again and knows what was done.
|
||
/// </summary>
|
||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only; the job resumes where it stopped.</exception>
|
||
/// <exception cref="IOException">The volume lacks the space (<c>Archive.Error.ConvertSpace</c>); what was converted stays, and a retry resumes.</exception>
|
||
internal async Task<ConvertResult> RunAsync(Archive archive, ConvertJobParameters parameters, Guid jobId, ConvertRequester by, ConvertCallbacks callbacks,
|
||
CancellationToken cancellationToken)
|
||
{
|
||
ArgumentNullException.ThrowIfNull(archive);
|
||
ArgumentNullException.ThrowIfNull(parameters);
|
||
ArgumentNullException.ThrowIfNull(by);
|
||
ArgumentNullException.ThrowIfNull(callbacks);
|
||
var options = new ConvertOptions(parameters.Target, parameters.Quality);
|
||
if (!options.IsValid || !Enum.IsDefined(parameters.Mode)) throw new InvalidOperationException($"Convert job {jobId} has invalid options.");
|
||
var paths = new ArchivePaths(archive.RootPath);
|
||
ArchiveVolume.EnsureWritable(paths);
|
||
var journal = ConvertJournal.Read(paths, jobId) ?? new ConvertJournal { JobId = jobId };
|
||
var steps = parameters.Sequences.DistinctBy(s => s.Id).ToList();
|
||
var staged = journal.Staged(paths);
|
||
var pins = journal.Pins(paths);
|
||
var confirmed = ConfirmedPhotos.Read(paths, jobId);
|
||
// What is still to do: every sequence not final in an earlier attempt that is still in the archive, its sidecar settled
|
||
// first when a writer a crash interrupted left a journal in its folder (this job's replacement among them).
|
||
var rows = new Dictionary<Guid, Sequence>();
|
||
foreach (var step in steps.Where(s => journal.Of(s.Id)?.Final != true))
|
||
{
|
||
if (await _db.Sequences.GetAsync(step.Id, cancellationToken).ConfigureAwait(false) is { } row && row.ArchiveId == archive.Id)
|
||
rows[step.Id] = await SettledRowAsync(archive, paths, row, cancellationToken).ConfigureAwait(false);
|
||
}
|
||
var plan = await PlanAsync(archive, [.. rows.Keys], options, parameters.Mode, jobId, cancellationToken).ConfigureAwait(false);
|
||
if (plan.Sequences.Any(p => p.Blocked.Contains(ConvertBlock.Offline))) throw new ArchiveOfflineException(paths.Root, ArchiveVolume.Probe(paths));
|
||
// What each of them converts, as far as the job can tell before it comes to it (what it pinned, or the plan still confirmed),
|
||
// and the bytes its photos not staged yet are estimated at.
|
||
var byId = steps.ToDictionary(s => s.Id);
|
||
var progress = new Progress();
|
||
var unstaged = new Dictionary<Guid, long>();
|
||
foreach (var p in plan.Sequences.Where(p => p.Blocked.Count == 0))
|
||
{
|
||
if (Confirmed(byId[p.SequenceId], confirmed.Of(p.SequenceId), p, pins.Of(p.SequenceId)) is not { Frames.Count: > 0 } converts) continue;
|
||
progress.Count(converts.SequenceId, converts.Frames.Count, staged.Of(converts.SequenceId, converts.FramesVersion).Count);
|
||
// What a new sequence this job made holds already takes no space again.
|
||
var filed = parameters.Mode == ConvertMode.NewSequence
|
||
? await FiledAsync(archive, SessionKey(jobId, converts.SequenceId), cancellationToken).ConfigureAwait(false)
|
||
: null;
|
||
unstaged[converts.SequenceId] = Unstaged(paths, jobId, converts, staged, filed);
|
||
}
|
||
var space = new SpaceWatch(() => FreeSpace(paths));
|
||
EnsureSpace(unstaged.Values.Sum(), space);
|
||
await callbacks.Progress(progress.Done, progress.Total).ConfigureAwait(false);
|
||
|
||
var results = new List<ConvertSequenceResult>();
|
||
var followUps = new List<ArchiveJobRequest>();
|
||
foreach (var step in steps)
|
||
{
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
var noted = journal.Of(step.Id);
|
||
var final = noted?.Final == true;
|
||
StepDone done;
|
||
if (final)
|
||
{
|
||
// Done with in an earlier attempt, and the passes it needs too unless a pause, a yield, a stop or a crash came first.
|
||
done = new StepDone(new ConvertSequenceResult(step.Id, noted!.Title ?? step.Id.ToString(), noted.Outcome!.Value, noted.Frames, noted.NewSequenceId),
|
||
noted.Had, noted.Queued ? null : await FollowUpsAsync(archive, noted, cancellationToken).ConfigureAwait(false));
|
||
}
|
||
else if (!rows.TryGetValue(step.Id, out var row))
|
||
{
|
||
done = new StepDone(new ConvertSequenceResult(step.Id, step.Id.ToString(), ConvertOutcome.Changed, 0));
|
||
}
|
||
else
|
||
{
|
||
ArchiveVolume.EnsureWritable(paths);
|
||
// The space for what is left, measured again: the volume may have filled since, and the files staged so far tell
|
||
// how far off the estimate is.
|
||
EnsureSpace(unstaged.Values.Sum(), space);
|
||
done = await ConvertOneAsync(archive, paths, step, row, options, parameters.Mode, jobId, by, journal, confirmed, pins, staged, progress, space,
|
||
callbacks, cancellationToken).ConfigureAwait(false);
|
||
}
|
||
unstaged.Remove(step.Id);
|
||
var result = done.Result;
|
||
results.Add(result);
|
||
if (!final)
|
||
{
|
||
journal.Note(result, done.Had);
|
||
journal.Write(paths);
|
||
// What a sequence done with left staged, or staged for photos that are no longer the ones confirmed, is of no use
|
||
// to anything. A sequence blocked or failed keeps its staged files, which a retry verifies and takes.
|
||
if (result.Final || result.Outcome == ConvertOutcome.Changed) DeleteStaging(StagingOf(paths, jobId, step.Id));
|
||
Log(callbacks, result);
|
||
if (callbacks.Done is { } told) await told(result).ConfigureAwait(false);
|
||
}
|
||
if (done.FollowUps is { } passes)
|
||
{
|
||
if (passes.Count > 0)
|
||
{
|
||
followUps.AddRange(passes);
|
||
if (callbacks.Queue is { } queue) await queue(passes).ConfigureAwait(false);
|
||
}
|
||
journal.Of(step.Id)!.Queued = true;
|
||
journal.Write(paths);
|
||
}
|
||
await callbacks.Checkpoint(progress.Done, progress.Total).ConfigureAwait(false);
|
||
}
|
||
// Done with every sequence: what the job staged and its journal go. Otherwise both stay for a retry, until the job is gone
|
||
// from the history (ArchiveMaintenance).
|
||
if (results.All(r => r.Final))
|
||
{
|
||
DeleteStaging(Path.Combine(paths.StagingDirectory, StagingFolder(jobId)));
|
||
ConvertJournal.Delete(paths, jobId);
|
||
}
|
||
return new ConvertResult(results, followUps);
|
||
}
|
||
|
||
/// <summary>
|
||
/// One sequence: under its lease, its sidecar settled, planned again and held to what was confirmed (the photos pinned the
|
||
/// first time); then stage A without the lease, and stage B. A New sequence that has its copy already is not converted
|
||
/// again (<see cref="CopiedAsync"/>).
|
||
/// </summary>
|
||
private async Task<StepDone> ConvertOneAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options,
|
||
ConvertMode mode, Guid jobId, ConvertRequester by, ConvertJournal journal, ConfirmedPhotos confirmedPhotos, PinLog pins, StagedLog staged,
|
||
Progress progress, SpaceWatch space, ConvertCallbacks callbacks, CancellationToken cancellationToken)
|
||
{
|
||
// Another job holds it now (a Reorganize paused since the job started, a render, a conversion of it queued first): it
|
||
// waits for the retry.
|
||
var busy = await HeldAsync(jobId, cancellationToken).ConfigureAwait(false);
|
||
if (busy.Contains(step.Id) || busy.Contains(archive.Id)) return new StepDone(Result(row, ConvertOutcome.Blocked));
|
||
var key = SessionKey(jobId, row.Id);
|
||
if (mode == ConvertMode.NewSequence)
|
||
{
|
||
try
|
||
{
|
||
if (await CopiedAsync(archive, paths, step, row, options, jobId, by, journal, pins, staged, key, progress, cancellationToken).ConfigureAwait(false)
|
||
is { } copied)
|
||
return copied;
|
||
}
|
||
catch (ArchiveOfflineException)
|
||
{
|
||
throw;
|
||
}
|
||
catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the copy this conversion made of it could not be finished yet", ex);
|
||
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex)));
|
||
}
|
||
}
|
||
var photos = confirmedPhotos.Of(step.Id);
|
||
SequencePaths folder;
|
||
SequenceManifest manifest;
|
||
ConvertSequencePlan planned;
|
||
try
|
||
{
|
||
await using (await _writer.LockAsync(paths, step.Id, cancellationToken).ConfigureAwait(false))
|
||
{
|
||
// Where the index has it now (a Reorganize may have moved its folder since the job planned), its sidecar as every
|
||
// writer reads it: a rename of its files, a deletion, a replacement, a restore or an ingest a crash left half-way is
|
||
// settled first, so the plan sees each photo where it is.
|
||
if (await _db.Sequences.GetAsync(step.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id)
|
||
return new StepDone(Result(row, ConvertOutcome.Changed));
|
||
row = now;
|
||
folder = paths.Sequence(row.Folder);
|
||
// Moved away by another index of the archive, maybe with another sequence in the folder it left (this index's next
|
||
// scan finds both): nothing of it is settled, staged or replaced there; the retry after that scan converts it.
|
||
if (!SequenceFolders.Holds(folder, row.Id)) return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Errors.Archive.FolderMoved")));
|
||
manifest = SequenceWriter.ReadLocked(folder, row.Id);
|
||
FrameTrash.EnsureReplaced(folder);
|
||
var pin = pins.Of(step.Id);
|
||
if (mode == ConvertMode.Replace && ReplacedEarlier(manifest, step, photos, pin, staged.Of(step.Id, step.FramesVersion), options) is { } frames)
|
||
{
|
||
// Replaced by this job already: a crash after its stage B, before the journal said so (the settle may just have
|
||
// completed it).
|
||
var synced = await SyncedAsync(archive, folder, row, manifest).ConfigureAwait(false);
|
||
var had = journal.Of(step.Id)?.Had ?? row.Status;
|
||
return new StepDone(Result(synced, ConvertOutcome.Converted, frames), had, _frames.FollowUps(archive, synced, had));
|
||
}
|
||
var (sidecar, at, title, decoder) = (manifest, folder, row.DisplayTitle, _writer.Options.Decoder);
|
||
var (fresh, present) = await Task.Run(() =>
|
||
{
|
||
// As the planner keeps them (by name, whatever the case), so the photos left out are asked again alike.
|
||
IReadOnlySet<string> files = new HashSet<string>(FileRenames.ListPresent(at, sidecar), StringComparer.OrdinalIgnoreCase);
|
||
return (ConvertPlanner.Plan(sidecar, at, files, options, decoder, title), files);
|
||
}, cancellationToken).ConfigureAwait(false);
|
||
if (Confirmed(step, photos, fresh, pin) is not { } confirmed)
|
||
{
|
||
// A photo it converts that another program holds now cannot be told from one that changed: the retry, once it
|
||
// is free, can.
|
||
if (await Task.Run(() => Held(photos, pin, sidecar, at, present, fresh, options, decoder), cancellationToken).ConfigureAwait(false))
|
||
{
|
||
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: a photo it converts cannot be opened now; it is not converted");
|
||
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.PhotoHeld")));
|
||
}
|
||
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the photos it converts are no longer those of the plan confirmed; it is not converted");
|
||
return new StepDone(Result(row, ConvertOutcome.Changed));
|
||
}
|
||
planned = confirmed;
|
||
if (pin is null)
|
||
{
|
||
// From here on the job converts these photos of it, whatever comes in or is verified meanwhile, on a retry too.
|
||
pins.Append(step.Id, planned);
|
||
journal.Begin(step.Id, row.DisplayTitle, row.Status);
|
||
journal.Write(paths);
|
||
}
|
||
}
|
||
}
|
||
catch (ArchiveOfflineException)
|
||
{
|
||
throw;
|
||
}
|
||
catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"The sidecar of {row.DisplayTitle} cannot be read; it is not converted", ex);
|
||
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex)));
|
||
}
|
||
var staging = StagingOf(paths, jobId, row.Id);
|
||
// A new sequence a resumed job finds again holds the photos it filed before it stopped.
|
||
var filed = mode == ConvertMode.NewSequence
|
||
? await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false)
|
||
: new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||
var noted = staged.Of(row.Id, planned.FramesVersion);
|
||
var ready = await Task.Run(() => Ready(manifest, planned, noted, staging, filed), cancellationToken).ConfigureAwait(false);
|
||
progress.Count(row.Id, planned.Frames.Count, ready.Count);
|
||
await callbacks.Progress(progress.Done, progress.Total).ConfigureAwait(false);
|
||
|
||
var failure = await StageAsync(row, manifest, folder, planned, ready, staging, options, staged, progress, space, callbacks, cancellationToken)
|
||
.ConfigureAwait(false);
|
||
if (failure is not null) return new StepDone(Result(row, ConvertOutcome.Failed, error: failure));
|
||
try
|
||
{
|
||
return mode == ConvertMode.Replace
|
||
? await ReplaceAsync(archive, paths, row, planned, ready, staging, options, jobId, by.Name, cancellationToken).ConfigureAwait(false)
|
||
: await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, journal, cancellationToken).ConfigureAwait(false);
|
||
}
|
||
catch (LeaseUnavailableException ex)
|
||
{
|
||
return new StepDone(Result(row, ConvertOutcome.Blocked, error: ex.Message));
|
||
}
|
||
catch (ArchiveOfflineException)
|
||
{
|
||
throw;
|
||
}
|
||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the converted photos could not take their place", ex);
|
||
var cause = ex switch
|
||
{
|
||
UnstagedException => ex.Message,
|
||
UnauthorizedAccessException => Loc.L("Archive.Failed.Access"),
|
||
_ => Loc.L("Archive.Failed.Files"),
|
||
};
|
||
return new StepDone(Result(row, ConvertOutcome.Failed, error: Loc.L("Archive.Convert.NotPlaced", cause)));
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Why a sequence's sidecar could not be settled or read under its lease, as one catalog sentence for the job log (the
|
||
/// exception goes to the diagnostics log): another writer holds it, access is denied, or the file is not its sidecar.
|
||
/// </summary>
|
||
private static string Blocker(Exception ex) => ex switch
|
||
{
|
||
LeaseUnavailableException => ex.Message,
|
||
UnauthorizedAccessException => Loc.L("Archive.Failed.Access"),
|
||
InvalidDataException or InvalidOperationException => Loc.L("Archive.Convert.Blocked.Unreadable"),
|
||
_ => Loc.L("Archive.Failed.Files"),
|
||
};
|
||
|
||
/// <summary>
|
||
/// What a <see cref="ConvertMode.NewSequence"/> conversion does with a source that has its copy already when the job comes
|
||
/// to it (a retry, a resume): the copy this job made (its journal names it, or its session key when a crash came before
|
||
/// the journal did) is finished, whatever changed in the source since, once it holds every photo the job pinned, and one
|
||
/// gone since is not made again without a new plan; a copy in this format that another conversion made since this one was
|
||
/// confirmed (<see cref="ConvertRequester.Confirmed"/>) supersedes this one's. Null when neither: the sequence converts,
|
||
/// and a copy holding only some of its photos (a crash while they were filed) gets the rest.
|
||
/// </summary>
|
||
private async Task<StepDone?> CopiedAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options, Guid jobId,
|
||
ConvertRequester by, ConvertJournal journal, PinLog pins, StagedLog staged, string key, Progress progress, CancellationToken cancellationToken)
|
||
{
|
||
if ((journal.Of(step.Id)?.NewSequenceId ?? await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, cancellationToken).ConfigureAwait(false)) is { } made)
|
||
{
|
||
if (await _db.Sequences.GetAsync(made, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id)
|
||
{
|
||
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it is gone; no other is made without a new plan");
|
||
return new StepDone(Result(row, ConvertOutcome.Changed));
|
||
}
|
||
return pins.Of(step.Id) is { } pin
|
||
? await FinishCopyAsync(archive, paths, row, pin, staged, key, made, progress, cancellationToken).ConfigureAwait(false)
|
||
: null;
|
||
}
|
||
if (by.Confirmed is not { } confirmed
|
||
|| await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } later)
|
||
return null;
|
||
var (other, done) = later;
|
||
if (!done)
|
||
{
|
||
// Its job has not finished the copy (it failed while filing it, or still runs): neither a second copy nor done with.
|
||
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed and has not finished it");
|
||
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.CopyUnfinished")));
|
||
}
|
||
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed; no second copy is made");
|
||
progress.Count(row.Id, 0, 0);
|
||
return new StepDone(Result(row, ConvertOutcome.Superseded));
|
||
}
|
||
|
||
/// <summary>
|
||
/// The copy this job made of a source, finished when it holds the conversion of every photo pinned (a crash came after
|
||
/// they were filed, before the journal said so): it takes the source's tags, rating, notes and project as stage B gives
|
||
/// them, whatever became of the photos of the source since. Null while a photo pinned is not in it, or when the source
|
||
/// is no longer where this index has it: the job converts the sequence, which files the rest.
|
||
/// </summary>
|
||
private async Task<StepDone?> FinishCopyAsync(Archive archive, ArchivePaths paths, Sequence row, PinnedSequence pin, StagedLog staged, string key, Guid made,
|
||
Progress progress, CancellationToken cancellationToken)
|
||
{
|
||
var conversions = staged.Of(row.Id, pin.FramesVersion).ToDictionary(c => c.FrameId);
|
||
var filed = await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false);
|
||
var ready = new Dictionary<Guid, StagedConversion>();
|
||
foreach (var photo in pin.Frames)
|
||
{
|
||
if (!conversions.TryGetValue(photo.Id, out var conversion) || !string.Equals(conversion.Source, photo.Sha256, StringComparison.OrdinalIgnoreCase)
|
||
|| !filed.Contains(conversion.Sha256))
|
||
return null;
|
||
ready[photo.Id] = conversion;
|
||
}
|
||
SequenceManifest manifest;
|
||
string? project;
|
||
await using (await _writer.LockAsync(paths, row.Id, cancellationToken).ConfigureAwait(false))
|
||
{
|
||
if (await _db.Sequences.GetAsync(row.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id) return null;
|
||
row = now;
|
||
var folder = paths.Sequence(row.Folder);
|
||
if (!SequenceFolders.Holds(folder, row.Id)) return null;
|
||
manifest = SequenceWriter.ReadLocked(folder, row.Id);
|
||
project = row.SettingsJson ?? (File.Exists(folder.ProjectPath) ? AtomicFile.ReadAllText(folder.ProjectPath) : null);
|
||
}
|
||
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it holds every photo already; it is finished, no other is made");
|
||
progress.Count(row.Id, pin.Frames.Count, pin.Frames.Count);
|
||
return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, made, pin.Frames.Count, cancellationToken).ConfigureAwait(false);
|
||
}
|
||
|
||
/// <summary>
|
||
/// A copy of <paramref name="source"/> in this conversion's format that another conversion made since <paramref name="confirmed"/>:
|
||
/// a sequence filed under another job's session key for that source (<see cref="SessionKey"/>) whose sidecar says it was
|
||
/// made since then and holds photos of that format, and whether that conversion is done (only a done one's copy is whole).
|
||
/// Null when there is none.
|
||
/// </summary>
|
||
private async Task<(Guid Id, bool Done)?> LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed,
|
||
CancellationToken cancellationToken)
|
||
{
|
||
foreach (var (key, id) in await _db.Sequences.ListSessionKeysAsync(archive.Id, $"{KeyPrefix}%:{source:N}", cancellationToken).ConfigureAwait(false))
|
||
{
|
||
if (JobOfKey(key) is not { } other || other == jobId) continue;
|
||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id) continue;
|
||
if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format))
|
||
return (id, await _db.Jobs.GetAsync(other, cancellationToken).ConfigureAwait(false) is { State: JobState.Done });
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/// <summary>
|
||
/// What the job converts of a sequence whose plan is now <paramref name="fresh"/>: the photos it pinned when it first came to
|
||
/// it, each still planned with the content it had then; the first time, the photos confirmed (<paramref name="photos"/>,
|
||
/// <see cref="ConfirmedPhotos"/>) while each is still planned with the content confirmed (the step's digest). Photos that
|
||
/// came in, were verified or could be opened again since stay as they are, so an upload while the job waits its turn
|
||
/// neither changes what it converts nor holds it back. Without the photos confirmed (they could not be written, or a job
|
||
/// an earlier build queued): the whole plan while it has the step's digest, or its frames version. Null when a photo
|
||
/// confirmed no longer converts as it was: the sequence changed after the plan, and the user plans again.
|
||
/// </summary>
|
||
internal static ConvertSequencePlan? Confirmed(ConvertSequence step, IReadOnlyList<Guid>? photos, ConvertSequencePlan fresh, PinnedSequence? pin)
|
||
{
|
||
var now = fresh.Frames.ToDictionary(f => f.FrameId);
|
||
if (pin is not null)
|
||
{
|
||
var frames = new List<ConvertFramePlan>(pin.Frames.Count);
|
||
foreach (var pinned in pin.Frames)
|
||
{
|
||
if (!now.TryGetValue(pinned.Id, out var frame) || !string.Equals(frame.Sha256, pinned.Sha256, StringComparison.OrdinalIgnoreCase)) return null;
|
||
frames.Add(frame);
|
||
}
|
||
return frames.Count > 0 ? fresh with { FramesVersion = pin.FramesVersion, Frames = frames } : null;
|
||
}
|
||
if (step is { Frames: { } confirmedDigest } && photos is { Count: > 0 })
|
||
{
|
||
var frames = new List<ConvertFramePlan>(photos.Count);
|
||
foreach (var id in photos)
|
||
{
|
||
if (!now.TryGetValue(id, out var frame)) return null;
|
||
frames.Add(frame);
|
||
}
|
||
var held = ConvertSequencePlan.DigestOf(frames.Select(f => (f.FrameId, f.Sha256)));
|
||
return string.Equals(held, confirmedDigest, StringComparison.Ordinal) ? fresh with { Frames = frames } : null;
|
||
}
|
||
if (fresh.Frames.Count == 0) return null;
|
||
var confirmed = step.Frames is { } digest
|
||
? string.Equals(fresh.Digest(), digest, StringComparison.Ordinal)
|
||
: fresh.FramesVersion == step.FramesVersion;
|
||
return confirmed ? fresh : null;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Whether the plan made now (<paramref name="fresh"/>) leaves out a photo the job converts of the sequence (one it pinned,
|
||
/// else one of <paramref name="photos"/>, those confirmed) only because its file could not be opened
|
||
/// (<see cref="ConvertSkip.Inaccessible"/>: another program holds it): the sequence then waits for the retry rather than
|
||
/// counting as changed. Photos the job does not convert do not count, nor does a confirmed photo that changed meanwhile.
|
||
/// Without the photos confirmed: any photo of the sequence that could not be opened.
|
||
/// </summary>
|
||
private static bool Held(IReadOnlyList<Guid>? photos, PinnedSequence? pin, SequenceManifest manifest, SequencePaths folder, IReadOnlySet<string> present,
|
||
ConvertSequencePlan fresh, ConvertOptions options, ImageDecoder decoder)
|
||
{
|
||
IReadOnlyList<Guid>? converts = pin is not null ? [.. pin.Frames.Select(f => f.Id)] : photos;
|
||
if (converts is null) return fresh.Skipped.Any(s => s.Reason == ConvertSkip.Inaccessible);
|
||
var planned = fresh.Frames.Select(f => f.FrameId).ToHashSet();
|
||
var frames = manifest.Frames.ToDictionary(f => f.Id);
|
||
return converts.Any(id => !planned.Contains(id) && frames.TryGetValue(id, out var frame)
|
||
&& ConvertPlanner.Skip(frame, folder, present, options, decoder) == ConvertSkip.Inaccessible);
|
||
}
|
||
|
||
/// <summary>Whether every photo of <paramref name="planned"/> is a frame of <paramref name="manifest"/> still holding the content it was planned with.</summary>
|
||
private static bool Holds(SequenceManifest manifest, ConvertSequencePlan planned)
|
||
{
|
||
var frames = manifest.Frames.ToDictionary(f => f.Id);
|
||
return planned.Frames.All(p => frames.TryGetValue(p.FrameId, out var f) && string.Equals(f.Sha256, p.Sha256, StringComparison.OrdinalIgnoreCase));
|
||
}
|
||
|
||
private static ConvertSequenceResult Result(Sequence row, ConvertOutcome outcome, int frames = 0, string? error = null) =>
|
||
new(row.Id, row.DisplayTitle, outcome, frames, Error: error);
|
||
|
||
/// <summary>
|
||
/// The staged conversions of a sequence that still hold: of a frame the plan converts, at the content it had then,
|
||
/// whose staged file is there with its size and SHA-256 (re-hashed now: a pause, a crash or a clean-up may have taken it
|
||
/// since), or which a new sequence holds already (<paramref name="filed"/>).
|
||
/// </summary>
|
||
private static Dictionary<Guid, StagedConversion> Ready(SequenceManifest manifest, ConvertSequencePlan planned, IReadOnlyList<StagedConversion> noted,
|
||
string staging, IReadOnlySet<string> filed)
|
||
{
|
||
var frames = manifest.Frames.ToDictionary(f => f.Id);
|
||
var converted = planned.Frames.Select(f => f.FrameId).ToHashSet();
|
||
var ready = new Dictionary<Guid, StagedConversion>();
|
||
foreach (var conversion in noted)
|
||
{
|
||
if (!converted.Contains(conversion.FrameId) || !frames.TryGetValue(conversion.FrameId, out var frame)
|
||
|| !string.Equals(frame.Sha256, conversion.Source, StringComparison.OrdinalIgnoreCase))
|
||
continue;
|
||
if (filed.Contains(conversion.Sha256) || Intact(Path.Combine(staging, conversion.File), conversion)) ready[conversion.FrameId] = conversion;
|
||
}
|
||
return ready;
|
||
}
|
||
|
||
private static bool Intact(string path, StagedConversion conversion)
|
||
{
|
||
if (!Present(path, conversion)) return false;
|
||
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.SequentialScan);
|
||
return string.Equals(Convert.ToHexStringLower(SHA256.HashData(stream)), conversion.Sha256, StringComparison.OrdinalIgnoreCase);
|
||
}
|
||
|
||
/// <summary>Whether a staged conversion's file is there at its size (not re-hashed).</summary>
|
||
private static bool Present(string path, StagedConversion conversion) => new FileInfo(path) is { Exists: true } file && file.Length == conversion.Size;
|
||
|
||
/// <summary>
|
||
/// Stage A: converts the planned frames not staged yet, several at once, each staged file appended to the journal as it
|
||
/// is consumed (in frame order). A photo starts only while the volume has the space for it and for every photo handed to
|
||
/// the work before it whose file has not landed yet (<see cref="SpaceWatch.Start"/>: converting or waiting for its turn,
|
||
/// up to twice as many as convert at once, at the end of a sequence as anywhere), with the margin and the reserve: short
|
||
/// of it the job stops (<c>Archive.Error.ConvertSpace</c>) before the volume fills. A converted file is encoded in memory
|
||
/// and lands whole, by a rename, before it is verified: what those photos will write is not in the free space measured
|
||
/// yet, and what landed is, so each counts until its file lands, never twice. Returns the first failure (the file and why,
|
||
/// localised; the exception goes to the diagnostics log); the frames converted meanwhile stay staged for a retry.
|
||
/// </summary>
|
||
/// <exception cref="IOException">The volume lacks the space for the next photo; what was staged stays.</exception>
|
||
private async Task<string?> StageAsync(Sequence row, SequenceManifest manifest, SequencePaths folder, ConvertSequencePlan planned,
|
||
Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options, StagedLog staged, Progress progress, SpaceWatch space,
|
||
ConvertCallbacks callbacks, CancellationToken cancellationToken)
|
||
{
|
||
var frames = manifest.Frames.ToDictionary(f => f.Id);
|
||
var todo = planned.Frames.Where(f => !ready.ContainsKey(f.FrameId)).ToList();
|
||
if (todo.Count == 0) return null;
|
||
System.IO.Directory.CreateDirectory(staging);
|
||
var decoder = _writer.Options.Decoder;
|
||
var largest = todo.Max(f => (long)Math.Max(frames[f.FrameId].W, 0) * Math.Max(frames[f.FrameId].H, 0));
|
||
// Photos at once as the host's parallelism and memory allow; each encoder then works alone.
|
||
var atOnce = DecodeBudget.PhotosAtOnce(_writer.Options.Parallelism(), _writer.Options.MemoryBudget(), largest);
|
||
var threads = atOnce > 1 ? 1 : -1;
|
||
var estimates = todo.ToDictionary(f => f.FrameId, f => f.EstimatedBytes);
|
||
string? failure = null;
|
||
Exception? error = null;
|
||
string? full = null;
|
||
await using var work = new OrderedWork<Staged>(atOnce, async result =>
|
||
{
|
||
if (result.Conversion is not { } conversion)
|
||
{
|
||
if (failure is null)
|
||
{
|
||
error = result.Error;
|
||
failure = Loc.L("Archive.Convert.PhotoFailed", result.File, FrameConverter.FailureText(result.Error!));
|
||
}
|
||
return;
|
||
}
|
||
staged.Append(row.Id, planned.FramesVersion, conversion);
|
||
ready[conversion.FrameId] = conversion;
|
||
space.Staged(conversion.Size, estimates[conversion.FrameId]);
|
||
await callbacks.Progress(++progress.Done, progress.Total).ConfigureAwait(false);
|
||
}, cancellationToken);
|
||
for (var i = 0; i < todo.Count && failure is null; i++)
|
||
{
|
||
var frame = frames[todo[i].FrameId];
|
||
var target = Path.Combine(staging, frame.Id.ToString("N") + options.Extension);
|
||
if (space.Start(todo[i].EstimatedBytes, target) is { } message)
|
||
{
|
||
full = message;
|
||
break;
|
||
}
|
||
// A photo that never starts (the job stops: cancelled, or a write of the journal failed) is never counted off,
|
||
// and the watch goes with the job.
|
||
await work.AddAsync(token =>
|
||
{
|
||
try
|
||
{
|
||
return Stage(frame, manifest, folder, target, options, decoder, threads, token);
|
||
}
|
||
finally
|
||
{
|
||
space.Ended(target);
|
||
}
|
||
}).ConfigureAwait(false);
|
||
}
|
||
await work.CompleteAsync().ConfigureAwait(false);
|
||
if (full is not null)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the conversion stops, {full}");
|
||
throw new IOException(full);
|
||
}
|
||
if (failure is not null) Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: not converted", error);
|
||
return failure;
|
||
}
|
||
|
||
/// <summary>One photo of stage A: hashed against its record, converted into <paramref name="target"/> (in the staging folder) and verified.</summary>
|
||
private static Staged Stage(SidecarFrame frame, SequenceManifest manifest, SequencePaths folder, string target, ConvertOptions options, ImageDecoder decoder,
|
||
int threads, CancellationToken cancellationToken)
|
||
{
|
||
var source = folder.File(frame.File);
|
||
try
|
||
{
|
||
// Converted only as archived: a photo whose bytes no longer match its hash waits for a verify.
|
||
string sha;
|
||
using (var stream = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, 1 << 20, FileOptions.SequentialScan))
|
||
sha = Convert.ToHexStringLower(SHA256.HashData(stream));
|
||
if (!string.Equals(sha, frame.Sha256, StringComparison.OrdinalIgnoreCase))
|
||
throw new ConversionException(ConversionFailure.Changed, $"{frame.File} no longer matches its SHA-256; verify the sequence");
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
var file = FrameConverter.Convert(new ConvertSource(source, frame, manifest.Camera, manifest.Lens), options, decoder, target, threads, cancellationToken);
|
||
return new Staged(frame.File, new StagedConversion(frame.Id, sha, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height), null);
|
||
}
|
||
catch (OperationCanceledException)
|
||
{
|
||
throw;
|
||
}
|
||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or NotSupportedException or FormatException
|
||
or InvalidOperationException or ArgumentException)
|
||
{
|
||
return new Staged(frame.File, null, ex);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Stage B of <see cref="ConvertMode.Replace"/>: the staged files take their frames' place (<see cref="FrameTrash.ReplaceAsync"/>)
|
||
/// while every photo pinned still holds the content converted (photos that came in since stay as they are, so the frames
|
||
/// version is not held to the plan's), each named as filing names a photo of the sequence (<see cref="ConvertOptions.FileName"/>:
|
||
/// the archive's file template, the frame and the sequence as the sidecar has them under the lease), made unique among
|
||
/// every file there.
|
||
/// </summary>
|
||
private async Task<StepDone> ReplaceAsync(Archive archive, ArchivePaths paths, Sequence row, ConvertSequencePlan planned,
|
||
Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options, Guid jobId, string? replacedBy, CancellationToken cancellationToken)
|
||
{
|
||
var template = PathTemplate.Parse(archive.FileTemplate);
|
||
var swaps = planned.Frames.Select(f =>
|
||
{
|
||
var conversion = ready[f.FrameId];
|
||
return new FrameSwap(f.FrameId, paths.ToRelative(Path.Combine(staging, conversion.File)), [], (current, sequence) => options.FileName(template, current, sequence),
|
||
current => Converted(current, conversion, options));
|
||
}).ToList();
|
||
var replaced = await _frames.ReplaceAsync(archive, row.Id, null, swaps, replacedBy, jobId, null, (now, _, _) => Task.FromResult(Holds(now, planned)),
|
||
cancellationToken).ConfigureAwait(false);
|
||
if (replaced is null) return new StepDone(Result(row, ConvertOutcome.Changed));
|
||
return new StepDone(Result(row, ConvertOutcome.Converted, swaps.Count), replaced.Had, _frames.FollowUps(archive, replaced.Sequence, replaced.Had));
|
||
}
|
||
|
||
/// <summary>
|
||
/// A converted frame's record: the frame as it is with the converted file's size, hash, format and upright size, read at
|
||
/// full-resolution tier, without the embedded preview it no longer has, without the flags of its old file, and with the
|
||
/// old file as its <see cref="SidecarFrame.Source"/>; capture time, exposure and the other facts stay.
|
||
/// </summary>
|
||
internal static SidecarFrame Converted(SidecarFrame current, StagedConversion conversion, ConvertOptions options) => current with
|
||
{
|
||
Size = conversion.Size,
|
||
Sha256 = conversion.Sha256.ToLowerInvariant(),
|
||
Format = options.Format,
|
||
W = conversion.Width,
|
||
H = conversion.Height,
|
||
Orientation = 1,
|
||
Tier = DecodeTier.Native,
|
||
PreviewOffset = null,
|
||
PreviewLength = null,
|
||
PreviewW = null,
|
||
PreviewH = null,
|
||
Flags = current.Flags & ~FrameTrash.ContentFlags,
|
||
Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size },
|
||
};
|
||
|
||
/// <summary>
|
||
/// Stage B of <see cref="ConvertMode.NewSequence"/>: once the source's photos pinned are checked under its lease as a Replace
|
||
/// checks them, the staged files filed into the sequence of <paramref name="key"/> (made on first use, titled "title (FORMAT)",
|
||
/// owned by the source's owner, else by who asked for it) and named as the ingest names every photo it files there
|
||
/// (<see cref="Core.Naming.FrameNaming"/>: <c>{name}</c> the original's name on the card, <c>{ext}</c> the target's), with the
|
||
/// source's capture facts; then the source's tags, rating, notes and project (its positions translated to the new frames).
|
||
/// Every conversion not filed yet must still be staged whole: one gone since (a clean-up, a quarantine) fails the sequence
|
||
/// rather than leave its photo out. The photos the plan did not convert are not in the new sequence (<see cref="ConvertSequenceResult.LeftOut"/>).
|
||
/// Once its photos are filed, the journal names the new sequence (<see cref="ConvertJournal.Made"/>), so that a retry or a
|
||
/// resume finishes it and never makes a second one.
|
||
/// </summary>
|
||
private async Task<StepDone> CreateAsync(Archive archive, ArchivePaths paths, Sequence row, ConvertSequencePlan planned,
|
||
Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options, string key, Guid? requestedBy, ConvertJournal journal,
|
||
CancellationToken cancellationToken)
|
||
{
|
||
SequenceManifest manifest;
|
||
string? project;
|
||
await using (await _writer.LockAsync(paths, row.Id, cancellationToken).ConfigureAwait(false))
|
||
{
|
||
if (await _db.Sequences.GetAsync(row.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id)
|
||
return new StepDone(Result(row, ConvertOutcome.Changed));
|
||
row = now;
|
||
var folder = paths.Sequence(row.Folder);
|
||
manifest = SequenceWriter.ReadLocked(folder, row.Id);
|
||
if (!Holds(manifest, planned)) return new StepDone(Result(row, ConvertOutcome.Changed));
|
||
// The source's project, read where its sidecar was just read as its own: another index may move the sequence away
|
||
// later, and maybe another one into this folder, whose project the new sequence must not take.
|
||
project = row.SettingsJson ?? (File.Exists(folder.ProjectPath) ? AtomicFile.ReadAllText(folder.ProjectPath) : null);
|
||
}
|
||
var frames = manifest.Frames.ToDictionary(f => f.Id);
|
||
var filed = await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false);
|
||
var pending = new List<StagedFrame>();
|
||
foreach (var converted in planned.Frames)
|
||
{
|
||
var conversion = ready[converted.FrameId];
|
||
if (filed.Contains(conversion.Sha256)) continue;
|
||
var frame = frames[converted.FrameId];
|
||
var path = Path.Combine(staging, conversion.File);
|
||
if (!Present(path, conversion)) throw new UnstagedException(frame.File, path);
|
||
pending.Add(new StagedFrame(path, frame.Original ?? Path.GetFileName(frame.File), conversion.Size, conversion.Sha256, Metadata(path, frame), null, null)
|
||
{
|
||
NamedAs = options.WithExtension(Path.GetFileName(frame.File)),
|
||
});
|
||
}
|
||
var title = $"{manifest.Title ?? row.DisplayTitle} ({options.Label})";
|
||
// A user id, as every filing records it (the owner a row is given), never a user name.
|
||
var owner = (row.OwnerUserId ?? requestedBy)?.ToString();
|
||
// Once the first file moves, the commit runs to its end whatever the caller cancels meanwhile.
|
||
if (pending.Count > 0)
|
||
await _ingest.CommitSessionAsync(archive, new SequenceTarget(key, title, null, owner), pending, CancellationToken.None).ConfigureAwait(false);
|
||
var id = await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, CancellationToken.None).ConfigureAwait(false)
|
||
?? throw new IOException($"No converted photo of {row.DisplayTitle} could be filed into a new sequence.");
|
||
journal.Made(row.Id, id);
|
||
journal.Write(paths);
|
||
return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, id, planned.Frames.Count, cancellationToken).ConfigureAwait(false);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The end of stage B of <see cref="ConvertMode.NewSequence"/>, once the new sequence <paramref name="id"/> holds the
|
||
/// <paramref name="frames"/> photos converted: the source's tags, rating, notes and project given to it (<see cref="CopyAsync"/>),
|
||
/// and the passes it needs, from what the source has. The photos of the source it does not hold are counted as left out.
|
||
/// </summary>
|
||
private async Task<StepDone> CompleteCopyAsync(Archive archive, ArchivePaths paths, Sequence row, SequenceManifest manifest, string? project,
|
||
Dictionary<Guid, StagedConversion> ready, Guid id, int frames, CancellationToken cancellationToken)
|
||
{
|
||
await CopyAsync(paths, row, manifest, project, ready, id, cancellationToken).ConfigureAwait(false);
|
||
var made = await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false)
|
||
?? throw new InvalidOperationException($"The new sequence {id} is not indexed.");
|
||
var leftOut = manifest.Frames.Count(f => !ready.ContainsKey(f.Id));
|
||
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: {frames} photo(s) converted into the new sequence {made.DisplayTitle}, {leftOut} left in the source alone");
|
||
var had = row.Status & (SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt);
|
||
return new StepDone(new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Created, frames, id, LeftOut: leftOut), had,
|
||
_frames.FollowUps(archive, made, had));
|
||
}
|
||
|
||
/// <summary>
|
||
/// The metadata a converted copy is filed with: what its file says, with the source frame's capture time, offset and
|
||
/// time source (the sidecar wins, to the tick) and its intervalometer flag, which the MakerNote left behind carried.
|
||
/// </summary>
|
||
private static FrameMetadata Metadata(string path, SidecarFrame source)
|
||
{
|
||
var read = MetadataReader.Read(path, CancellationToken.None);
|
||
var interval = (source.Flags & FrameFlags.Intervalometer) != 0 ? true : read.IntervalShooting;
|
||
return source.CapturedAt is { } captured
|
||
? read with { CapturedAt = captured, UtcOffsetMinutes = source.UtcOffset, TimestampSource = source.TimeSource, IntervalShooting = interval }
|
||
: read with { IntervalShooting = interval };
|
||
}
|
||
|
||
/// <summary>
|
||
/// Gives the new sequence the source's tags, rating and notes, and its project (<paramref name="json"/>, read under the
|
||
/// source's lease) with the frame positions (exclusions, hero, keyframes) moved from the source's frames to the copies of
|
||
/// them; a project that cannot be read is left out (logged). The project is written under the new sequence's lease into
|
||
/// its folder as the index has it then, which must hold it (<see cref="SequenceFolders.Holds"/>).
|
||
/// </summary>
|
||
/// <exception cref="InvalidOperationException">Another index moved the new sequence away meanwhile; its project is not written.</exception>
|
||
private async Task CopyAsync(ArchivePaths paths, Sequence source, SequenceManifest manifest, string? json, Dictionary<Guid, StagedConversion> ready, Guid id,
|
||
CancellationToken cancellationToken)
|
||
{
|
||
await _service.EditSequenceInfoAsync(id, info => info with { Tags = source.Tags, Rating = source.Rating, Notes = source.Notes }, cancellationToken)
|
||
.ConfigureAwait(false);
|
||
if (string.IsNullOrWhiteSpace(json)) return;
|
||
await using (await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false))
|
||
{
|
||
var made = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) ?? throw new InvalidOperationException($"The new sequence {id} is not indexed.");
|
||
var folder = paths.Sequence(made.Folder);
|
||
// Another index of the archive moved it away, and maybe another sequence into its folder (this index's next scan finds
|
||
// both): its project goes nowhere, as every writer into a sequence's folder decides.
|
||
if (!SequenceFolders.Holds(folder, id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||
var copy = Read(folder, id) ?? throw new InvalidDataException($"The sidecar of the new sequence {made.DisplayTitle} cannot be read.");
|
||
// The source's frames as the copies the conversion made of them; a frame not converted keeps its own photo, which the
|
||
// new sequence lacks (its positions wait as pending frames).
|
||
var from = new FramePhotos(manifest.Frames.Select(f => (f.I, ready.TryGetValue(f.Id, out var c)
|
||
? new FramePhoto(c.Sha256.ToLowerInvariant(), c.Size, f.CapturedAt)
|
||
: new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
|
||
var to = new FramePhotos(copy.Frames.Select(f => (f.I, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
|
||
try
|
||
{
|
||
var translated = ProjectSettings.TranslateFramePositions(json, from, to) ?? json;
|
||
var project = ProjectStore.Parse(translated);
|
||
await new ProjectStore(_db, _writer).UpdateLockedAsync(paths, made, _ => project, cancellationToken).ConfigureAwait(false);
|
||
}
|
||
catch (InvalidDataException ex)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"{source.DisplayTitle}: its project cannot be read; the new sequence {made.DisplayTitle} starts without it", ex);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>The SHA-256 of the frames the sequence of <paramref name="key"/> holds; empty when there is none yet.</summary>
|
||
private async Task<IReadOnlySet<string>> FiledAsync(Archive archive, string key, CancellationToken cancellationToken)
|
||
{
|
||
var shas = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, cancellationToken).ConfigureAwait(false) is not { } id) return shas;
|
||
for (var after = -1; ;)
|
||
{
|
||
var page = await _db.Frames.ListBySequenceAsync(id, after, 1000, cancellationToken).ConfigureAwait(false);
|
||
foreach (var frame in page.Where(f => f.Sha256 is not null)) shas.Add(frame.Sha256!);
|
||
if (page.Count < 1000) return shas;
|
||
after = page[^1].Index;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// The photos converted, when this job replaced the sequence's frames already (a crash after its stage B, before the
|
||
/// journal said so): every photo it pinned is a frame holding a file of the target's format converted from the content
|
||
/// pinned. Without a pin (its journal lost), every photo confirmed (<paramref name="photos"/>) is a frame holding a
|
||
/// conversion to the target whose sources have the step's digest; without those either: the frames holding a conversion
|
||
/// to the target are, by their sources, the photos of the step's digest, or (a job an earlier build queued) every staged
|
||
/// conversion is a frame holding the converted content. Null otherwise.
|
||
/// </summary>
|
||
private static int? ReplacedEarlier(SequenceManifest manifest, ConvertSequence step, IReadOnlyList<Guid>? photos, PinnedSequence? pin,
|
||
IReadOnlyList<StagedConversion> noted, ConvertOptions options)
|
||
{
|
||
var frames = manifest.Frames.ToDictionary(f => f.Id);
|
||
if (pin is not null)
|
||
{
|
||
return pin.Frames.Count > 0 && pin.Frames.All(p => frames.TryGetValue(p.Id, out var f) && f.Format == options.Format
|
||
&& string.Equals(f.Source?.Sha256, p.Sha256, StringComparison.OrdinalIgnoreCase)
|
||
&& !string.Equals(f.Sha256, p.Sha256, StringComparison.OrdinalIgnoreCase))
|
||
? pin.Frames.Count
|
||
: null;
|
||
}
|
||
if (step is { Frames: { } confirmed } && photos is { Count: > 0 })
|
||
{
|
||
var sources = new List<(Guid, string?)>(photos.Count);
|
||
foreach (var id in photos)
|
||
{
|
||
if (!frames.TryGetValue(id, out var f) || f.Format != options.Format || f.Source?.Sha256 is not { } source) return null;
|
||
sources.Add((id, source));
|
||
}
|
||
return string.Equals(ConvertSequencePlan.DigestOf(sources), confirmed, StringComparison.Ordinal) ? photos.Count : null;
|
||
}
|
||
if (step.Frames is { } digest)
|
||
{
|
||
var converted = manifest.Frames.Where(f => f.Format == options.Format && f.Source?.Sha256 is not null).ToList();
|
||
return converted.Count > 0 && string.Equals(ConvertSequencePlan.DigestOf(converted.Select(f => (f.Id, f.Source!.Sha256))), digest, StringComparison.Ordinal)
|
||
? converted.Count
|
||
: null;
|
||
}
|
||
if (noted.Count == 0) return null;
|
||
return noted.All(c => frames.TryGetValue(c.FrameId, out var f) && string.Equals(f.Sha256, c.Sha256, StringComparison.OrdinalIgnoreCase)
|
||
&& string.Equals(f.Source?.Sha256, c.Source, StringComparison.OrdinalIgnoreCase))
|
||
? noted.Select(c => c.FrameId).Distinct().Count()
|
||
: null;
|
||
}
|
||
|
||
/// <summary>
|
||
/// The row of a sequence this job replaced the frames of, the index brought in line with its sidecar when a crash left it
|
||
/// behind (after the sidecar was written, before the index followed); a failure is logged, and the next scan does it. The
|
||
/// caller holds the sequence's lease.
|
||
/// </summary>
|
||
private async Task<Sequence> SyncedAsync(Archive archive, SequencePaths folder, Sequence row, SequenceManifest manifest)
|
||
{
|
||
if (row.FramesVersion == manifest.Versions.Frames) return row;
|
||
try
|
||
{
|
||
// The photos moved with the replacement: the next incremental scan looks at them again.
|
||
await _service.Indexer.SyncAsync(archive, folder, manifest, null, Uncovered, photosChanged: true, CancellationToken.None).ConfigureAwait(false);
|
||
return await _db.Sequences.GetAsync(row.Id, CancellationToken.None).ConfigureAwait(false) ?? row;
|
||
}
|
||
catch (Exception ex) when (ex is DatabaseException or IOException or UnauthorizedAccessException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the index did not follow its converted frames; the next scan resyncs it", ex);
|
||
return row;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// The row of a sequence still to convert, its sidecar settled first under its lease when a writer a crash interrupted left
|
||
/// a journal in its folder (<see cref="SequenceWriter.ReadLocked"/>): planned as its files lie half-way, photos in the
|
||
/// trash would count as missing. The row as it was when there is none, or when the settle cannot run now (the sequence's
|
||
/// turn reports it).
|
||
/// </summary>
|
||
private async Task<Sequence> SettledRowAsync(Archive archive, ArchivePaths paths, Sequence row, CancellationToken cancellationToken)
|
||
{
|
||
if (!HasJournal(paths.Sequence(row.Folder))) return row;
|
||
try
|
||
{
|
||
await using (await _writer.LockAsync(paths, row.Id, cancellationToken).ConfigureAwait(false))
|
||
{
|
||
if (await _db.Sequences.GetAsync(row.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id) return row;
|
||
SequenceWriter.ReadLocked(paths.Sequence(now.Folder), now.Id);
|
||
return now;
|
||
}
|
||
}
|
||
catch (ArchiveOfflineException)
|
||
{
|
||
throw;
|
||
}
|
||
catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: what a crash left half-way in it could not be settled yet", ex);
|
||
return row;
|
||
}
|
||
}
|
||
|
||
/// <summary>Whether a writer left a journal in the sequence's folder: a rename of its files, a deletion, a replacement, a restore or an ingest.</summary>
|
||
private static bool HasJournal(SequencePaths folder) =>
|
||
File.Exists(folder.FileRenameJournalPath) || File.Exists(folder.RemovalJournalPath) || File.Exists(folder.ReplaceJournalPath)
|
||
|| File.Exists(folder.RestoreJournalPath) || File.Exists(folder.IngestJournalPath);
|
||
|
||
/// <summary>The sidecar of a sequence, read without its lease; null when it cannot be read or names another sequence.</summary>
|
||
private static SequenceManifest? Read(SequencePaths folder, Guid id)
|
||
{
|
||
try
|
||
{
|
||
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||
return manifest.Id == id ? manifest : null;
|
||
}
|
||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"The sidecar of {folder.RelativeFolder} cannot be read", ex);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// The passes a sequence this job converted or made in an earlier attempt needs, from its journal (a pause, a yield, a stop
|
||
/// or a crash came before they were queued); null for any other outcome.
|
||
/// </summary>
|
||
private async Task<IReadOnlyList<ArchiveJobRequest>?> FollowUpsAsync(Archive archive, ConvertJournalSequence noted, CancellationToken cancellationToken)
|
||
{
|
||
if (noted.Outcome == ConvertOutcome.Converted)
|
||
{
|
||
return await _db.Sequences.GetAsync(noted.Id, cancellationToken).ConfigureAwait(false) is { } row && row.ArchiveId == archive.Id
|
||
? _frames.FollowUps(archive, row, noted.Had ?? row.Status)
|
||
: [];
|
||
}
|
||
if (noted is { Outcome: ConvertOutcome.Created, NewSequenceId: { } id })
|
||
{
|
||
return await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } made && made.ArchiveId == archive.Id
|
||
? _frames.FollowUps(archive, made, noted.Had ?? 0)
|
||
: [];
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/// <summary>
|
||
/// The bytes the photos <paramref name="plan"/> converts are estimated at, but for those whose staged file is there already,
|
||
/// or which the new sequence this job made holds (<paramref name="filed"/>, the SHA-256 of its photos).
|
||
/// </summary>
|
||
private static long Unstaged(ArchivePaths paths, Guid jobId, ConvertSequencePlan plan, StagedLog staged, IReadOnlySet<string>? filed)
|
||
{
|
||
var folder = StagingOf(paths, jobId, plan.SequenceId);
|
||
var there = staged.Of(plan.SequenceId, plan.FramesVersion)
|
||
.Where(c => filed?.Contains(c.Sha256) == true || Present(Path.Combine(folder, c.File), c))
|
||
.Select(c => c.FrameId).ToHashSet();
|
||
return plan.Frames.Where(f => !there.Contains(f.FrameId)).Sum(f => f.EstimatedBytes);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Refuses to go on while the volume lacks the space the photos still to convert need: their estimates, <paramref name="estimated"/>
|
||
/// bytes, as the files staged so far scale them.
|
||
/// </summary>
|
||
/// <exception cref="IOException">The volume lacks it (<c>Archive.Error.ConvertSpace</c>).</exception>
|
||
private static void EnsureSpace(long estimated, SpaceWatch space)
|
||
{
|
||
if (estimated > 0 && space.Short(estimated) is { } message) throw new IOException(message);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Free bytes where a conversion writes, as the host counts them (<see cref="ArchiveServiceOptions.FreeSpace"/>): the volume of
|
||
/// the archive root and the one of its staging folder, the tighter; null when neither can be measured.
|
||
/// </summary>
|
||
private long? FreeSpace(ArchivePaths paths)
|
||
{
|
||
long? free = null;
|
||
foreach (var folder in (string[])[paths.Root, paths.StagingDirectory])
|
||
{
|
||
if (!System.IO.Directory.Exists(folder) || Measure(folder) is not { } bytes) continue;
|
||
free = free is { } known ? Math.Min(known, bytes) : bytes;
|
||
}
|
||
return free;
|
||
}
|
||
|
||
private long? Measure(string folder)
|
||
{
|
||
if (_writer.Options.FreeSpace is { } measure) return measure(folder);
|
||
try
|
||
{
|
||
// Windows wants a drive root (a network share is measured by the host); Unix accepts any directory and reports the
|
||
// volume it is mounted on.
|
||
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(folder)! : folder).AvailableFreeSpace;
|
||
}
|
||
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private static void Log(ConvertCallbacks callbacks, ConvertSequenceResult result)
|
||
{
|
||
var line = result.Outcome switch
|
||
{
|
||
ConvertOutcome.Converted => Loc.L("Archive.Log.Converted", result.Title, result.Frames),
|
||
ConvertOutcome.Created => Loc.L("Archive.Log.ConvertedNew", result.Title, result.Frames),
|
||
ConvertOutcome.Nothing => Loc.L("Archive.Log.ConvertNothing", result.Title),
|
||
ConvertOutcome.Superseded => Loc.L("Archive.Log.ConvertSuperseded", result.Title),
|
||
ConvertOutcome.Changed => Loc.L("Archive.Log.ConvertChanged", result.Title),
|
||
// Busy, or why the retry may do better (a photo another program holds, a folder another index moved), in one sentence.
|
||
ConvertOutcome.Blocked when result.Error is null => Loc.L("Archive.Log.ConvertBlocked", result.Title),
|
||
_ => Loc.L("Archive.Log.ConvertFailed", result.Title, result.Error),
|
||
};
|
||
callbacks.Log(line);
|
||
// A new sequence holds only the photos converted: those the plan skipped stay in the source alone.
|
||
if (result is { Outcome: ConvertOutcome.Created, LeftOut: > 0 }) callbacks.Log(Loc.L("Archive.Log.ConvertLeftOut", result.Title, result.LeftOut));
|
||
}
|
||
|
||
/// <summary>
|
||
/// Two sizes as the space error gives them: GB from one gigabyte, MB below, with more decimals when one would read as the
|
||
/// other ("needs 0.5 GB, 0.5 GB free").
|
||
/// </summary>
|
||
private static (string Required, string Free) Sizes(long required, long free)
|
||
{
|
||
var (a, b) = (Size(required, "0.#"), Size(free, "0.#"));
|
||
return a != b ? (a, b) : (Size(required, "0.###"), Size(free, "0.###"));
|
||
}
|
||
|
||
private static string Size(long bytes, string digits) => bytes >= 1_000_000_000
|
||
? Loc.L("Common.Size.GB", (bytes / 1e9).ToString(digits, Loc.Culture))
|
||
: Loc.L("Common.Size.MB", (bytes / 1e6).ToString(digits, Loc.Culture));
|
||
|
||
/// <summary>The session key of the new sequence a conversion makes from <paramref name="source"/>.</summary>
|
||
internal static string SessionKey(Guid job, Guid source) => $"{KeyPrefix}{job:N}:{source:N}";
|
||
|
||
/// <summary>The conversion job a session key of <see cref="SessionKey"/> names; null for any other key.</summary>
|
||
private static Guid? JobOfKey(string key) =>
|
||
key.Split(':') is ["convert", var job, _] && Guid.TryParseExact(job, "N", out var id) ? id : null;
|
||
|
||
/// <summary>The folder under <c>.chronos/staging</c> a conversion job stages into, one subfolder per sequence.</summary>
|
||
internal static string StagingFolder(Guid job) => ConvertJournal.Prefix + job.ToString("N");
|
||
|
||
private static string StagingOf(ArchivePaths paths, Guid job, Guid sequence) =>
|
||
Path.Combine(paths.StagingDirectory, StagingFolder(job), sequence.ToString("N"));
|
||
|
||
/// <summary>Deletes what a conversion staged that no step needs any more (converted copies, never originals); a failure is logged.</summary>
|
||
internal static void DeleteStaging(string folder)
|
||
{
|
||
try
|
||
{
|
||
if (System.IO.Directory.Exists(folder)) System.IO.Directory.Delete(folder, recursive: true);
|
||
}
|
||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"{folder} could not be deleted; maintenance removes it later", ex);
|
||
}
|
||
}
|
||
|
||
/// <summary>Photos converted so far and in all, counted per sequence.</summary>
|
||
private sealed class Progress
|
||
{
|
||
private readonly Dictionary<Guid, (int Frames, int Staged)> _counted = [];
|
||
|
||
public long Total { get; private set; }
|
||
|
||
public long Done { get; set; }
|
||
|
||
/// <summary>Counts the photos a sequence converts and those staged already, in place of what was counted for it before.</summary>
|
||
public void Count(Guid sequence, int frames, int staged)
|
||
{
|
||
if (_counted.TryGetValue(sequence, out var counted))
|
||
{
|
||
Total -= counted.Frames;
|
||
Done -= counted.Staged;
|
||
}
|
||
_counted[sequence] = (frames, staged);
|
||
Total += frames;
|
||
Done += staged;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// The free space a running conversion watches: measured where it writes (<paramref name="free"/>), against estimates scaled
|
||
/// by how far the files it staged came out above theirs (photos of a 14-bit RAW carry more noise than the estimate's bits
|
||
/// per pixel allow for), and, in stage A, with the photos started whose converted file is not there yet (<see cref="Start"/>):
|
||
/// a converted file is encoded in memory and lands whole, by a rename, before its photo's work verifies it, so the free
|
||
/// space measured holds every file landed, and none of what the others will write.
|
||
/// </summary>
|
||
internal sealed class SpaceWatch(Func<long?> free)
|
||
{
|
||
private readonly ConcurrentDictionary<string, Starting> _started = new(StringComparer.Ordinal);
|
||
private long _actual;
|
||
private long _estimated;
|
||
|
||
/// <summary>The bytes staged over their estimates, at least 1.</summary>
|
||
public double Ratio => _estimated > 0 ? Math.Max(1, (double)_actual / _estimated) : 1;
|
||
|
||
/// <summary>
|
||
/// The estimated bytes of the photos started (<see cref="Start"/>) whose converted file has not landed yet: the file is
|
||
/// not at its path, or is still the one that was there when the photo started (one a crash left, which it replaces).
|
||
/// </summary>
|
||
public long Writing => _started.Sum(s => Landed(s.Key) == s.Value.Before ? s.Value.Estimate : 0);
|
||
|
||
public void Staged(long actual, long estimated)
|
||
{
|
||
_actual += actual;
|
||
_estimated += Math.Max(1, estimated);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Why a photo estimated at <paramref name="estimated"/> bytes, converted into <paramref name="target"/>, cannot start now:
|
||
/// the volume lacks the space for it and for the photos started before it whose file has not landed yet
|
||
/// (<see cref="Writing"/>, <see cref="Short"/>), localized. Null when it can: it counts among those until its file lands
|
||
/// or its work ends (<see cref="Ended"/>).
|
||
/// </summary>
|
||
public string? Start(long estimated, string target)
|
||
{
|
||
var before = Landed(target);
|
||
if (Short(Math.Max(0, estimated) + Writing) is { } message) return message;
|
||
_started[target] = new Starting(Math.Max(0, estimated), before);
|
||
return null;
|
||
}
|
||
|
||
/// <summary>The work of the photo converted into <paramref name="target"/> ended: its file landed, or never will.</summary>
|
||
public void Ended(string target) => _started.TryRemove(target, out _);
|
||
|
||
/// <summary>
|
||
/// Why the volume cannot take <paramref name="estimated"/> bytes more (scaled, with the margin and the reserve), localized;
|
||
/// null when it can, or when its free space cannot be measured.
|
||
/// </summary>
|
||
public string? Short(long estimated)
|
||
{
|
||
var space = new ConvertSpace(ConvertPlanner.Required((long)Math.Ceiling(estimated * Ratio)), free());
|
||
if (space.Enough) return null;
|
||
var (required, left) = Sizes(space.Required, space.Free ?? 0);
|
||
return Loc.L("Archive.Error.ConvertSpace", required, left);
|
||
}
|
||
|
||
/// <summary>The file at <paramref name="path"/> as a landing changes it, its size and time of writing; null when there is none.</summary>
|
||
private static (long Size, long Written)? Landed(string path) =>
|
||
new FileInfo(path) is { Exists: true } file ? (file.Length, file.LastWriteTimeUtc.Ticks) : null;
|
||
|
||
/// <summary>A photo started: its estimate, and the file at its path then.</summary>
|
||
private sealed record Starting(long Estimate, (long Size, long Written)? Before);
|
||
}
|
||
|
||
/// <summary>What a sequence came to: its result, the statuses its follow-ups follow from, and those follow-ups (null: none to queue).</summary>
|
||
private sealed record StepDone(ConvertSequenceResult Result, SequenceStatus? Had = null, IReadOnlyList<ArchiveJobRequest>? FollowUps = null);
|
||
|
||
/// <summary>A photo of stage A: its file and its staged conversion, or why it failed.</summary>
|
||
private sealed record Staged(string File, StagedConversion? Conversion, Exception? Error);
|
||
|
||
/// <summary>
|
||
/// The staged conversion of a photo a new sequence is about to file is gone (a clean-up, a quarantine): the sequence fails
|
||
/// rather than leave the photo out, and the retry converts it again. Its message is the job log's catalog sentence.
|
||
/// </summary>
|
||
private sealed class UnstagedException(string file, string path) : FileNotFoundException(Loc.L("Archive.Convert.Unstaged", file), path);
|
||
}
|
||
|
||
/// <summary>One photo converted and staged: its frame, the content it was converted from, the staged file's name, size and SHA-256, and its size in pixels.</summary>
|
||
/// <param name="Source">SHA-256 of the original it was converted from.</param>
|
||
/// <param name="File">The staged file's name inside the sequence's staging folder.</param>
|
||
internal sealed record StagedConversion(Guid FrameId, string Source, string File, long Size, string Sha256, int Width, int Height);
|
||
|
||
/// <summary>A line of the staged log: a staged conversion of a sequence at a frames version.</summary>
|
||
internal sealed record StagedLine(Guid Sequence, int FramesVersion, StagedConversion Conversion);
|
||
|
||
/// <summary>The photos of a sequence a conversion job pinned: the frames version then, and each photo with the content converted.</summary>
|
||
internal sealed record PinnedSequence(Guid Sequence, int FramesVersion, IReadOnlyList<PinnedFrame> Frames);
|
||
|
||
/// <summary>A photo pinned: its frame and the SHA-256 of the content converted.</summary>
|
||
internal sealed record PinnedFrame(Guid Id, string Sha256);
|
||
|
||
/// <summary>
|
||
/// The journal of a conversion job (<c>.chronos/journal/convert-<job>.json</c>, owner review 2, R7): what each sequence
|
||
/// came to, the new sequence a New sequence conversion made of it as soon as its photos are filed, and whether the passes
|
||
/// it needs were queued, written after each one; with it, the photos confirmed (<c>convert-<job>.photos</c>,
|
||
/// <see cref="ConfirmedPhotos"/>, written when the job is queued), the pin log (<c>convert-<job>.pins</c>,
|
||
/// <see cref="PinLog"/>), the photos the job converts of each sequence, and the staged log (<c>convert-<job>.staged</c>),
|
||
/// one line per photo staged, both appended and flushed as they grow, so a job of thousands of photos journals each once. A
|
||
/// resumed or retried job keeps the sequences that are final, the copies made, the photos pinned, and the staged files
|
||
/// whose hash still verifies. All of it goes when every sequence is final, or once the job is gone (<see cref="ArchiveMaintenance"/>).
|
||
/// </summary>
|
||
internal sealed class ConvertJournal
|
||
{
|
||
/// <summary>How the journal files and the staging folder of a conversion job begin, before the job's id.</summary>
|
||
public const string Prefix = "convert-";
|
||
|
||
public Guid JobId { get; set; }
|
||
|
||
public List<ConvertJournalSequence> Sequences { get; set; } = [];
|
||
|
||
public ConvertJournalSequence? Of(Guid id) => Sequences.FirstOrDefault(s => s.Id == id);
|
||
|
||
/// <summary>Notes a sequence the job comes to and pins, with the statuses it had then.</summary>
|
||
public void Begin(Guid id, string title, SequenceStatus had)
|
||
{
|
||
var noted = Of(id);
|
||
if (noted is null) Sequences.Add(noted = new ConvertJournalSequence { Id = id });
|
||
noted.Title = title;
|
||
noted.Had = had;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Notes what a sequence came to and, when it was converted or made, the statuses its follow-ups follow from. The new
|
||
/// sequence a New sequence conversion made of it stays named whatever it came to since (<see cref="Made"/>).
|
||
/// </summary>
|
||
public void Note(ConvertSequenceResult result, SequenceStatus? had = null)
|
||
{
|
||
var noted = Of(result.SequenceId);
|
||
if (noted is null) Sequences.Add(noted = new ConvertJournalSequence { Id = result.SequenceId });
|
||
noted.Title = result.Title;
|
||
noted.Outcome = result.Outcome;
|
||
noted.Frames = result.Frames;
|
||
noted.NewSequenceId = result.NewSequenceId ?? noted.NewSequenceId;
|
||
if (had is { } status) noted.Had = status;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Notes the new sequence a New sequence conversion filed the photos of <paramref name="id"/> into, before anything else is
|
||
/// done with it: a retry or a resume finishes that one and makes no other.
|
||
/// </summary>
|
||
public void Made(Guid id, Guid newSequenceId)
|
||
{
|
||
var noted = Of(id);
|
||
if (noted is null) Sequences.Add(noted = new ConvertJournalSequence { Id = id });
|
||
noted.NewSequenceId = newSequenceId;
|
||
}
|
||
|
||
public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.json");
|
||
|
||
public static string StagedPathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.staged");
|
||
|
||
public static string PinsPathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.pins");
|
||
|
||
/// <summary>
|
||
/// The conversion job a journal file or a staging folder belongs to, from its name (<c>convert-<job></c> and any
|
||
/// extension); null for any other name.
|
||
/// </summary>
|
||
public static Guid? JobOf(string name)
|
||
{
|
||
if (!name.StartsWith(Prefix, StringComparison.Ordinal)) return null;
|
||
var rest = name[Prefix.Length..];
|
||
var dot = rest.IndexOf('.', StringComparison.Ordinal);
|
||
return Guid.TryParseExact(dot < 0 ? rest : rest[..dot], "N", out var job) ? job : null;
|
||
}
|
||
|
||
/// <summary>The journal of <paramref name="job"/>; null when there is none, or it is torn.</summary>
|
||
public static ConvertJournal? Read(ArchivePaths paths, Guid job)
|
||
{
|
||
var path = PathFor(paths, job);
|
||
if (!File.Exists(path)) return null;
|
||
try
|
||
{
|
||
return JsonSerializer.Deserialize(AtomicFile.ReadAllBytes(path), ArchiveJobJson.Default.ConvertJournal);
|
||
}
|
||
catch (JsonException ex)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"The conversion journal {path} is torn; the job starts over", ex);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
public void Write(ArchivePaths paths)
|
||
{
|
||
System.IO.Directory.CreateDirectory(paths.JournalDirectory);
|
||
AtomicFile.WriteAllBytes(PathFor(paths, JobId), JsonSerializer.SerializeToUtf8Bytes(this, ArchiveJobJson.Default.ConvertJournal));
|
||
}
|
||
|
||
/// <summary>The staged log of this job, read now.</summary>
|
||
public StagedLog Staged(ArchivePaths paths) => StagedLog.Open(StagedPathFor(paths, JobId));
|
||
|
||
/// <summary>The pin log of this job, read now.</summary>
|
||
public PinLog Pins(ArchivePaths paths) => PinLog.Open(PinsPathFor(paths, JobId));
|
||
|
||
/// <summary>The journal files of <paramref name="job"/>, whichever exist: the photos confirmed among them (<see cref="ConfirmedPhotos"/>).</summary>
|
||
public static IEnumerable<string> Files(ArchivePaths paths, Guid job) =>
|
||
[StagedPathFor(paths, job), PinsPathFor(paths, job), ConfirmedPhotos.PathFor(paths, job), PathFor(paths, job)];
|
||
|
||
public static void Delete(ArchivePaths paths, Guid job)
|
||
{
|
||
foreach (var path in Files(paths, job))
|
||
{
|
||
if (File.Exists(path)) File.Delete(path);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>What the journal knows of one sequence of a conversion.</summary>
|
||
internal sealed class ConvertJournalSequence
|
||
{
|
||
public Guid Id { get; set; }
|
||
|
||
public string? Title { get; set; }
|
||
|
||
public ConvertOutcome? Outcome { get; set; }
|
||
|
||
public int Frames { get; set; }
|
||
|
||
public Guid? NewSequenceId { get; set; }
|
||
|
||
/// <summary>
|
||
/// The statuses the passes it needs follow from: the sequence's before its frames were replaced, or the source's a new
|
||
/// sequence takes on (a deep analysis, thumbnails, proxies).
|
||
/// </summary>
|
||
public SequenceStatus? Had { get; set; }
|
||
|
||
/// <summary>The passes it needs were queued (a resumed job queues them when they were not).</summary>
|
||
public bool Queued { get; set; }
|
||
|
||
[JsonIgnore]
|
||
public bool Final => ConvertSequenceResult.IsFinal(Outcome);
|
||
}
|
||
|
||
/// <summary>The staged log of a conversion job (<see cref="ConvertJournal"/>): read once, appended to line by line.</summary>
|
||
internal sealed class StagedLog
|
||
{
|
||
private readonly string _path;
|
||
private readonly Dictionary<(Guid, int), Dictionary<Guid, StagedConversion>> _lines = [];
|
||
|
||
private StagedLog(string path) => _path = path;
|
||
|
||
/// <summary>The log at <paramref name="path"/>; a line torn by a crash, or unreadable, is skipped.</summary>
|
||
public static StagedLog Open(string path)
|
||
{
|
||
var log = new StagedLog(path);
|
||
foreach (var line in JournalLines.Read(path))
|
||
{
|
||
try
|
||
{
|
||
if (JsonSerializer.Deserialize(line, ArchiveJobJson.Default.StagedLine) is { } staged) log.Add(staged);
|
||
}
|
||
catch (JsonException)
|
||
{
|
||
// Torn while appended: that photo is converted again.
|
||
}
|
||
}
|
||
return log;
|
||
}
|
||
|
||
/// <summary>The staged conversions of a sequence at a frames version, latest per frame.</summary>
|
||
public IReadOnlyList<StagedConversion> Of(Guid sequence, int framesVersion) =>
|
||
_lines.TryGetValue((sequence, framesVersion), out var frames) ? [.. frames.Values] : [];
|
||
|
||
/// <summary>Appends a staged conversion and flushes it to disk.</summary>
|
||
public void Append(Guid sequence, int framesVersion, StagedConversion conversion)
|
||
{
|
||
var line = new StagedLine(sequence, framesVersion, conversion);
|
||
JournalLines.Append(_path, JsonSerializer.SerializeToUtf8Bytes(line, ArchiveJobJson.Default.StagedLine));
|
||
Add(line);
|
||
}
|
||
|
||
private void Add(StagedLine line)
|
||
{
|
||
if (!_lines.TryGetValue((line.Sequence, line.FramesVersion), out var frames)) _lines[(line.Sequence, line.FramesVersion)] = frames = [];
|
||
frames[line.Conversion.FrameId] = line.Conversion;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// The pin log of a conversion job (<see cref="ConvertJournal"/>): the photos it converts of each sequence, pinned under the
|
||
/// sequence's lease the first time the job comes to it, while its plan is still the one confirmed. A retry converts those,
|
||
/// never photos that came in or were verified since. One line per sequence, appended and flushed.
|
||
/// </summary>
|
||
internal sealed class PinLog
|
||
{
|
||
private readonly string _path;
|
||
private readonly Dictionary<Guid, PinnedSequence> _pins = [];
|
||
|
||
private PinLog(string path) => _path = path;
|
||
|
||
/// <summary>The log at <paramref name="path"/>; a line torn by a crash is skipped (that sequence is held to the plan confirmed again).</summary>
|
||
public static PinLog Open(string path)
|
||
{
|
||
var log = new PinLog(path);
|
||
foreach (var line in JournalLines.Read(path))
|
||
{
|
||
try
|
||
{
|
||
if (JsonSerializer.Deserialize(line, ArchiveJobJson.Default.PinnedSequence) is { } pinned) log._pins[pinned.Sequence] = pinned;
|
||
}
|
||
catch (JsonException)
|
||
{
|
||
// Torn while appended.
|
||
}
|
||
}
|
||
return log;
|
||
}
|
||
|
||
public PinnedSequence? Of(Guid sequence) => _pins.GetValueOrDefault(sequence);
|
||
|
||
/// <summary>Pins the photos <paramref name="planned"/> converts, and flushes it to disk.</summary>
|
||
public void Append(Guid sequence, ConvertSequencePlan planned)
|
||
{
|
||
var pinned = new PinnedSequence(sequence, planned.FramesVersion, [.. planned.Frames.Select(f => new PinnedFrame(f.FrameId, f.Sha256 ?? string.Empty))]);
|
||
JournalLines.Append(_path, JsonSerializer.SerializeToUtf8Bytes(pinned, ArchiveJobJson.Default.PinnedSequence));
|
||
_pins[sequence] = pinned;
|
||
}
|
||
}
|
||
|
||
/// <summary>A line of <see cref="ConfirmedPhotos"/>: a sequence, and the frame ids of the photos converted of it, 16 bytes each, in base64.</summary>
|
||
internal sealed record ConfirmedLine(Guid Sequence, string Photos);
|
||
|
||
/// <summary>
|
||
/// The photos a conversion job converts of each sequence, as its plan was confirmed (<c>.chronos/journal/convert-<job>.photos</c>,
|
||
/// owner review 2, R7): written whole under the job's id before it is queued (<see cref="MassConverter.QueueAsync"/>), one
|
||
/// line per sequence with the frame ids in the plan's order. The job's row keeps each sequence's digest of those photos
|
||
/// with their content (<see cref="ConvertSequence.Frames"/>), never the ids: it stays as small as the number of sequences
|
||
/// whatever the number of photos, for every list and check that reads jobs. Under each sequence's lease the job takes these
|
||
/// photos from its plan made again and goes on only while they have that digest; a sequence this file does not name (it
|
||
/// could not be written, or a job an earlier build queued) is held to its whole plan. It goes with the journal
|
||
/// (<see cref="ConvertJournal.Files"/>).
|
||
/// </summary>
|
||
internal sealed class ConfirmedPhotos
|
||
{
|
||
private readonly Dictionary<Guid, IReadOnlyList<Guid>> _photos = [];
|
||
|
||
public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{ConvertJournal.Prefix}{job:N}.photos");
|
||
|
||
/// <summary>The photos confirmed of <paramref name="sequence"/>, in the plan's order; null when the file does not name it.</summary>
|
||
public IReadOnlyList<Guid>? Of(Guid sequence) => _photos.GetValueOrDefault(sequence);
|
||
|
||
/// <summary>The photos confirmed for <paramref name="job"/>: none when the file is not there; a line that cannot be read names nothing.</summary>
|
||
public static ConfirmedPhotos Read(ArchivePaths paths, Guid job)
|
||
{
|
||
var read = new ConfirmedPhotos();
|
||
foreach (var line in JournalLines.Read(PathFor(paths, job)))
|
||
{
|
||
try
|
||
{
|
||
if (JsonSerializer.Deserialize(line, ArchiveJobJson.Default.ConfirmedLine) is { } confirmed) read._photos[confirmed.Sequence] = Decode(confirmed.Photos);
|
||
}
|
||
catch (Exception ex) when (ex is JsonException or FormatException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"A line of the photos confirmed for conversion job {job} cannot be read; that sequence is held to its whole plan", ex);
|
||
}
|
||
}
|
||
return read;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Writes the photos <paramref name="plan"/> converts of each sequence it converts, for the job <paramref name="job"/>, whole
|
||
/// (a reader finds all of it or none); false, logged, when it cannot be written.
|
||
/// </summary>
|
||
public static bool TryWrite(ArchivePaths paths, Guid job, ConvertPlan plan)
|
||
{
|
||
ArgumentNullException.ThrowIfNull(plan);
|
||
try
|
||
{
|
||
using var lines = new MemoryStream();
|
||
foreach (var sequence in plan.Converted)
|
||
{
|
||
lines.Write(JsonSerializer.SerializeToUtf8Bytes(new ConfirmedLine(sequence.SequenceId, Encode(sequence.Frames.Select(f => f.FrameId))),
|
||
ArchiveJobJson.Default.ConfirmedLine));
|
||
lines.WriteByte((byte)'\n');
|
||
}
|
||
System.IO.Directory.CreateDirectory(paths.JournalDirectory);
|
||
AtomicFile.WriteAllBytes(PathFor(paths, job), lines.GetBuffer().AsSpan(0, (int)lines.Length));
|
||
return true;
|
||
}
|
||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be written; it holds each sequence to its whole plan", ex);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/// <summary>Deletes the file of <paramref name="job"/>; a failure is logged, and maintenance removes it later.</summary>
|
||
public static void Delete(ArchivePaths paths, Guid job)
|
||
{
|
||
try
|
||
{
|
||
File.Delete(PathFor(paths, job));
|
||
}
|
||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||
{
|
||
Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be deleted; maintenance removes them later", ex);
|
||
}
|
||
}
|
||
|
||
/// <summary>Frame ids as a line holds them: their 16 bytes each, one after the other, in base64.</summary>
|
||
public static string Encode(IEnumerable<Guid> ids)
|
||
{
|
||
ArgumentNullException.ThrowIfNull(ids);
|
||
IReadOnlyCollection<Guid> frames = ids as IReadOnlyCollection<Guid> ?? [.. ids];
|
||
var bytes = new byte[frames.Count * 16];
|
||
var at = 0;
|
||
foreach (var frame in frames)
|
||
{
|
||
frame.TryWriteBytes(bytes.AsSpan(at, 16));
|
||
at += 16;
|
||
}
|
||
return Convert.ToBase64String(bytes);
|
||
}
|
||
|
||
/// <summary>The frame ids <see cref="Encode"/> wrote, in order.</summary>
|
||
/// <exception cref="FormatException">The text is not frame ids as <see cref="Encode"/> writes them.</exception>
|
||
public static IReadOnlyList<Guid> Decode(string text)
|
||
{
|
||
ArgumentNullException.ThrowIfNull(text);
|
||
var bytes = Convert.FromBase64String(text);
|
||
if (bytes.Length % 16 != 0) throw new FormatException("The text is not frame ids.");
|
||
var ids = new Guid[bytes.Length / 16];
|
||
for (var i = 0; i < ids.Length; i++) ids[i] = new Guid(bytes.AsSpan(i * 16, 16));
|
||
return ids;
|
||
}
|
||
}
|
||
|
||
/// <summary>Files of JSON lines a job appends to as it goes: each line written whole and flushed to disk, read back skipping blank ones.</summary>
|
||
internal static class JournalLines
|
||
{
|
||
public static IEnumerable<string> Read(string path) =>
|
||
File.Exists(path) ? File.ReadLines(path, Encoding.UTF8).Where(l => !string.IsNullOrWhiteSpace(l)) : [];
|
||
|
||
public static void Append(string path, byte[] json)
|
||
{
|
||
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||
using var stream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read);
|
||
stream.Write(json);
|
||
stream.WriteByte((byte)'\n');
|
||
stream.Flush(flushToDisk: true);
|
||
}
|
||
}
|