Files
Chronos/src/Chronos.Data/Archiving/MassConverter.cs
T
Alby96andClaude Opus 5.5 efa87f68ce
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
feat: owner review 2 phase 4b: mass conversion after import (#13)
Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-11-03 10:41:50 +01:00

1734 lines
102 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Collections.Concurrent;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Chronos.Core.Archiving;
using Chronos.Core.Conversion;
using Chronos.Core.Diagnostics;
using Chronos.Core.Imaging;
using Chronos.Core.IO;
using Chronos.Core.Localization;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
using Chronos.Core.Naming;
using Chronos.Core.Project;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Processing;
namespace Chronos.Data.Archiving;
/// <summary>What a conversion did with one sequence (owner review 2, R7).</summary>
public enum ConvertOutcome
{
/// <summary>Its frames were replaced by the converted files (<see cref="ConvertMode.Replace"/>).</summary>
Converted,
/// <summary>The converted files went into a new sequence (<see cref="ConvertMode.NewSequence"/>).</summary>
Created,
/// <summary>It had nothing to convert.</summary>
Nothing,
/// <summary>
/// Done with, no copy made (<see cref="ConvertMode.NewSequence"/>): another conversion made its copy in this format since
/// this one was confirmed (a retry of an older conversion, or a duplicate queued beside it), so a second copy is not made.
/// </summary>
Superseded,
/// <summary>
/// Left as it was: a photo it converts no longer holds the content confirmed (it changed, is gone, or no longer converts),
/// or it is no longer in the archive. A retry finds the same: the user plans again.
/// </summary>
Changed,
/// <summary>
/// Left as it was for now: it is busy, another index moved its folder, a photo it converts is held by another program, or
/// its sidecar cannot be read or settled; a retry converts it once that is over.
/// </summary>
Blocked,
/// <summary>Left as it was: a photo could not be converted (all or nothing), or the files could not move.</summary>
Failed,
}
/// <summary>
/// One sequence of a conversion: what happened, how many photos, the new sequence when one was made, the failure, and the
/// photos of the source a new sequence does not hold (those the plan did not convert stay in the source alone).
/// </summary>
public sealed record ConvertSequenceResult(Guid SequenceId, string Title, ConvertOutcome Outcome, int Frames, Guid? NewSequenceId = null, string? Error = null,
int LeftOut = 0)
{
/// <summary>Whether the sequence is done with: converted, made, with nothing to convert, or copied by another conversion since.</summary>
public bool Final => IsFinal(Outcome);
/// <summary>Whether <paramref name="outcome"/> leaves a sequence done with (<see cref="Final"/>).</summary>
internal static bool IsFinal(ConvertOutcome? outcome) =>
outcome is ConvertOutcome.Converted or ConvertOutcome.Created or ConvertOutcome.Nothing or ConvertOutcome.Superseded;
}
/// <summary>What a conversion did, and the passes the sequences it changed or made need (queued as follow-ups as each one ended).</summary>
public sealed record ConvertResult(IReadOnlyList<ConvertSequenceResult> Sequences, IReadOnlyList<ArchiveJobRequest> FollowUps)
{
/// <summary>Photos converted, into their sequence or a new one.</summary>
public int Frames => Sequences.Where(s => s.Outcome is ConvertOutcome.Converted or ConvertOutcome.Created).Sum(s => s.Frames);
/// <summary>The sequences left as they were (changed, blocked, failed).</summary>
public IEnumerable<ConvertSequenceResult> Kept => Sequences.Where(s => !s.Final);
}
/// <summary>
/// How a running conversion reports: progress in photos, a checkpoint after each sequence, job log lines, each sequence it
/// came to an end with in this run once its outcome is journaled (to tell the sequences it changed or made while the rest
/// of the job runs, as a Reorganize tells each step), and the passes a sequence it converted or made needs, to queue then
/// too (the journal says whether they were, so a pause, a yield, a stop or a crash later loses none).
/// </summary>
internal sealed record ConvertCallbacks(Func<long, long, ValueTask> Progress, Func<long, long, ValueTask> Checkpoint, Action<string> Log,
Func<ConvertSequenceResult, ValueTask>? Done = null, Func<IReadOnlyList<ArchiveJobRequest>, ValueTask>? Queue = null);
/// <summary>
/// Who asked for a conversion: the user's name, kept in the trash records of a Replace, and id, the owner of a new sequence
/// when its source has none; and when the plan was confirmed (the job queued): a copy another conversion made of a source
/// since then supersedes this one's (<see cref="ConvertOutcome.Superseded"/>). Null: no copy made by another counts.
/// </summary>
internal sealed record ConvertRequester(string? Name, Guid? Id, DateTimeOffset? Confirmed = null);
/// <summary>
/// A conversion of sequences a confirmed plan converts is queued, running or paused already, so its job was not queued
/// (<see cref="MassConverter.QueueAsync"/>): a dry run made now shows them busy.
/// </summary>
public sealed class ConversionQueuedException(IReadOnlyList<Guid> sequenceIds) : InvalidOperationException(Loc.L("Archive.Convert.AlreadyQueued"))
{
/// <summary>The sequences another conversion holds.</summary>
public IReadOnlyList<Guid> SequenceIds { get; } = sequenceIds;
}
/// <summary>
/// Mass conversion of existing sequences (owner review 2, R7; BRIEF §12.5): <see cref="PlanAsync(Archive, IReadOnlyList{Guid}, ConvertOptions, ConvertMode, CancellationToken)"/>
/// is the dry run (<see cref="ConvertPlanner"/> per sequence, what blocks it, the space needed against the free space where
/// the conversion writes, <see cref="ArchiveServiceOptions.FreeSpace"/>), and a <see cref="JobKind.Convert"/> job applies the
/// confirmed plan (queued by <see cref="QueueAsync"/>, which refuses a second conversion of the same sequences) one sequence
/// at a time. Under the sequence's lease, its sidecar settled where the index has it then (<see cref="SequenceWriter.ReadLocked"/>),
/// the job plans it again and takes the photos confirmed (<see cref="ConfirmedPhotos"/>), going on only while each is
/// still planned with its content confirmed (<see cref="ConvertSequence.Frames"/>); it pins them then (<see cref="PinLog"/>),
/// so a photo that came in, was verified or could be opened again since the plan stays as it is, on a retry too. Stage A,
/// without the lease: each photo is hashed against its record and converted (<see cref="FrameConverter"/>, several at once
/// as the host's parallelism and memory allow) into <c>.chronos/staging/convert-&lt;job&gt;/&lt;sequence&gt;/</c>, the free
/// space checked before each one, each staged file appended to the job's journal (<see cref="ConvertJournal"/>), so a resumed or retried job keeps the staged files
/// whose hash still verifies. Stage B, all or nothing, under the sequence's lease and while every photo pinned still holds
/// the content converted: <see cref="ConvertMode.Replace"/> swaps the frames' files (<see cref="FrameTrash.ReplaceAsync"/>:
/// the originals into one trash entry of kind <see cref="TrashKind.Replaced"/>, same ids and indices, tombstones naming the
/// replacements; the converted files named as filing names a photo of the sequence, <see cref="ConvertOptions.FileName"/>);
/// <see cref="ConvertMode.NewSequence"/> files the converted copies into a new sequence through the archive ingest (session
/// key <c>convert:&lt;job&gt;:&lt;source&gt;</c>, so a resume finds it again), titled "title (FORMAT)", owned as the source is,
/// with the source's tags, rating, notes and project, the source left as it is (the photos the plan did not convert stay
/// there alone, as the job log says). A retry or a resume finishes the copy it made already rather than make another, and
/// makes none of a source that another conversion copied in the same format since this one was confirmed
/// (<see cref="ConvertOutcome.Superseded"/>). A sequence whose photos changed, that is busy, or has a photo that fails is
/// left as it was and reported.
/// </summary>
public sealed class MassConverter
{
/// <summary>What the frames of a sequence no longer have once their files were replaced (<see cref="FrameTrash.ReplaceAsync"/>).</summary>
private const SequenceStatus Uncovered = SequenceStatus.AnalyzedLight | SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
/// <summary>How the session key of a new sequence a conversion makes begins (<see cref="SessionKey"/>).</summary>
private const string KeyPrefix = "convert:";
/// <summary>Held while a confirmed conversion is checked against those pending and queued (<see cref="QueueAsync"/>): one at a time in this process.</summary>
private static readonly SemaphoreSlim Queueing = new(1, 1);
private readonly ChronosDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIngest _ingest;
private readonly FrameTrash _frames;
private readonly SequenceTrash _trash;
private readonly ArchiveService _service;
internal MassConverter(ChronosDatabase db, SequenceWriter writer, ArchiveIngest ingest, FrameTrash frames, SequenceTrash trash, ArchiveService service)
{
_db = db;
_writer = writer;
_ingest = ingest;
_frames = frames;
_trash = trash;
_service = service;
}
/// <summary>
/// The dry run of converting <paramref name="sequenceIds"/> of <paramref name="archive"/>: per sequence the frames
/// converted and those skipped with why, the bytes before and after, what blocks it (busy, a conversion of it already
/// queued, offline, unreadable); the space needed and free; the fingerprint a confirmation sends back. Reads only.
/// </summary>
/// <exception cref="ArgumentException">The options are invalid, or a sequence is not in the archive.</exception>
public Task<ConvertPlan> PlanAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, ConvertOptions options, ConvertMode mode,
CancellationToken cancellationToken = default) => PlanAsync(archive, sequenceIds, options, mode, null, cancellationToken);
/// <param name="job">The job asking (a conversion planning again): what it holds does not make its sequences busy.</param>
internal async Task<ConvertPlan> PlanAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, ConvertOptions options, ConvertMode mode, Guid? job,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(archive);
ArgumentNullException.ThrowIfNull(sequenceIds);
ArgumentNullException.ThrowIfNull(options);
if (!options.IsValid) throw new ArgumentException("Unknown target or a quality outside 1–100.", nameof(options));
if (!Enum.IsDefined(mode)) throw new ArgumentException($"Unknown mode {mode}.", nameof(mode));
var paths = new ArchivePaths(archive.RootPath);
var online = ArchiveVolume.Probe(paths) == ArchiveAvailability.Online;
var busy = await HeldAsync(job, cancellationToken).ConfigureAwait(false);
var plans = new List<ConvertSequencePlan>();
foreach (var id in sequenceIds.Distinct())
{
cancellationToken.ThrowIfCancellationRequested();
var row = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
if (row is null || row.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Name}.", nameof(sequenceIds));
plans.Add(await PlanOneAsync(paths, row, online, busy.Contains(id) || busy.Contains(archive.Id), options, cancellationToken).ConfigureAwait(false));
}
var estimated = plans.Where(p => p.Converts).Sum(p => p.BytesAfter);
return new ConvertPlan(archive.Id, options, mode, plans, new ConvertSpace(ConvertPlanner.Required(estimated), FreeSpace(paths)));
}
/// <summary>
/// The sequences and archives a dry run counts as busy (<see cref="ConvertBlock.Busy"/>): what running and paused jobs hold,
/// and the sequences of every conversion waiting in the queue, so the same conversion is not confirmed twice. Reads only.
/// </summary>
public async Task<IReadOnlySet<Guid>> BusyAsync(CancellationToken cancellationToken = default) =>
await HeldAsync(null, cancellationToken).ConfigureAwait(false);
/// <summary>
/// What makes a sequence busy for a dry run (<paramref name="job"/> null: <see cref="BusyAsync"/>), or for the conversion
/// job <paramref name="job"/> planning again: what other running and paused jobs hold, and the sequences of the conversions
/// queued before it, which come first (a duplicate queued after it must not hold it back, and one queued before it, by an
/// earlier build or another tab, must not make a second copy of the same sequence).
/// </summary>
private async Task<HashSet<Guid>> HeldAsync(Guid? job, CancellationToken cancellationToken)
{
var busy = new HashSet<Guid>(await _trash.BusyAsync(job, cancellationToken).ConfigureAwait(false));
if (job is not { } self)
busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken).ConfigureAwait(false));
else if (await _db.Jobs.GetCreatedAsync(self, cancellationToken).ConfigureAwait(false) is { } queued)
busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, queued, cancellationToken).ConfigureAwait(false));
return busy;
}
/// <summary>
/// Queues the job of the confirmed <paramref name="plan"/> (<see cref="ArchiveJobs.Convert"/>) through <paramref name="enqueue"/>,
/// unless a conversion of a sequence it converts is queued, running or paused already: another dialog or tab, an answer
/// lost and Apply pressed again, a dry run made before that one was queued or none at all. Checked and queued under one
/// lock of this process, so two confirmations at once queue one job. First, which photos it converts are written under
/// the job's id beside the journal it starts (<see cref="ConfirmedPhotos"/>); <paramref name="enqueue"/> queues the
/// request under that id (<see cref="ArchiveJobRequest.Id"/>), and runs in the caller's context (a desktop's tells the
/// user on its UI thread). When they cannot be written (the archive went away since its dry run), the job is queued all
/// the same and holds each sequence to its whole plan.
/// </summary>
/// <exception cref="ConversionQueuedException">A conversion of one of its sequences waits or runs; nothing was queued.</exception>
/// <exception cref="ArgumentException">The plan converts nothing.</exception>
public async Task QueueAsync(Archive archive, ConvertPlan plan, Func<ArchiveJobRequest, Task> enqueue, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(enqueue);
var request = ArchiveJobs.Convert(archive, plan);
var paths = new ArchivePaths(archive.RootPath);
var job = request.Id!.Value;
// No ConfigureAwait(false) here: each wait comes back to the caller's context, where enqueue is called.
await Queueing.WaitAsync(cancellationToken);
try
{
var pending = await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken);
if (request.Targets.Where(pending.Contains).ToList() is { Count: > 0 } queued) throw new ConversionQueuedException(queued);
var written = await Task.Run(() => ConfirmedPhotos.TryWrite(paths, job, plan), cancellationToken);
try
{
await enqueue(request);
}
catch
{
// Not queued: no job will read them.
if (written) ConfirmedPhotos.Delete(paths, job);
throw;
}
}
finally
{
Queueing.Release();
}
}
private async Task<ConvertSequencePlan> PlanOneAsync(ArchivePaths paths, Sequence row, bool online, bool busy, ConvertOptions options,
CancellationToken cancellationToken)
{
List<ConvertBlock> blocked = [];
if (!online) blocked.Add(ConvertBlock.Offline);
if (busy) blocked.Add(ConvertBlock.Busy);
var empty = new ConvertSequencePlan(row.Id, row.DisplayTitle, row.Folder, row.FramesVersion, [], [], blocked);
// Created without photos (no folder yet), or nothing to read while offline.
if (!online || row.FrameCount == 0 && ArchiveIngest.AwaitsFolder(row, paths)) return empty;
var folder = paths.Sequence(row.Folder);
if (Read(folder, row.Id) is not { } manifest) return empty with { Blocked = [.. blocked, ConvertBlock.Unreadable] };
var decoder = _writer.Options.Decoder;
var plan = await Task.Run(() => ConvertPlanner.Plan(manifest, folder, FileRenames.ListPresent(folder, manifest), options, decoder, row.DisplayTitle),
cancellationToken).ConfigureAwait(false);
return plan with { Blocked = blocked };
}
/// <summary>
/// Applies a confirmed conversion (<see cref="ConvertJobParameters"/>) as the job <paramref name="jobId"/>, resuming from its
/// journal: settles the sidecars a crash left half-way, refuses when the volume lacks the space the photos still to convert
/// need, then converts each sequence while what it converts is still what was confirmed (stage A, then stage B), the space
/// measured again before each. The passes a sequence converted or made needs are queued as soon as it is journaled.
/// Sequences left as they were are in the result; the journal and their staged files stay while there are any, so a retry
/// tries them again and knows what was done.
/// </summary>
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only; the job resumes where it stopped.</exception>
/// <exception cref="IOException">The volume lacks the space (<c>Archive.Error.ConvertSpace</c>); what was converted stays, and a retry resumes.</exception>
internal async Task<ConvertResult> RunAsync(Archive archive, ConvertJobParameters parameters, Guid jobId, ConvertRequester by, ConvertCallbacks callbacks,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(archive);
ArgumentNullException.ThrowIfNull(parameters);
ArgumentNullException.ThrowIfNull(by);
ArgumentNullException.ThrowIfNull(callbacks);
var options = new ConvertOptions(parameters.Target, parameters.Quality);
if (!options.IsValid || !Enum.IsDefined(parameters.Mode)) throw new InvalidOperationException($"Convert job {jobId} has invalid options.");
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var journal = ConvertJournal.Read(paths, jobId) ?? new ConvertJournal { JobId = jobId };
var steps = parameters.Sequences.DistinctBy(s => s.Id).ToList();
var staged = journal.Staged(paths);
var pins = journal.Pins(paths);
var confirmed = ConfirmedPhotos.Read(paths, jobId);
// What is still to do: every sequence not final in an earlier attempt that is still in the archive, its sidecar settled
// first when a writer a crash interrupted left a journal in its folder (this job's replacement among them).
var rows = new Dictionary<Guid, Sequence>();
foreach (var step in steps.Where(s => journal.Of(s.Id)?.Final != true))
{
if (await _db.Sequences.GetAsync(step.Id, cancellationToken).ConfigureAwait(false) is { } row && row.ArchiveId == archive.Id)
rows[step.Id] = await SettledRowAsync(archive, paths, row, cancellationToken).ConfigureAwait(false);
}
var plan = await PlanAsync(archive, [.. rows.Keys], options, parameters.Mode, jobId, cancellationToken).ConfigureAwait(false);
if (plan.Sequences.Any(p => p.Blocked.Contains(ConvertBlock.Offline))) throw new ArchiveOfflineException(paths.Root, ArchiveVolume.Probe(paths));
// What each of them converts, as far as the job can tell before it comes to it (what it pinned, or the plan still confirmed),
// and the bytes its photos not staged yet are estimated at.
var byId = steps.ToDictionary(s => s.Id);
var progress = new Progress();
var unstaged = new Dictionary<Guid, long>();
foreach (var p in plan.Sequences.Where(p => p.Blocked.Count == 0))
{
if (Confirmed(byId[p.SequenceId], confirmed.Of(p.SequenceId), p, pins.Of(p.SequenceId)) is not { Frames.Count: > 0 } converts) continue;
progress.Count(converts.SequenceId, converts.Frames.Count, staged.Of(converts.SequenceId, converts.FramesVersion).Count);
// What a new sequence this job made holds already takes no space again.
var filed = parameters.Mode == ConvertMode.NewSequence
? await FiledAsync(archive, SessionKey(jobId, converts.SequenceId), cancellationToken).ConfigureAwait(false)
: null;
unstaged[converts.SequenceId] = Unstaged(paths, jobId, converts, staged, filed);
}
var space = new SpaceWatch(() => FreeSpace(paths));
EnsureSpace(unstaged.Values.Sum(), space);
await callbacks.Progress(progress.Done, progress.Total).ConfigureAwait(false);
var results = new List<ConvertSequenceResult>();
var followUps = new List<ArchiveJobRequest>();
foreach (var step in steps)
{
cancellationToken.ThrowIfCancellationRequested();
var noted = journal.Of(step.Id);
var final = noted?.Final == true;
StepDone done;
if (final)
{
// Done with in an earlier attempt, and the passes it needs too unless a pause, a yield, a stop or a crash came first.
done = new StepDone(new ConvertSequenceResult(step.Id, noted!.Title ?? step.Id.ToString(), noted.Outcome!.Value, noted.Frames, noted.NewSequenceId),
noted.Had, noted.Queued ? null : await FollowUpsAsync(archive, noted, cancellationToken).ConfigureAwait(false));
}
else if (!rows.TryGetValue(step.Id, out var row))
{
done = new StepDone(new ConvertSequenceResult(step.Id, step.Id.ToString(), ConvertOutcome.Changed, 0));
}
else
{
ArchiveVolume.EnsureWritable(paths);
// The space for what is left, measured again: the volume may have filled since, and the files staged so far tell
// how far off the estimate is.
EnsureSpace(unstaged.Values.Sum(), space);
done = await ConvertOneAsync(archive, paths, step, row, options, parameters.Mode, jobId, by, journal, confirmed, pins, staged, progress, space,
callbacks, cancellationToken).ConfigureAwait(false);
}
unstaged.Remove(step.Id);
var result = done.Result;
results.Add(result);
if (!final)
{
journal.Note(result, done.Had);
journal.Write(paths);
// What a sequence done with left staged, or staged for photos that are no longer the ones confirmed, is of no use
// to anything. A sequence blocked or failed keeps its staged files, which a retry verifies and takes.
if (result.Final || result.Outcome == ConvertOutcome.Changed) DeleteStaging(StagingOf(paths, jobId, step.Id));
Log(callbacks, result);
if (callbacks.Done is { } told) await told(result).ConfigureAwait(false);
}
if (done.FollowUps is { } passes)
{
if (passes.Count > 0)
{
followUps.AddRange(passes);
if (callbacks.Queue is { } queue) await queue(passes).ConfigureAwait(false);
}
journal.Of(step.Id)!.Queued = true;
journal.Write(paths);
}
await callbacks.Checkpoint(progress.Done, progress.Total).ConfigureAwait(false);
}
// Done with every sequence: what the job staged and its journal go. Otherwise both stay for a retry, until the job is gone
// from the history (ArchiveMaintenance).
if (results.All(r => r.Final))
{
DeleteStaging(Path.Combine(paths.StagingDirectory, StagingFolder(jobId)));
ConvertJournal.Delete(paths, jobId);
}
return new ConvertResult(results, followUps);
}
/// <summary>
/// One sequence: under its lease, its sidecar settled, planned again and held to what was confirmed (the photos pinned the
/// first time); then stage A without the lease, and stage B. A New sequence that has its copy already is not converted
/// again (<see cref="CopiedAsync"/>).
/// </summary>
private async Task<StepDone> ConvertOneAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options,
ConvertMode mode, Guid jobId, ConvertRequester by, ConvertJournal journal, ConfirmedPhotos confirmedPhotos, PinLog pins, StagedLog staged,
Progress progress, SpaceWatch space, ConvertCallbacks callbacks, CancellationToken cancellationToken)
{
// Another job holds it now (a Reorganize paused since the job started, a render, a conversion of it queued first): it
// waits for the retry.
var busy = await HeldAsync(jobId, cancellationToken).ConfigureAwait(false);
if (busy.Contains(step.Id) || busy.Contains(archive.Id)) return new StepDone(Result(row, ConvertOutcome.Blocked));
var key = SessionKey(jobId, row.Id);
if (mode == ConvertMode.NewSequence)
{
try
{
if (await CopiedAsync(archive, paths, step, row, options, jobId, by, journal, pins, staged, key, progress, cancellationToken).ConfigureAwait(false)
is { } copied)
return copied;
}
catch (ArchiveOfflineException)
{
throw;
}
catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
{
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the copy this conversion made of it could not be finished yet", ex);
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex)));
}
}
var photos = confirmedPhotos.Of(step.Id);
SequencePaths folder;
SequenceManifest manifest;
ConvertSequencePlan planned;
try
{
await using (await _writer.LockAsync(paths, step.Id, cancellationToken).ConfigureAwait(false))
{
// Where the index has it now (a Reorganize may have moved its folder since the job planned), its sidecar as every
// writer reads it: a rename of its files, a deletion, a replacement, a restore or an ingest a crash left half-way is
// settled first, so the plan sees each photo where it is.
if (await _db.Sequences.GetAsync(step.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id)
return new StepDone(Result(row, ConvertOutcome.Changed));
row = now;
folder = paths.Sequence(row.Folder);
// Moved away by another index of the archive, maybe with another sequence in the folder it left (this index's next
// scan finds both): nothing of it is settled, staged or replaced there; the retry after that scan converts it.
if (!SequenceFolders.Holds(folder, row.Id)) return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Errors.Archive.FolderMoved")));
manifest = SequenceWriter.ReadLocked(folder, row.Id);
FrameTrash.EnsureReplaced(folder);
var pin = pins.Of(step.Id);
if (mode == ConvertMode.Replace && ReplacedEarlier(manifest, step, photos, pin, staged.Of(step.Id, step.FramesVersion), options) is { } frames)
{
// Replaced by this job already: a crash after its stage B, before the journal said so (the settle may just have
// completed it).
var synced = await SyncedAsync(archive, folder, row, manifest).ConfigureAwait(false);
var had = journal.Of(step.Id)?.Had ?? row.Status;
return new StepDone(Result(synced, ConvertOutcome.Converted, frames), had, _frames.FollowUps(archive, synced, had));
}
var (sidecar, at, title, decoder) = (manifest, folder, row.DisplayTitle, _writer.Options.Decoder);
var (fresh, present) = await Task.Run(() =>
{
// As the planner keeps them (by name, whatever the case), so the photos left out are asked again alike.
IReadOnlySet<string> files = new HashSet<string>(FileRenames.ListPresent(at, sidecar), StringComparer.OrdinalIgnoreCase);
return (ConvertPlanner.Plan(sidecar, at, files, options, decoder, title), files);
}, cancellationToken).ConfigureAwait(false);
if (Confirmed(step, photos, fresh, pin) is not { } confirmed)
{
// A photo it converts that another program holds now cannot be told from one that changed: the retry, once it
// is free, can.
if (await Task.Run(() => Held(photos, pin, sidecar, at, present, fresh, options, decoder), cancellationToken).ConfigureAwait(false))
{
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: a photo it converts cannot be opened now; it is not converted");
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.PhotoHeld")));
}
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the photos it converts are no longer those of the plan confirmed; it is not converted");
return new StepDone(Result(row, ConvertOutcome.Changed));
}
planned = confirmed;
if (pin is null)
{
// From here on the job converts these photos of it, whatever comes in or is verified meanwhile, on a retry too.
pins.Append(step.Id, planned);
journal.Begin(step.Id, row.DisplayTitle, row.Status);
journal.Write(paths);
}
}
}
catch (ArchiveOfflineException)
{
throw;
}
catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
{
Core.Diagnostics.Log.Warning("archive", $"The sidecar of {row.DisplayTitle} cannot be read; it is not converted", ex);
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex)));
}
var staging = StagingOf(paths, jobId, row.Id);
// A new sequence a resumed job finds again holds the photos it filed before it stopped.
var filed = mode == ConvertMode.NewSequence
? await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false)
: new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var noted = staged.Of(row.Id, planned.FramesVersion);
var ready = await Task.Run(() => Ready(manifest, planned, noted, staging, filed), cancellationToken).ConfigureAwait(false);
progress.Count(row.Id, planned.Frames.Count, ready.Count);
await callbacks.Progress(progress.Done, progress.Total).ConfigureAwait(false);
var failure = await StageAsync(row, manifest, folder, planned, ready, staging, options, staged, progress, space, callbacks, cancellationToken)
.ConfigureAwait(false);
if (failure is not null) return new StepDone(Result(row, ConvertOutcome.Failed, error: failure));
try
{
return mode == ConvertMode.Replace
? await ReplaceAsync(archive, paths, row, planned, ready, staging, options, jobId, by.Name, cancellationToken).ConfigureAwait(false)
: await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, journal, cancellationToken).ConfigureAwait(false);
}
catch (LeaseUnavailableException ex)
{
return new StepDone(Result(row, ConvertOutcome.Blocked, error: ex.Message));
}
catch (ArchiveOfflineException)
{
throw;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
{
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the converted photos could not take their place", ex);
var cause = ex switch
{
UnstagedException => ex.Message,
UnauthorizedAccessException => Loc.L("Archive.Failed.Access"),
_ => Loc.L("Archive.Failed.Files"),
};
return new StepDone(Result(row, ConvertOutcome.Failed, error: Loc.L("Archive.Convert.NotPlaced", cause)));
}
}
/// <summary>
/// Why a sequence's sidecar could not be settled or read under its lease, as one catalog sentence for the job log (the
/// exception goes to the diagnostics log): another writer holds it, access is denied, or the file is not its sidecar.
/// </summary>
private static string Blocker(Exception ex) => ex switch
{
LeaseUnavailableException => ex.Message,
UnauthorizedAccessException => Loc.L("Archive.Failed.Access"),
InvalidDataException or InvalidOperationException => Loc.L("Archive.Convert.Blocked.Unreadable"),
_ => Loc.L("Archive.Failed.Files"),
};
/// <summary>
/// What a <see cref="ConvertMode.NewSequence"/> conversion does with a source that has its copy already when the job comes
/// to it (a retry, a resume): the copy this job made (its journal names it, or its session key when a crash came before
/// the journal did) is finished, whatever changed in the source since, once it holds every photo the job pinned, and one
/// gone since is not made again without a new plan; a copy in this format that another conversion made since this one was
/// confirmed (<see cref="ConvertRequester.Confirmed"/>) supersedes this one's. Null when neither: the sequence converts,
/// and a copy holding only some of its photos (a crash while they were filed) gets the rest.
/// </summary>
private async Task<StepDone?> CopiedAsync(Archive archive, ArchivePaths paths, ConvertSequence step, Sequence row, ConvertOptions options, Guid jobId,
ConvertRequester by, ConvertJournal journal, PinLog pins, StagedLog staged, string key, Progress progress, CancellationToken cancellationToken)
{
if ((journal.Of(step.Id)?.NewSequenceId ?? await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, cancellationToken).ConfigureAwait(false)) is { } made)
{
if (await _db.Sequences.GetAsync(made, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id)
{
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it is gone; no other is made without a new plan");
return new StepDone(Result(row, ConvertOutcome.Changed));
}
return pins.Of(step.Id) is { } pin
? await FinishCopyAsync(archive, paths, row, pin, staged, key, made, progress, cancellationToken).ConfigureAwait(false)
: null;
}
if (by.Confirmed is not { } confirmed
|| await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } later)
return null;
var (other, done) = later;
if (!done)
{
// Its job has not finished the copy (it failed while filing it, or still runs): neither a second copy nor done with.
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed and has not finished it");
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.CopyUnfinished")));
}
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed; no second copy is made");
progress.Count(row.Id, 0, 0);
return new StepDone(Result(row, ConvertOutcome.Superseded));
}
/// <summary>
/// The copy this job made of a source, finished when it holds the conversion of every photo pinned (a crash came after
/// they were filed, before the journal said so): it takes the source's tags, rating, notes and project as stage B gives
/// them, whatever became of the photos of the source since. Null while a photo pinned is not in it, or when the source
/// is no longer where this index has it: the job converts the sequence, which files the rest.
/// </summary>
private async Task<StepDone?> FinishCopyAsync(Archive archive, ArchivePaths paths, Sequence row, PinnedSequence pin, StagedLog staged, string key, Guid made,
Progress progress, CancellationToken cancellationToken)
{
var conversions = staged.Of(row.Id, pin.FramesVersion).ToDictionary(c => c.FrameId);
var filed = await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false);
var ready = new Dictionary<Guid, StagedConversion>();
foreach (var photo in pin.Frames)
{
if (!conversions.TryGetValue(photo.Id, out var conversion) || !string.Equals(conversion.Source, photo.Sha256, StringComparison.OrdinalIgnoreCase)
|| !filed.Contains(conversion.Sha256))
return null;
ready[photo.Id] = conversion;
}
SequenceManifest manifest;
string? project;
await using (await _writer.LockAsync(paths, row.Id, cancellationToken).ConfigureAwait(false))
{
if (await _db.Sequences.GetAsync(row.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id) return null;
row = now;
var folder = paths.Sequence(row.Folder);
if (!SequenceFolders.Holds(folder, row.Id)) return null;
manifest = SequenceWriter.ReadLocked(folder, row.Id);
project = row.SettingsJson ?? (File.Exists(folder.ProjectPath) ? AtomicFile.ReadAllText(folder.ProjectPath) : null);
}
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the copy this conversion made of it holds every photo already; it is finished, no other is made");
progress.Count(row.Id, pin.Frames.Count, pin.Frames.Count);
return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, made, pin.Frames.Count, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// A copy of <paramref name="source"/> in this conversion's format that another conversion made since <paramref name="confirmed"/>:
/// a sequence filed under another job's session key for that source (<see cref="SessionKey"/>) whose sidecar says it was
/// made since then and holds photos of that format, and whether that conversion is done (only a done one's copy is whole).
/// Null when there is none.
/// </summary>
private async Task<(Guid Id, bool Done)?> LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed,
CancellationToken cancellationToken)
{
foreach (var (key, id) in await _db.Sequences.ListSessionKeysAsync(archive.Id, $"{KeyPrefix}%:{source:N}", cancellationToken).ConfigureAwait(false))
{
if (JobOfKey(key) is not { } other || other == jobId) continue;
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id) continue;
if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format))
return (id, await _db.Jobs.GetAsync(other, cancellationToken).ConfigureAwait(false) is { State: JobState.Done });
}
return null;
}
/// <summary>
/// What the job converts of a sequence whose plan is now <paramref name="fresh"/>: the photos it pinned when it first came to
/// it, each still planned with the content it had then; the first time, the photos confirmed (<paramref name="photos"/>,
/// <see cref="ConfirmedPhotos"/>) while each is still planned with the content confirmed (the step's digest). Photos that
/// came in, were verified or could be opened again since stay as they are, so an upload while the job waits its turn
/// neither changes what it converts nor holds it back. Without the photos confirmed (they could not be written, or a job
/// an earlier build queued): the whole plan while it has the step's digest, or its frames version. Null when a photo
/// confirmed no longer converts as it was: the sequence changed after the plan, and the user plans again.
/// </summary>
internal static ConvertSequencePlan? Confirmed(ConvertSequence step, IReadOnlyList<Guid>? photos, ConvertSequencePlan fresh, PinnedSequence? pin)
{
var now = fresh.Frames.ToDictionary(f => f.FrameId);
if (pin is not null)
{
var frames = new List<ConvertFramePlan>(pin.Frames.Count);
foreach (var pinned in pin.Frames)
{
if (!now.TryGetValue(pinned.Id, out var frame) || !string.Equals(frame.Sha256, pinned.Sha256, StringComparison.OrdinalIgnoreCase)) return null;
frames.Add(frame);
}
return frames.Count > 0 ? fresh with { FramesVersion = pin.FramesVersion, Frames = frames } : null;
}
if (step is { Frames: { } confirmedDigest } && photos is { Count: > 0 })
{
var frames = new List<ConvertFramePlan>(photos.Count);
foreach (var id in photos)
{
if (!now.TryGetValue(id, out var frame)) return null;
frames.Add(frame);
}
var held = ConvertSequencePlan.DigestOf(frames.Select(f => (f.FrameId, f.Sha256)));
return string.Equals(held, confirmedDigest, StringComparison.Ordinal) ? fresh with { Frames = frames } : null;
}
if (fresh.Frames.Count == 0) return null;
var confirmed = step.Frames is { } digest
? string.Equals(fresh.Digest(), digest, StringComparison.Ordinal)
: fresh.FramesVersion == step.FramesVersion;
return confirmed ? fresh : null;
}
/// <summary>
/// Whether the plan made now (<paramref name="fresh"/>) leaves out a photo the job converts of the sequence (one it pinned,
/// else one of <paramref name="photos"/>, those confirmed) only because its file could not be opened
/// (<see cref="ConvertSkip.Inaccessible"/>: another program holds it): the sequence then waits for the retry rather than
/// counting as changed. Photos the job does not convert do not count, nor does a confirmed photo that changed meanwhile.
/// Without the photos confirmed: any photo of the sequence that could not be opened.
/// </summary>
private static bool Held(IReadOnlyList<Guid>? photos, PinnedSequence? pin, SequenceManifest manifest, SequencePaths folder, IReadOnlySet<string> present,
ConvertSequencePlan fresh, ConvertOptions options, ImageDecoder decoder)
{
IReadOnlyList<Guid>? converts = pin is not null ? [.. pin.Frames.Select(f => f.Id)] : photos;
if (converts is null) return fresh.Skipped.Any(s => s.Reason == ConvertSkip.Inaccessible);
var planned = fresh.Frames.Select(f => f.FrameId).ToHashSet();
var frames = manifest.Frames.ToDictionary(f => f.Id);
return converts.Any(id => !planned.Contains(id) && frames.TryGetValue(id, out var frame)
&& ConvertPlanner.Skip(frame, folder, present, options, decoder) == ConvertSkip.Inaccessible);
}
/// <summary>Whether every photo of <paramref name="planned"/> is a frame of <paramref name="manifest"/> still holding the content it was planned with.</summary>
private static bool Holds(SequenceManifest manifest, ConvertSequencePlan planned)
{
var frames = manifest.Frames.ToDictionary(f => f.Id);
return planned.Frames.All(p => frames.TryGetValue(p.FrameId, out var f) && string.Equals(f.Sha256, p.Sha256, StringComparison.OrdinalIgnoreCase));
}
private static ConvertSequenceResult Result(Sequence row, ConvertOutcome outcome, int frames = 0, string? error = null) =>
new(row.Id, row.DisplayTitle, outcome, frames, Error: error);
/// <summary>
/// The staged conversions of a sequence that still hold: of a frame the plan converts, at the content it had then,
/// whose staged file is there with its size and SHA-256 (re-hashed now: a pause, a crash or a clean-up may have taken it
/// since), or which a new sequence holds already (<paramref name="filed"/>).
/// </summary>
private static Dictionary<Guid, StagedConversion> Ready(SequenceManifest manifest, ConvertSequencePlan planned, IReadOnlyList<StagedConversion> noted,
string staging, IReadOnlySet<string> filed)
{
var frames = manifest.Frames.ToDictionary(f => f.Id);
var converted = planned.Frames.Select(f => f.FrameId).ToHashSet();
var ready = new Dictionary<Guid, StagedConversion>();
foreach (var conversion in noted)
{
if (!converted.Contains(conversion.FrameId) || !frames.TryGetValue(conversion.FrameId, out var frame)
|| !string.Equals(frame.Sha256, conversion.Source, StringComparison.OrdinalIgnoreCase))
continue;
if (filed.Contains(conversion.Sha256) || Intact(Path.Combine(staging, conversion.File), conversion)) ready[conversion.FrameId] = conversion;
}
return ready;
}
private static bool Intact(string path, StagedConversion conversion)
{
if (!Present(path, conversion)) return false;
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.SequentialScan);
return string.Equals(Convert.ToHexStringLower(SHA256.HashData(stream)), conversion.Sha256, StringComparison.OrdinalIgnoreCase);
}
/// <summary>Whether a staged conversion's file is there at its size (not re-hashed).</summary>
private static bool Present(string path, StagedConversion conversion) => new FileInfo(path) is { Exists: true } file && file.Length == conversion.Size;
/// <summary>
/// Stage A: converts the planned frames not staged yet, several at once, each staged file appended to the journal as it
/// is consumed (in frame order). A photo starts only while the volume has the space for it and for every photo handed to
/// the work before it whose file has not landed yet (<see cref="SpaceWatch.Start"/>: converting or waiting for its turn,
/// up to twice as many as convert at once, at the end of a sequence as anywhere), with the margin and the reserve: short
/// of it the job stops (<c>Archive.Error.ConvertSpace</c>) before the volume fills. A converted file is encoded in memory
/// and lands whole, by a rename, before it is verified: what those photos will write is not in the free space measured
/// yet, and what landed is, so each counts until its file lands, never twice. Returns the first failure (the file and why,
/// localised; the exception goes to the diagnostics log); the frames converted meanwhile stay staged for a retry.
/// </summary>
/// <exception cref="IOException">The volume lacks the space for the next photo; what was staged stays.</exception>
private async Task<string?> StageAsync(Sequence row, SequenceManifest manifest, SequencePaths folder, ConvertSequencePlan planned,
Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options, StagedLog staged, Progress progress, SpaceWatch space,
ConvertCallbacks callbacks, CancellationToken cancellationToken)
{
var frames = manifest.Frames.ToDictionary(f => f.Id);
var todo = planned.Frames.Where(f => !ready.ContainsKey(f.FrameId)).ToList();
if (todo.Count == 0) return null;
System.IO.Directory.CreateDirectory(staging);
var decoder = _writer.Options.Decoder;
var largest = todo.Max(f => (long)Math.Max(frames[f.FrameId].W, 0) * Math.Max(frames[f.FrameId].H, 0));
// Photos at once as the host's parallelism and memory allow; each encoder then works alone.
var atOnce = DecodeBudget.PhotosAtOnce(_writer.Options.Parallelism(), _writer.Options.MemoryBudget(), largest);
var threads = atOnce > 1 ? 1 : -1;
var estimates = todo.ToDictionary(f => f.FrameId, f => f.EstimatedBytes);
string? failure = null;
Exception? error = null;
string? full = null;
await using var work = new OrderedWork<Staged>(atOnce, async result =>
{
if (result.Conversion is not { } conversion)
{
if (failure is null)
{
error = result.Error;
failure = Loc.L("Archive.Convert.PhotoFailed", result.File, FrameConverter.FailureText(result.Error!));
}
return;
}
staged.Append(row.Id, planned.FramesVersion, conversion);
ready[conversion.FrameId] = conversion;
space.Staged(conversion.Size, estimates[conversion.FrameId]);
await callbacks.Progress(++progress.Done, progress.Total).ConfigureAwait(false);
}, cancellationToken);
for (var i = 0; i < todo.Count && failure is null; i++)
{
var frame = frames[todo[i].FrameId];
var target = Path.Combine(staging, frame.Id.ToString("N") + options.Extension);
if (space.Start(todo[i].EstimatedBytes, target) is { } message)
{
full = message;
break;
}
// A photo that never starts (the job stops: cancelled, or a write of the journal failed) is never counted off,
// and the watch goes with the job.
await work.AddAsync(token =>
{
try
{
return Stage(frame, manifest, folder, target, options, decoder, threads, token);
}
finally
{
space.Ended(target);
}
}).ConfigureAwait(false);
}
await work.CompleteAsync().ConfigureAwait(false);
if (full is not null)
{
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the conversion stops, {full}");
throw new IOException(full);
}
if (failure is not null) Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: not converted", error);
return failure;
}
/// <summary>One photo of stage A: hashed against its record, converted into <paramref name="target"/> (in the staging folder) and verified.</summary>
private static Staged Stage(SidecarFrame frame, SequenceManifest manifest, SequencePaths folder, string target, ConvertOptions options, ImageDecoder decoder,
int threads, CancellationToken cancellationToken)
{
var source = folder.File(frame.File);
try
{
// Converted only as archived: a photo whose bytes no longer match its hash waits for a verify.
string sha;
using (var stream = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, 1 << 20, FileOptions.SequentialScan))
sha = Convert.ToHexStringLower(SHA256.HashData(stream));
if (!string.Equals(sha, frame.Sha256, StringComparison.OrdinalIgnoreCase))
throw new ConversionException(ConversionFailure.Changed, $"{frame.File} no longer matches its SHA-256; verify the sequence");
cancellationToken.ThrowIfCancellationRequested();
var file = FrameConverter.Convert(new ConvertSource(source, frame, manifest.Camera, manifest.Lens), options, decoder, target, threads, cancellationToken);
return new Staged(frame.File, new StagedConversion(frame.Id, sha, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height), null);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or NotSupportedException or FormatException
or InvalidOperationException or ArgumentException)
{
return new Staged(frame.File, null, ex);
}
}
/// <summary>
/// Stage B of <see cref="ConvertMode.Replace"/>: the staged files take their frames' place (<see cref="FrameTrash.ReplaceAsync"/>)
/// while every photo pinned still holds the content converted (photos that came in since stay as they are, so the frames
/// version is not held to the plan's), each named as filing names a photo of the sequence (<see cref="ConvertOptions.FileName"/>:
/// the archive's file template, the frame and the sequence as the sidecar has them under the lease), made unique among
/// every file there.
/// </summary>
private async Task<StepDone> ReplaceAsync(Archive archive, ArchivePaths paths, Sequence row, ConvertSequencePlan planned,
Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options, Guid jobId, string? replacedBy, CancellationToken cancellationToken)
{
var template = PathTemplate.Parse(archive.FileTemplate);
var swaps = planned.Frames.Select(f =>
{
var conversion = ready[f.FrameId];
return new FrameSwap(f.FrameId, paths.ToRelative(Path.Combine(staging, conversion.File)), [], (current, sequence) => options.FileName(template, current, sequence),
current => Converted(current, conversion, options));
}).ToList();
var replaced = await _frames.ReplaceAsync(archive, row.Id, null, swaps, replacedBy, jobId, null, (now, _, _) => Task.FromResult(Holds(now, planned)),
cancellationToken).ConfigureAwait(false);
if (replaced is null) return new StepDone(Result(row, ConvertOutcome.Changed));
return new StepDone(Result(row, ConvertOutcome.Converted, swaps.Count), replaced.Had, _frames.FollowUps(archive, replaced.Sequence, replaced.Had));
}
/// <summary>
/// A converted frame's record: the frame as it is with the converted file's size, hash, format and upright size, read at
/// full-resolution tier, without the embedded preview it no longer has, without the flags of its old file, and with the
/// old file as its <see cref="SidecarFrame.Source"/>; capture time, exposure and the other facts stay.
/// </summary>
internal static SidecarFrame Converted(SidecarFrame current, StagedConversion conversion, ConvertOptions options) => current with
{
Size = conversion.Size,
Sha256 = conversion.Sha256.ToLowerInvariant(),
Format = options.Format,
W = conversion.Width,
H = conversion.Height,
Orientation = 1,
Tier = DecodeTier.Native,
PreviewOffset = null,
PreviewLength = null,
PreviewW = null,
PreviewH = null,
Flags = current.Flags & ~FrameTrash.ContentFlags,
Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size },
};
/// <summary>
/// Stage B of <see cref="ConvertMode.NewSequence"/>: once the source's photos pinned are checked under its lease as a Replace
/// checks them, the staged files filed into the sequence of <paramref name="key"/> (made on first use, titled "title (FORMAT)",
/// owned by the source's owner, else by who asked for it) and named as the ingest names every photo it files there
/// (<see cref="Core.Naming.FrameNaming"/>: <c>{name}</c> the original's name on the card, <c>{ext}</c> the target's), with the
/// source's capture facts; then the source's tags, rating, notes and project (its positions translated to the new frames).
/// Every conversion not filed yet must still be staged whole: one gone since (a clean-up, a quarantine) fails the sequence
/// rather than leave its photo out. The photos the plan did not convert are not in the new sequence (<see cref="ConvertSequenceResult.LeftOut"/>).
/// Once its photos are filed, the journal names the new sequence (<see cref="ConvertJournal.Made"/>), so that a retry or a
/// resume finishes it and never makes a second one.
/// </summary>
private async Task<StepDone> CreateAsync(Archive archive, ArchivePaths paths, Sequence row, ConvertSequencePlan planned,
Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options, string key, Guid? requestedBy, ConvertJournal journal,
CancellationToken cancellationToken)
{
SequenceManifest manifest;
string? project;
await using (await _writer.LockAsync(paths, row.Id, cancellationToken).ConfigureAwait(false))
{
if (await _db.Sequences.GetAsync(row.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id)
return new StepDone(Result(row, ConvertOutcome.Changed));
row = now;
var folder = paths.Sequence(row.Folder);
manifest = SequenceWriter.ReadLocked(folder, row.Id);
if (!Holds(manifest, planned)) return new StepDone(Result(row, ConvertOutcome.Changed));
// The source's project, read where its sidecar was just read as its own: another index may move the sequence away
// later, and maybe another one into this folder, whose project the new sequence must not take.
project = row.SettingsJson ?? (File.Exists(folder.ProjectPath) ? AtomicFile.ReadAllText(folder.ProjectPath) : null);
}
var frames = manifest.Frames.ToDictionary(f => f.Id);
var filed = await FiledAsync(archive, key, cancellationToken).ConfigureAwait(false);
var pending = new List<StagedFrame>();
foreach (var converted in planned.Frames)
{
var conversion = ready[converted.FrameId];
if (filed.Contains(conversion.Sha256)) continue;
var frame = frames[converted.FrameId];
var path = Path.Combine(staging, conversion.File);
if (!Present(path, conversion)) throw new UnstagedException(frame.File, path);
pending.Add(new StagedFrame(path, frame.Original ?? Path.GetFileName(frame.File), conversion.Size, conversion.Sha256, Metadata(path, frame), null, null)
{
NamedAs = options.WithExtension(Path.GetFileName(frame.File)),
});
}
var title = $"{manifest.Title ?? row.DisplayTitle} ({options.Label})";
// A user id, as every filing records it (the owner a row is given), never a user name.
var owner = (row.OwnerUserId ?? requestedBy)?.ToString();
// Once the first file moves, the commit runs to its end whatever the caller cancels meanwhile.
if (pending.Count > 0)
await _ingest.CommitSessionAsync(archive, new SequenceTarget(key, title, null, owner), pending, CancellationToken.None).ConfigureAwait(false);
var id = await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, CancellationToken.None).ConfigureAwait(false)
?? throw new IOException($"No converted photo of {row.DisplayTitle} could be filed into a new sequence.");
journal.Made(row.Id, id);
journal.Write(paths);
return await CompleteCopyAsync(archive, paths, row, manifest, project, ready, id, planned.Frames.Count, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// The end of stage B of <see cref="ConvertMode.NewSequence"/>, once the new sequence <paramref name="id"/> holds the
/// <paramref name="frames"/> photos converted: the source's tags, rating, notes and project given to it (<see cref="CopyAsync"/>),
/// and the passes it needs, from what the source has. The photos of the source it does not hold are counted as left out.
/// </summary>
private async Task<StepDone> CompleteCopyAsync(Archive archive, ArchivePaths paths, Sequence row, SequenceManifest manifest, string? project,
Dictionary<Guid, StagedConversion> ready, Guid id, int frames, CancellationToken cancellationToken)
{
await CopyAsync(paths, row, manifest, project, ready, id, cancellationToken).ConfigureAwait(false);
var made = await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false)
?? throw new InvalidOperationException($"The new sequence {id} is not indexed.");
var leftOut = manifest.Frames.Count(f => !ready.ContainsKey(f.Id));
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: {frames} photo(s) converted into the new sequence {made.DisplayTitle}, {leftOut} left in the source alone");
var had = row.Status & (SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt);
return new StepDone(new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Created, frames, id, LeftOut: leftOut), had,
_frames.FollowUps(archive, made, had));
}
/// <summary>
/// The metadata a converted copy is filed with: what its file says, with the source frame's capture time, offset and
/// time source (the sidecar wins, to the tick) and its intervalometer flag, which the MakerNote left behind carried.
/// </summary>
private static FrameMetadata Metadata(string path, SidecarFrame source)
{
var read = MetadataReader.Read(path, CancellationToken.None);
var interval = (source.Flags & FrameFlags.Intervalometer) != 0 ? true : read.IntervalShooting;
return source.CapturedAt is { } captured
? read with { CapturedAt = captured, UtcOffsetMinutes = source.UtcOffset, TimestampSource = source.TimeSource, IntervalShooting = interval }
: read with { IntervalShooting = interval };
}
/// <summary>
/// Gives the new sequence the source's tags, rating and notes, and its project (<paramref name="json"/>, read under the
/// source's lease) with the frame positions (exclusions, hero, keyframes) moved from the source's frames to the copies of
/// them; a project that cannot be read is left out (logged). The project is written under the new sequence's lease into
/// its folder as the index has it then, which must hold it (<see cref="SequenceFolders.Holds"/>).
/// </summary>
/// <exception cref="InvalidOperationException">Another index moved the new sequence away meanwhile; its project is not written.</exception>
private async Task CopyAsync(ArchivePaths paths, Sequence source, SequenceManifest manifest, string? json, Dictionary<Guid, StagedConversion> ready, Guid id,
CancellationToken cancellationToken)
{
await _service.EditSequenceInfoAsync(id, info => info with { Tags = source.Tags, Rating = source.Rating, Notes = source.Notes }, cancellationToken)
.ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(json)) return;
await using (await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false))
{
var made = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) ?? throw new InvalidOperationException($"The new sequence {id} is not indexed.");
var folder = paths.Sequence(made.Folder);
// Another index of the archive moved it away, and maybe another sequence into its folder (this index's next scan finds
// both): its project goes nowhere, as every writer into a sequence's folder decides.
if (!SequenceFolders.Holds(folder, id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
var copy = Read(folder, id) ?? throw new InvalidDataException($"The sidecar of the new sequence {made.DisplayTitle} cannot be read.");
// The source's frames as the copies the conversion made of them; a frame not converted keeps its own photo, which the
// new sequence lacks (its positions wait as pending frames).
var from = new FramePhotos(manifest.Frames.Select(f => (f.I, ready.TryGetValue(f.Id, out var c)
? new FramePhoto(c.Sha256.ToLowerInvariant(), c.Size, f.CapturedAt)
: new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
var to = new FramePhotos(copy.Frames.Select(f => (f.I, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
try
{
var translated = ProjectSettings.TranslateFramePositions(json, from, to) ?? json;
var project = ProjectStore.Parse(translated);
await new ProjectStore(_db, _writer).UpdateLockedAsync(paths, made, _ => project, cancellationToken).ConfigureAwait(false);
}
catch (InvalidDataException ex)
{
Core.Diagnostics.Log.Warning("archive", $"{source.DisplayTitle}: its project cannot be read; the new sequence {made.DisplayTitle} starts without it", ex);
}
}
}
/// <summary>The SHA-256 of the frames the sequence of <paramref name="key"/> holds; empty when there is none yet.</summary>
private async Task<IReadOnlySet<string>> FiledAsync(Archive archive, string key, CancellationToken cancellationToken)
{
var shas = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, cancellationToken).ConfigureAwait(false) is not { } id) return shas;
for (var after = -1; ;)
{
var page = await _db.Frames.ListBySequenceAsync(id, after, 1000, cancellationToken).ConfigureAwait(false);
foreach (var frame in page.Where(f => f.Sha256 is not null)) shas.Add(frame.Sha256!);
if (page.Count < 1000) return shas;
after = page[^1].Index;
}
}
/// <summary>
/// The photos converted, when this job replaced the sequence's frames already (a crash after its stage B, before the
/// journal said so): every photo it pinned is a frame holding a file of the target's format converted from the content
/// pinned. Without a pin (its journal lost), every photo confirmed (<paramref name="photos"/>) is a frame holding a
/// conversion to the target whose sources have the step's digest; without those either: the frames holding a conversion
/// to the target are, by their sources, the photos of the step's digest, or (a job an earlier build queued) every staged
/// conversion is a frame holding the converted content. Null otherwise.
/// </summary>
private static int? ReplacedEarlier(SequenceManifest manifest, ConvertSequence step, IReadOnlyList<Guid>? photos, PinnedSequence? pin,
IReadOnlyList<StagedConversion> noted, ConvertOptions options)
{
var frames = manifest.Frames.ToDictionary(f => f.Id);
if (pin is not null)
{
return pin.Frames.Count > 0 && pin.Frames.All(p => frames.TryGetValue(p.Id, out var f) && f.Format == options.Format
&& string.Equals(f.Source?.Sha256, p.Sha256, StringComparison.OrdinalIgnoreCase)
&& !string.Equals(f.Sha256, p.Sha256, StringComparison.OrdinalIgnoreCase))
? pin.Frames.Count
: null;
}
if (step is { Frames: { } confirmed } && photos is { Count: > 0 })
{
var sources = new List<(Guid, string?)>(photos.Count);
foreach (var id in photos)
{
if (!frames.TryGetValue(id, out var f) || f.Format != options.Format || f.Source?.Sha256 is not { } source) return null;
sources.Add((id, source));
}
return string.Equals(ConvertSequencePlan.DigestOf(sources), confirmed, StringComparison.Ordinal) ? photos.Count : null;
}
if (step.Frames is { } digest)
{
var converted = manifest.Frames.Where(f => f.Format == options.Format && f.Source?.Sha256 is not null).ToList();
return converted.Count > 0 && string.Equals(ConvertSequencePlan.DigestOf(converted.Select(f => (f.Id, f.Source!.Sha256))), digest, StringComparison.Ordinal)
? converted.Count
: null;
}
if (noted.Count == 0) return null;
return noted.All(c => frames.TryGetValue(c.FrameId, out var f) && string.Equals(f.Sha256, c.Sha256, StringComparison.OrdinalIgnoreCase)
&& string.Equals(f.Source?.Sha256, c.Source, StringComparison.OrdinalIgnoreCase))
? noted.Select(c => c.FrameId).Distinct().Count()
: null;
}
/// <summary>
/// The row of a sequence this job replaced the frames of, the index brought in line with its sidecar when a crash left it
/// behind (after the sidecar was written, before the index followed); a failure is logged, and the next scan does it. The
/// caller holds the sequence's lease.
/// </summary>
private async Task<Sequence> SyncedAsync(Archive archive, SequencePaths folder, Sequence row, SequenceManifest manifest)
{
if (row.FramesVersion == manifest.Versions.Frames) return row;
try
{
// The photos moved with the replacement: the next incremental scan looks at them again.
await _service.Indexer.SyncAsync(archive, folder, manifest, null, Uncovered, photosChanged: true, CancellationToken.None).ConfigureAwait(false);
return await _db.Sequences.GetAsync(row.Id, CancellationToken.None).ConfigureAwait(false) ?? row;
}
catch (Exception ex) when (ex is DatabaseException or IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the index did not follow its converted frames; the next scan resyncs it", ex);
return row;
}
}
/// <summary>
/// The row of a sequence still to convert, its sidecar settled first under its lease when a writer a crash interrupted left
/// a journal in its folder (<see cref="SequenceWriter.ReadLocked"/>): planned as its files lie half-way, photos in the
/// trash would count as missing. The row as it was when there is none, or when the settle cannot run now (the sequence's
/// turn reports it).
/// </summary>
private async Task<Sequence> SettledRowAsync(Archive archive, ArchivePaths paths, Sequence row, CancellationToken cancellationToken)
{
if (!HasJournal(paths.Sequence(row.Folder))) return row;
try
{
await using (await _writer.LockAsync(paths, row.Id, cancellationToken).ConfigureAwait(false))
{
if (await _db.Sequences.GetAsync(row.Id, cancellationToken).ConfigureAwait(false) is not { } now || now.ArchiveId != archive.Id) return row;
SequenceWriter.ReadLocked(paths.Sequence(now.Folder), now.Id);
return now;
}
}
catch (ArchiveOfflineException)
{
throw;
}
catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
{
Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: what a crash left half-way in it could not be settled yet", ex);
return row;
}
}
/// <summary>Whether a writer left a journal in the sequence's folder: a rename of its files, a deletion, a replacement, a restore or an ingest.</summary>
private static bool HasJournal(SequencePaths folder) =>
File.Exists(folder.FileRenameJournalPath) || File.Exists(folder.RemovalJournalPath) || File.Exists(folder.ReplaceJournalPath)
|| File.Exists(folder.RestoreJournalPath) || File.Exists(folder.IngestJournalPath);
/// <summary>The sidecar of a sequence, read without its lease; null when it cannot be read or names another sequence.</summary>
private static SequenceManifest? Read(SequencePaths folder, Guid id)
{
try
{
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
return manifest.Id == id ? manifest : null;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
{
Core.Diagnostics.Log.Warning("archive", $"The sidecar of {folder.RelativeFolder} cannot be read", ex);
return null;
}
}
/// <summary>
/// The passes a sequence this job converted or made in an earlier attempt needs, from its journal (a pause, a yield, a stop
/// or a crash came before they were queued); null for any other outcome.
/// </summary>
private async Task<IReadOnlyList<ArchiveJobRequest>?> FollowUpsAsync(Archive archive, ConvertJournalSequence noted, CancellationToken cancellationToken)
{
if (noted.Outcome == ConvertOutcome.Converted)
{
return await _db.Sequences.GetAsync(noted.Id, cancellationToken).ConfigureAwait(false) is { } row && row.ArchiveId == archive.Id
? _frames.FollowUps(archive, row, noted.Had ?? row.Status)
: [];
}
if (noted is { Outcome: ConvertOutcome.Created, NewSequenceId: { } id })
{
return await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } made && made.ArchiveId == archive.Id
? _frames.FollowUps(archive, made, noted.Had ?? 0)
: [];
}
return null;
}
/// <summary>
/// The bytes the photos <paramref name="plan"/> converts are estimated at, but for those whose staged file is there already,
/// or which the new sequence this job made holds (<paramref name="filed"/>, the SHA-256 of its photos).
/// </summary>
private static long Unstaged(ArchivePaths paths, Guid jobId, ConvertSequencePlan plan, StagedLog staged, IReadOnlySet<string>? filed)
{
var folder = StagingOf(paths, jobId, plan.SequenceId);
var there = staged.Of(plan.SequenceId, plan.FramesVersion)
.Where(c => filed?.Contains(c.Sha256) == true || Present(Path.Combine(folder, c.File), c))
.Select(c => c.FrameId).ToHashSet();
return plan.Frames.Where(f => !there.Contains(f.FrameId)).Sum(f => f.EstimatedBytes);
}
/// <summary>
/// Refuses to go on while the volume lacks the space the photos still to convert need: their estimates, <paramref name="estimated"/>
/// bytes, as the files staged so far scale them.
/// </summary>
/// <exception cref="IOException">The volume lacks it (<c>Archive.Error.ConvertSpace</c>).</exception>
private static void EnsureSpace(long estimated, SpaceWatch space)
{
if (estimated > 0 && space.Short(estimated) is { } message) throw new IOException(message);
}
/// <summary>
/// Free bytes where a conversion writes, as the host counts them (<see cref="ArchiveServiceOptions.FreeSpace"/>): the volume of
/// the archive root and the one of its staging folder, the tighter; null when neither can be measured.
/// </summary>
private long? FreeSpace(ArchivePaths paths)
{
long? free = null;
foreach (var folder in (string[])[paths.Root, paths.StagingDirectory])
{
if (!System.IO.Directory.Exists(folder) || Measure(folder) is not { } bytes) continue;
free = free is { } known ? Math.Min(known, bytes) : bytes;
}
return free;
}
private long? Measure(string folder)
{
if (_writer.Options.FreeSpace is { } measure) return measure(folder);
try
{
// Windows wants a drive root (a network share is measured by the host); Unix accepts any directory and reports the
// volume it is mounted on.
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(folder)! : folder).AvailableFreeSpace;
}
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
{
return null;
}
}
private static void Log(ConvertCallbacks callbacks, ConvertSequenceResult result)
{
var line = result.Outcome switch
{
ConvertOutcome.Converted => Loc.L("Archive.Log.Converted", result.Title, result.Frames),
ConvertOutcome.Created => Loc.L("Archive.Log.ConvertedNew", result.Title, result.Frames),
ConvertOutcome.Nothing => Loc.L("Archive.Log.ConvertNothing", result.Title),
ConvertOutcome.Superseded => Loc.L("Archive.Log.ConvertSuperseded", result.Title),
ConvertOutcome.Changed => Loc.L("Archive.Log.ConvertChanged", result.Title),
// Busy, or why the retry may do better (a photo another program holds, a folder another index moved), in one sentence.
ConvertOutcome.Blocked when result.Error is null => Loc.L("Archive.Log.ConvertBlocked", result.Title),
_ => Loc.L("Archive.Log.ConvertFailed", result.Title, result.Error),
};
callbacks.Log(line);
// A new sequence holds only the photos converted: those the plan skipped stay in the source alone.
if (result is { Outcome: ConvertOutcome.Created, LeftOut: > 0 }) callbacks.Log(Loc.L("Archive.Log.ConvertLeftOut", result.Title, result.LeftOut));
}
/// <summary>
/// Two sizes as the space error gives them: GB from one gigabyte, MB below, with more decimals when one would read as the
/// other ("needs 0.5 GB, 0.5 GB free").
/// </summary>
private static (string Required, string Free) Sizes(long required, long free)
{
var (a, b) = (Size(required, "0.#"), Size(free, "0.#"));
return a != b ? (a, b) : (Size(required, "0.###"), Size(free, "0.###"));
}
private static string Size(long bytes, string digits) => bytes >= 1_000_000_000
? Loc.L("Common.Size.GB", (bytes / 1e9).ToString(digits, Loc.Culture))
: Loc.L("Common.Size.MB", (bytes / 1e6).ToString(digits, Loc.Culture));
/// <summary>The session key of the new sequence a conversion makes from <paramref name="source"/>.</summary>
internal static string SessionKey(Guid job, Guid source) => $"{KeyPrefix}{job:N}:{source:N}";
/// <summary>The conversion job a session key of <see cref="SessionKey"/> names; null for any other key.</summary>
private static Guid? JobOfKey(string key) =>
key.Split(':') is ["convert", var job, _] && Guid.TryParseExact(job, "N", out var id) ? id : null;
/// <summary>The folder under <c>.chronos/staging</c> a conversion job stages into, one subfolder per sequence.</summary>
internal static string StagingFolder(Guid job) => ConvertJournal.Prefix + job.ToString("N");
private static string StagingOf(ArchivePaths paths, Guid job, Guid sequence) =>
Path.Combine(paths.StagingDirectory, StagingFolder(job), sequence.ToString("N"));
/// <summary>Deletes what a conversion staged that no step needs any more (converted copies, never originals); a failure is logged.</summary>
internal static void DeleteStaging(string folder)
{
try
{
if (System.IO.Directory.Exists(folder)) System.IO.Directory.Delete(folder, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("archive", $"{folder} could not be deleted; maintenance removes it later", ex);
}
}
/// <summary>Photos converted so far and in all, counted per sequence.</summary>
private sealed class Progress
{
private readonly Dictionary<Guid, (int Frames, int Staged)> _counted = [];
public long Total { get; private set; }
public long Done { get; set; }
/// <summary>Counts the photos a sequence converts and those staged already, in place of what was counted for it before.</summary>
public void Count(Guid sequence, int frames, int staged)
{
if (_counted.TryGetValue(sequence, out var counted))
{
Total -= counted.Frames;
Done -= counted.Staged;
}
_counted[sequence] = (frames, staged);
Total += frames;
Done += staged;
}
}
/// <summary>
/// The free space a running conversion watches: measured where it writes (<paramref name="free"/>), against estimates scaled
/// by how far the files it staged came out above theirs (photos of a 14-bit RAW carry more noise than the estimate's bits
/// per pixel allow for), and, in stage A, with the photos started whose converted file is not there yet (<see cref="Start"/>):
/// a converted file is encoded in memory and lands whole, by a rename, before its photo's work verifies it, so the free
/// space measured holds every file landed, and none of what the others will write.
/// </summary>
internal sealed class SpaceWatch(Func<long?> free)
{
private readonly ConcurrentDictionary<string, Starting> _started = new(StringComparer.Ordinal);
private long _actual;
private long _estimated;
/// <summary>The bytes staged over their estimates, at least 1.</summary>
public double Ratio => _estimated > 0 ? Math.Max(1, (double)_actual / _estimated) : 1;
/// <summary>
/// The estimated bytes of the photos started (<see cref="Start"/>) whose converted file has not landed yet: the file is
/// not at its path, or is still the one that was there when the photo started (one a crash left, which it replaces).
/// </summary>
public long Writing => _started.Sum(s => Landed(s.Key) == s.Value.Before ? s.Value.Estimate : 0);
public void Staged(long actual, long estimated)
{
_actual += actual;
_estimated += Math.Max(1, estimated);
}
/// <summary>
/// Why a photo estimated at <paramref name="estimated"/> bytes, converted into <paramref name="target"/>, cannot start now:
/// the volume lacks the space for it and for the photos started before it whose file has not landed yet
/// (<see cref="Writing"/>, <see cref="Short"/>), localized. Null when it can: it counts among those until its file lands
/// or its work ends (<see cref="Ended"/>).
/// </summary>
public string? Start(long estimated, string target)
{
var before = Landed(target);
if (Short(Math.Max(0, estimated) + Writing) is { } message) return message;
_started[target] = new Starting(Math.Max(0, estimated), before);
return null;
}
/// <summary>The work of the photo converted into <paramref name="target"/> ended: its file landed, or never will.</summary>
public void Ended(string target) => _started.TryRemove(target, out _);
/// <summary>
/// Why the volume cannot take <paramref name="estimated"/> bytes more (scaled, with the margin and the reserve), localized;
/// null when it can, or when its free space cannot be measured.
/// </summary>
public string? Short(long estimated)
{
var space = new ConvertSpace(ConvertPlanner.Required((long)Math.Ceiling(estimated * Ratio)), free());
if (space.Enough) return null;
var (required, left) = Sizes(space.Required, space.Free ?? 0);
return Loc.L("Archive.Error.ConvertSpace", required, left);
}
/// <summary>The file at <paramref name="path"/> as a landing changes it, its size and time of writing; null when there is none.</summary>
private static (long Size, long Written)? Landed(string path) =>
new FileInfo(path) is { Exists: true } file ? (file.Length, file.LastWriteTimeUtc.Ticks) : null;
/// <summary>A photo started: its estimate, and the file at its path then.</summary>
private sealed record Starting(long Estimate, (long Size, long Written)? Before);
}
/// <summary>What a sequence came to: its result, the statuses its follow-ups follow from, and those follow-ups (null: none to queue).</summary>
private sealed record StepDone(ConvertSequenceResult Result, SequenceStatus? Had = null, IReadOnlyList<ArchiveJobRequest>? FollowUps = null);
/// <summary>A photo of stage A: its file and its staged conversion, or why it failed.</summary>
private sealed record Staged(string File, StagedConversion? Conversion, Exception? Error);
/// <summary>
/// The staged conversion of a photo a new sequence is about to file is gone (a clean-up, a quarantine): the sequence fails
/// rather than leave the photo out, and the retry converts it again. Its message is the job log's catalog sentence.
/// </summary>
private sealed class UnstagedException(string file, string path) : FileNotFoundException(Loc.L("Archive.Convert.Unstaged", file), path);
}
/// <summary>One photo converted and staged: its frame, the content it was converted from, the staged file's name, size and SHA-256, and its size in pixels.</summary>
/// <param name="Source">SHA-256 of the original it was converted from.</param>
/// <param name="File">The staged file's name inside the sequence's staging folder.</param>
internal sealed record StagedConversion(Guid FrameId, string Source, string File, long Size, string Sha256, int Width, int Height);
/// <summary>A line of the staged log: a staged conversion of a sequence at a frames version.</summary>
internal sealed record StagedLine(Guid Sequence, int FramesVersion, StagedConversion Conversion);
/// <summary>The photos of a sequence a conversion job pinned: the frames version then, and each photo with the content converted.</summary>
internal sealed record PinnedSequence(Guid Sequence, int FramesVersion, IReadOnlyList<PinnedFrame> Frames);
/// <summary>A photo pinned: its frame and the SHA-256 of the content converted.</summary>
internal sealed record PinnedFrame(Guid Id, string Sha256);
/// <summary>
/// The journal of a conversion job (<c>.chronos/journal/convert-&lt;job&gt;.json</c>, owner review 2, R7): what each sequence
/// came to, the new sequence a New sequence conversion made of it as soon as its photos are filed, and whether the passes
/// it needs were queued, written after each one; with it, the photos confirmed (<c>convert-&lt;job&gt;.photos</c>,
/// <see cref="ConfirmedPhotos"/>, written when the job is queued), the pin log (<c>convert-&lt;job&gt;.pins</c>,
/// <see cref="PinLog"/>), the photos the job converts of each sequence, and the staged log (<c>convert-&lt;job&gt;.staged</c>),
/// one line per photo staged, both appended and flushed as they grow, so a job of thousands of photos journals each once. A
/// resumed or retried job keeps the sequences that are final, the copies made, the photos pinned, and the staged files
/// whose hash still verifies. All of it goes when every sequence is final, or once the job is gone (<see cref="ArchiveMaintenance"/>).
/// </summary>
internal sealed class ConvertJournal
{
/// <summary>How the journal files and the staging folder of a conversion job begin, before the job's id.</summary>
public const string Prefix = "convert-";
public Guid JobId { get; set; }
public List<ConvertJournalSequence> Sequences { get; set; } = [];
public ConvertJournalSequence? Of(Guid id) => Sequences.FirstOrDefault(s => s.Id == id);
/// <summary>Notes a sequence the job comes to and pins, with the statuses it had then.</summary>
public void Begin(Guid id, string title, SequenceStatus had)
{
var noted = Of(id);
if (noted is null) Sequences.Add(noted = new ConvertJournalSequence { Id = id });
noted.Title = title;
noted.Had = had;
}
/// <summary>
/// Notes what a sequence came to and, when it was converted or made, the statuses its follow-ups follow from. The new
/// sequence a New sequence conversion made of it stays named whatever it came to since (<see cref="Made"/>).
/// </summary>
public void Note(ConvertSequenceResult result, SequenceStatus? had = null)
{
var noted = Of(result.SequenceId);
if (noted is null) Sequences.Add(noted = new ConvertJournalSequence { Id = result.SequenceId });
noted.Title = result.Title;
noted.Outcome = result.Outcome;
noted.Frames = result.Frames;
noted.NewSequenceId = result.NewSequenceId ?? noted.NewSequenceId;
if (had is { } status) noted.Had = status;
}
/// <summary>
/// Notes the new sequence a New sequence conversion filed the photos of <paramref name="id"/> into, before anything else is
/// done with it: a retry or a resume finishes that one and makes no other.
/// </summary>
public void Made(Guid id, Guid newSequenceId)
{
var noted = Of(id);
if (noted is null) Sequences.Add(noted = new ConvertJournalSequence { Id = id });
noted.NewSequenceId = newSequenceId;
}
public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.json");
public static string StagedPathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.staged");
public static string PinsPathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{Prefix}{job:N}.pins");
/// <summary>
/// The conversion job a journal file or a staging folder belongs to, from its name (<c>convert-&lt;job&gt;</c> and any
/// extension); null for any other name.
/// </summary>
public static Guid? JobOf(string name)
{
if (!name.StartsWith(Prefix, StringComparison.Ordinal)) return null;
var rest = name[Prefix.Length..];
var dot = rest.IndexOf('.', StringComparison.Ordinal);
return Guid.TryParseExact(dot < 0 ? rest : rest[..dot], "N", out var job) ? job : null;
}
/// <summary>The journal of <paramref name="job"/>; null when there is none, or it is torn.</summary>
public static ConvertJournal? Read(ArchivePaths paths, Guid job)
{
var path = PathFor(paths, job);
if (!File.Exists(path)) return null;
try
{
return JsonSerializer.Deserialize(AtomicFile.ReadAllBytes(path), ArchiveJobJson.Default.ConvertJournal);
}
catch (JsonException ex)
{
Core.Diagnostics.Log.Warning("archive", $"The conversion journal {path} is torn; the job starts over", ex);
return null;
}
}
public void Write(ArchivePaths paths)
{
System.IO.Directory.CreateDirectory(paths.JournalDirectory);
AtomicFile.WriteAllBytes(PathFor(paths, JobId), JsonSerializer.SerializeToUtf8Bytes(this, ArchiveJobJson.Default.ConvertJournal));
}
/// <summary>The staged log of this job, read now.</summary>
public StagedLog Staged(ArchivePaths paths) => StagedLog.Open(StagedPathFor(paths, JobId));
/// <summary>The pin log of this job, read now.</summary>
public PinLog Pins(ArchivePaths paths) => PinLog.Open(PinsPathFor(paths, JobId));
/// <summary>The journal files of <paramref name="job"/>, whichever exist: the photos confirmed among them (<see cref="ConfirmedPhotos"/>).</summary>
public static IEnumerable<string> Files(ArchivePaths paths, Guid job) =>
[StagedPathFor(paths, job), PinsPathFor(paths, job), ConfirmedPhotos.PathFor(paths, job), PathFor(paths, job)];
public static void Delete(ArchivePaths paths, Guid job)
{
foreach (var path in Files(paths, job))
{
if (File.Exists(path)) File.Delete(path);
}
}
}
/// <summary>What the journal knows of one sequence of a conversion.</summary>
internal sealed class ConvertJournalSequence
{
public Guid Id { get; set; }
public string? Title { get; set; }
public ConvertOutcome? Outcome { get; set; }
public int Frames { get; set; }
public Guid? NewSequenceId { get; set; }
/// <summary>
/// The statuses the passes it needs follow from: the sequence's before its frames were replaced, or the source's a new
/// sequence takes on (a deep analysis, thumbnails, proxies).
/// </summary>
public SequenceStatus? Had { get; set; }
/// <summary>The passes it needs were queued (a resumed job queues them when they were not).</summary>
public bool Queued { get; set; }
[JsonIgnore]
public bool Final => ConvertSequenceResult.IsFinal(Outcome);
}
/// <summary>The staged log of a conversion job (<see cref="ConvertJournal"/>): read once, appended to line by line.</summary>
internal sealed class StagedLog
{
private readonly string _path;
private readonly Dictionary<(Guid, int), Dictionary<Guid, StagedConversion>> _lines = [];
private StagedLog(string path) => _path = path;
/// <summary>The log at <paramref name="path"/>; a line torn by a crash, or unreadable, is skipped.</summary>
public static StagedLog Open(string path)
{
var log = new StagedLog(path);
foreach (var line in JournalLines.Read(path))
{
try
{
if (JsonSerializer.Deserialize(line, ArchiveJobJson.Default.StagedLine) is { } staged) log.Add(staged);
}
catch (JsonException)
{
// Torn while appended: that photo is converted again.
}
}
return log;
}
/// <summary>The staged conversions of a sequence at a frames version, latest per frame.</summary>
public IReadOnlyList<StagedConversion> Of(Guid sequence, int framesVersion) =>
_lines.TryGetValue((sequence, framesVersion), out var frames) ? [.. frames.Values] : [];
/// <summary>Appends a staged conversion and flushes it to disk.</summary>
public void Append(Guid sequence, int framesVersion, StagedConversion conversion)
{
var line = new StagedLine(sequence, framesVersion, conversion);
JournalLines.Append(_path, JsonSerializer.SerializeToUtf8Bytes(line, ArchiveJobJson.Default.StagedLine));
Add(line);
}
private void Add(StagedLine line)
{
if (!_lines.TryGetValue((line.Sequence, line.FramesVersion), out var frames)) _lines[(line.Sequence, line.FramesVersion)] = frames = [];
frames[line.Conversion.FrameId] = line.Conversion;
}
}
/// <summary>
/// The pin log of a conversion job (<see cref="ConvertJournal"/>): the photos it converts of each sequence, pinned under the
/// sequence's lease the first time the job comes to it, while its plan is still the one confirmed. A retry converts those,
/// never photos that came in or were verified since. One line per sequence, appended and flushed.
/// </summary>
internal sealed class PinLog
{
private readonly string _path;
private readonly Dictionary<Guid, PinnedSequence> _pins = [];
private PinLog(string path) => _path = path;
/// <summary>The log at <paramref name="path"/>; a line torn by a crash is skipped (that sequence is held to the plan confirmed again).</summary>
public static PinLog Open(string path)
{
var log = new PinLog(path);
foreach (var line in JournalLines.Read(path))
{
try
{
if (JsonSerializer.Deserialize(line, ArchiveJobJson.Default.PinnedSequence) is { } pinned) log._pins[pinned.Sequence] = pinned;
}
catch (JsonException)
{
// Torn while appended.
}
}
return log;
}
public PinnedSequence? Of(Guid sequence) => _pins.GetValueOrDefault(sequence);
/// <summary>Pins the photos <paramref name="planned"/> converts, and flushes it to disk.</summary>
public void Append(Guid sequence, ConvertSequencePlan planned)
{
var pinned = new PinnedSequence(sequence, planned.FramesVersion, [.. planned.Frames.Select(f => new PinnedFrame(f.FrameId, f.Sha256 ?? string.Empty))]);
JournalLines.Append(_path, JsonSerializer.SerializeToUtf8Bytes(pinned, ArchiveJobJson.Default.PinnedSequence));
_pins[sequence] = pinned;
}
}
/// <summary>A line of <see cref="ConfirmedPhotos"/>: a sequence, and the frame ids of the photos converted of it, 16 bytes each, in base64.</summary>
internal sealed record ConfirmedLine(Guid Sequence, string Photos);
/// <summary>
/// The photos a conversion job converts of each sequence, as its plan was confirmed (<c>.chronos/journal/convert-&lt;job&gt;.photos</c>,
/// owner review 2, R7): written whole under the job's id before it is queued (<see cref="MassConverter.QueueAsync"/>), one
/// line per sequence with the frame ids in the plan's order. The job's row keeps each sequence's digest of those photos
/// with their content (<see cref="ConvertSequence.Frames"/>), never the ids: it stays as small as the number of sequences
/// whatever the number of photos, for every list and check that reads jobs. Under each sequence's lease the job takes these
/// photos from its plan made again and goes on only while they have that digest; a sequence this file does not name (it
/// could not be written, or a job an earlier build queued) is held to its whole plan. It goes with the journal
/// (<see cref="ConvertJournal.Files"/>).
/// </summary>
internal sealed class ConfirmedPhotos
{
private readonly Dictionary<Guid, IReadOnlyList<Guid>> _photos = [];
public static string PathFor(ArchivePaths paths, Guid job) => Path.Combine(paths.JournalDirectory, $"{ConvertJournal.Prefix}{job:N}.photos");
/// <summary>The photos confirmed of <paramref name="sequence"/>, in the plan's order; null when the file does not name it.</summary>
public IReadOnlyList<Guid>? Of(Guid sequence) => _photos.GetValueOrDefault(sequence);
/// <summary>The photos confirmed for <paramref name="job"/>: none when the file is not there; a line that cannot be read names nothing.</summary>
public static ConfirmedPhotos Read(ArchivePaths paths, Guid job)
{
var read = new ConfirmedPhotos();
foreach (var line in JournalLines.Read(PathFor(paths, job)))
{
try
{
if (JsonSerializer.Deserialize(line, ArchiveJobJson.Default.ConfirmedLine) is { } confirmed) read._photos[confirmed.Sequence] = Decode(confirmed.Photos);
}
catch (Exception ex) when (ex is JsonException or FormatException)
{
Core.Diagnostics.Log.Warning("archive", $"A line of the photos confirmed for conversion job {job} cannot be read; that sequence is held to its whole plan", ex);
}
}
return read;
}
/// <summary>
/// Writes the photos <paramref name="plan"/> converts of each sequence it converts, for the job <paramref name="job"/>, whole
/// (a reader finds all of it or none); false, logged, when it cannot be written.
/// </summary>
public static bool TryWrite(ArchivePaths paths, Guid job, ConvertPlan plan)
{
ArgumentNullException.ThrowIfNull(plan);
try
{
using var lines = new MemoryStream();
foreach (var sequence in plan.Converted)
{
lines.Write(JsonSerializer.SerializeToUtf8Bytes(new ConfirmedLine(sequence.SequenceId, Encode(sequence.Frames.Select(f => f.FrameId))),
ArchiveJobJson.Default.ConfirmedLine));
lines.WriteByte((byte)'\n');
}
System.IO.Directory.CreateDirectory(paths.JournalDirectory);
AtomicFile.WriteAllBytes(PathFor(paths, job), lines.GetBuffer().AsSpan(0, (int)lines.Length));
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be written; it holds each sequence to its whole plan", ex);
return false;
}
}
/// <summary>Deletes the file of <paramref name="job"/>; a failure is logged, and maintenance removes it later.</summary>
public static void Delete(ArchivePaths paths, Guid job)
{
try
{
File.Delete(PathFor(paths, job));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("archive", $"The photos confirmed for conversion job {job} could not be deleted; maintenance removes them later", ex);
}
}
/// <summary>Frame ids as a line holds them: their 16 bytes each, one after the other, in base64.</summary>
public static string Encode(IEnumerable<Guid> ids)
{
ArgumentNullException.ThrowIfNull(ids);
IReadOnlyCollection<Guid> frames = ids as IReadOnlyCollection<Guid> ?? [.. ids];
var bytes = new byte[frames.Count * 16];
var at = 0;
foreach (var frame in frames)
{
frame.TryWriteBytes(bytes.AsSpan(at, 16));
at += 16;
}
return Convert.ToBase64String(bytes);
}
/// <summary>The frame ids <see cref="Encode"/> wrote, in order.</summary>
/// <exception cref="FormatException">The text is not frame ids as <see cref="Encode"/> writes them.</exception>
public static IReadOnlyList<Guid> Decode(string text)
{
ArgumentNullException.ThrowIfNull(text);
var bytes = Convert.FromBase64String(text);
if (bytes.Length % 16 != 0) throw new FormatException("The text is not frame ids.");
var ids = new Guid[bytes.Length / 16];
for (var i = 0; i < ids.Length; i++) ids[i] = new Guid(bytes.AsSpan(i * 16, 16));
return ids;
}
}
/// <summary>Files of JSON lines a job appends to as it goes: each line written whole and flushed to disk, read back skipping blank ones.</summary>
internal static class JournalLines
{
public static IEnumerable<string> Read(string path) =>
File.Exists(path) ? File.ReadLines(path, Encoding.UTF8).Where(l => !string.IsNullOrWhiteSpace(l)) : [];
public static void Append(string path, byte[] json)
{
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(path)!);
using var stream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read);
stream.Write(json);
stream.WriteByte((byte)'\n');
stream.Flush(flushToDisk: true);
}
}