Files
Chronos/tests/Chronos.Core.Tests/Conversion/ConvertPlannerTests.cs
T
Alby96 0e52929274 feat(archive): convert sequences to JPEG, TIFF-16 or linear DNG after import
Owner review 2, R7 asks for mass conversion after import on the web panel and the desktop,
with a choice at every run between Replace and New sequence. This is the engine and the
server API both UIs build on.

- Core/Conversion: ConvertOptions (JPEG 4:4:4 at quality 1-100, default 95; TIFF-16 sRGB
  Deflate with the predictor; linear DNG, the archival choice; converted names keep the
  stem and take the target's extension in the old one's case). FrameConverter decodes at
  the native tier only, encodes with the in-house writers, writes Exif from the sidecar
  (capture time to the tick, offset, exposure) and the original (camera, serial, lens,
  modes, GPS), hashes the bytes as made, writes atomically and verifies: read back, the
  metadata again, a decode by the server's decoders. ExifWriter now writes the exposure
  program, metering mode, flash and white balance. ConvertPlanner: convertible frames,
  skips with reasons, size estimates per pixel, the plan's space and fingerprint.
- FrameTrash.ReplaceAsync swaps the files of frames in place: same ids and indices, the old
  files and their XMP companions into one Replaced trash entry (records, replacement
  hashes, a copy of analysis.json), tombstones aliasing old content to new, proxies of the
  swapped frames dropped, under .chronos/replace.pending.json, which the next writer undoes
  or completes. Restoring a Replaced entry swaps back while every converted frame is
  unchanged (the converted files become a Replaced entry in turn), else the originals come
  back as a new sequence.
- MassConverter and JobKind.Convert (19): the dry run (busy, offline, unreadable, space
  against ArchiveServiceOptions.FreeSpace), stage A converting into
  staging/convert-<job>/ in parallel with each source re-hashed and each staged file
  appended to journal/convert-<job>.staged, stage B per sequence all or nothing under the
  lease at the planned frames version: Replace, or a new sequence through the ingest with
  the source's tags, rating, notes and project. Resumes keep verified staged files; a
  sequence changed, busy or with a failed photo is left as it was and fails the job at the
  end, the journal kept for a retry.
- POST /api/v1/archive/convert: dry run with the plan, its space line and fingerprint, or a
  202 convert job; formats the server cannot read at full resolution are skipped as
  convert_on_pc; validated, audited (archive_converted), sequenceChanged converted/created.
  The server measures the space volume for the check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 809210d924cfc3f597ecdea26106fb3ff62d3762)
(cherry picked from commit e13d4c0bb1bd9441933b6f3e5b6220e865a7228c)
2026-10-27 22:01:23 +01:00

127 lines
7.3 KiB
C#

using Chronos.Core.Archiving;
using Chronos.Core.Conversion;
using Chronos.Core.Export;
using Chronos.Core.Imaging;
using Chronos.Core.Model;
using Chronos.Testing;
namespace Chronos.Core.Tests.Conversion;
/// <summary>The dry run of a conversion (owner review 2, R7): which frames convert, which stay and why, the sizes, the space and the fingerprint.</summary>
[Category("conversion")]
public sealed class ConvertPlannerTests(TestContext context)
{
private static SidecarFrame Frame(int i, string file, ImageFileFormat format, string? sha = "s", FrameFlags flags = FrameFlags.None,
DecodeTier tier = default) => new()
{
Id = Guid.NewGuid(),
I = i,
File = "originals/" + file,
Size = 1000 + i,
Sha256 = sha is null ? null : sha + i,
Format = format,
W = 6000,
H = 4000,
Flags = flags,
Tier = tier,
};
[Test]
public void EachFrameThatCannotConvertSaysWhy()
{
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("2026/roof");
SidecarFrame[] frames =
[
Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2),
Frame(1, "IMG_0002.DNG", ImageFileFormat.Dng),
Frame(2, "IMG_0003.CR2", ImageFileFormat.Cr2),
Frame(3, "IMG_0004.CR2", ImageFileFormat.Cr2, sha: null),
Frame(4, "IMG_0005.CR2", ImageFileFormat.Cr2, flags: FrameFlags.Undecodable),
Frame(5, "IMG_0006.CR3", ImageFileFormat.Cr3),
Frame(6, "IMG_0007.HEIC", ImageFileFormat.Heif),
Frame(7, "IMG_0008.JPG", ImageFileFormat.Jpeg, flags: FrameFlags.Missing),
Frame(8, "IMG_0009.CR2", ImageFileFormat.Cr2, flags: FrameFlags.PreviewOnly | FrameFlags.Excluded, tier: DecodeTier.EmbeddedPreview),
];
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames, Versions = new ManifestVersions { Frames = 7 } };
// The third photo's file is gone; the preview-only one probes as preview here.
var present = frames.Where(f => f.I != 2).Select(f => f.File).ToList();
var decoder = new ImageDecoder([new ProbeAs(DecodeTier.EmbeddedPreview)]);
var plan = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.LinearDng), decoder, "Roof");
Assert.Equal((manifest.Id, "Roof", "2026/roof", 7), (plan.SequenceId, plan.Title, plan.Folder, plan.FramesVersion));
Assert.SequenceEqual([0], plan.Frames.Select(f => f.Index));
Assert.SequenceEqual(
[(ConvertSkip.SameFormat, 1), (ConvertSkip.NoNativeDecoder, 2), (ConvertSkip.PreviewOnly, 1), (ConvertSkip.Undecodable, 1), (ConvertSkip.Missing, 2),
(ConvertSkip.Unhashed, 1)],
plan.Skipped.Select(s => (s.Reason, s.Frames)));
Assert.True(plan.Converts);
Assert.Equal(1000, plan.BytesBefore);
Assert.Equal(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 6000, 4000), plan.BytesAfter);
Assert.Equal(8, plan.SkippedFrames);
}
[Test]
public void APhotoRecordedAsPreviewOnlyElsewhereConvertsWhereItDecodes()
{
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
var frame = Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2, flags: FrameFlags.PreviewOnly, tier: DecodeTier.EmbeddedPreview);
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = [frame] };
var plan = ConvertPlanner.Plan(manifest, folder, [frame.File], new ConvertOptions(ConvertTarget.Tiff16), new ImageDecoder([new ProbeAs(DecodeTier.Native)]), "Roof");
Assert.Count(1, plan.Frames);
Assert.Empty(plan.Skipped);
}
[Test]
public void SizesAreEstimatedFromThePixels()
{
const long pixels = 6000L * 4000;
Assert.Equal((long)Math.Ceiling(pixels * 30 / 8.0), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 6000, 4000));
Assert.Equal((long)Math.Ceiling(pixels * 26 / 8.0), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 6000, 4000));
Assert.Equal((long)Math.Ceiling(pixels * ExportEstimator.JpegBitsPerPixel(95) * 1.25 / 8),
ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg), 6000, 4000));
Assert.Less(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg, 50), 6000, 4000),
ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg, 95), 6000, 4000));
// Unknown size: a 24 MP frame.
Assert.Equal(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 6000, 4000), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 0, 0));
Assert.Equal((long)Math.Ceiling(1_000_000_000 * 1.15) + (512L << 20), ConvertPlanner.Required(1_000_000_000));
Assert.True(new ConvertSpace(100, null).Enough);
Assert.True(new ConvertSpace(100, 100).Enough);
Assert.False(new ConvertSpace(100, 99).Enough);
}
[Test]
public void TheFingerprintFollowsWhatThePlanConverts()
{
var id = Guid.NewGuid();
ConvertSequencePlan Sequence(int version, int frames, params ConvertBlock[] blocked) =>
new(id, "Roof", "roof", version, [.. Enumerable.Range(0, frames).Select(i => new ConvertFramePlan(Guid.Empty, i, $"f{i}", ImageFileFormat.Cr2, 10, 20))],
[], blocked);
ConvertPlan Plan(ConvertOptions options, ConvertMode mode, ConvertSequencePlan sequence) => new(Guid.Empty, options, mode, [sequence], new ConvertSpace(0, null));
var dng = new ConvertOptions(ConvertTarget.LinearDng);
var reference = Plan(dng, ConvertMode.Replace, Sequence(3, 2)).Fingerprint();
Assert.Equal(64, reference.Length);
Assert.Equal(reference, Plan(dng with { Quality = 40 }, ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
Assert.NotEqual(reference, Plan(dng, ConvertMode.NewSequence, Sequence(3, 2)).Fingerprint());
Assert.NotEqual(reference, Plan(new ConvertOptions(ConvertTarget.Jpeg), ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
Assert.NotEqual(reference, Plan(dng, ConvertMode.Replace, Sequence(4, 2)).Fingerprint());
Assert.NotEqual(reference, Plan(dng, ConvertMode.Replace, Sequence(3, 1)).Fingerprint());
var busy = Plan(dng, ConvertMode.Replace, Sequence(3, 2, ConvertBlock.Busy));
Assert.NotEqual(reference, busy.Fingerprint());
Assert.Equal((0, 0L), (busy.Frames, busy.BytesAfter));
Assert.NotEqual(Plan(new ConvertOptions(ConvertTarget.Jpeg, 90), ConvertMode.Replace, Sequence(3, 2)).Fingerprint(),
Plan(new ConvertOptions(ConvertTarget.Jpeg, 80), ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
}
/// <summary>A provider that reads CR2 natively by format, and probes every file at a fixed tier.</summary>
private sealed class ProbeAs(DecodeTier tier) : IImageDecoderProvider
{
public string Name => "probe";
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Cr2 ? DecodeTier.Native : null;
public FileTier Probe(string path, ImageFileFormat format) => new(format, tier, Name, null, 0, 0);
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
}
}