Owner review 2, R7 asks for mass conversion after import on the web panel and the desktop, with a choice at every run between Replace and New sequence. This is the engine and the server API both UIs build on. - Core/Conversion: ConvertOptions (JPEG 4:4:4 at quality 1-100, default 95; TIFF-16 sRGB Deflate with the predictor; linear DNG, the archival choice; converted names keep the stem and take the target's extension in the old one's case). FrameConverter decodes at the native tier only, encodes with the in-house writers, writes Exif from the sidecar (capture time to the tick, offset, exposure) and the original (camera, serial, lens, modes, GPS), hashes the bytes as made, writes atomically and verifies: read back, the metadata again, a decode by the server's decoders. ExifWriter now writes the exposure program, metering mode, flash and white balance. ConvertPlanner: convertible frames, skips with reasons, size estimates per pixel, the plan's space and fingerprint. - FrameTrash.ReplaceAsync swaps the files of frames in place: same ids and indices, the old files and their XMP companions into one Replaced trash entry (records, replacement hashes, a copy of analysis.json), tombstones aliasing old content to new, proxies of the swapped frames dropped, under .chronos/replace.pending.json, which the next writer undoes or completes. Restoring a Replaced entry swaps back while every converted frame is unchanged (the converted files become a Replaced entry in turn), else the originals come back as a new sequence. - MassConverter and JobKind.Convert (19): the dry run (busy, offline, unreadable, space against ArchiveServiceOptions.FreeSpace), stage A converting into staging/convert-<job>/ in parallel with each source re-hashed and each staged file appended to journal/convert-<job>.staged, stage B per sequence all or nothing under the lease at the planned frames version: Replace, or a new sequence through the ingest with the source's tags, rating, notes and project. Resumes keep verified staged files; a sequence changed, busy or with a failed photo is left as it was and fails the job at the end, the journal kept for a retry. - POST /api/v1/archive/convert: dry run with the plan, its space line and fingerprint, or a 202 convert job; formats the server cannot read at full resolution are skipped as convert_on_pc; validated, audited (archive_converted), sequenceChanged converted/created. The server measures the space volume for the check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 809210d924cfc3f597ecdea26106fb3ff62d3762) (cherry picked from commit e13d4c0bb1bd9441933b6f3e5b6220e865a7228c)
127 lines
7.3 KiB
C#
127 lines
7.3 KiB
C#
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Conversion;
|
|
using Chronos.Core.Export;
|
|
using Chronos.Core.Imaging;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Core.Tests.Conversion;
|
|
|
|
/// <summary>The dry run of a conversion (owner review 2, R7): which frames convert, which stay and why, the sizes, the space and the fingerprint.</summary>
|
|
[Category("conversion")]
|
|
public sealed class ConvertPlannerTests(TestContext context)
|
|
{
|
|
private static SidecarFrame Frame(int i, string file, ImageFileFormat format, string? sha = "s", FrameFlags flags = FrameFlags.None,
|
|
DecodeTier tier = default) => new()
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
I = i,
|
|
File = "originals/" + file,
|
|
Size = 1000 + i,
|
|
Sha256 = sha is null ? null : sha + i,
|
|
Format = format,
|
|
W = 6000,
|
|
H = 4000,
|
|
Flags = flags,
|
|
Tier = tier,
|
|
};
|
|
|
|
[Test]
|
|
public void EachFrameThatCannotConvertSaysWhy()
|
|
{
|
|
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("2026/roof");
|
|
SidecarFrame[] frames =
|
|
[
|
|
Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2),
|
|
Frame(1, "IMG_0002.DNG", ImageFileFormat.Dng),
|
|
Frame(2, "IMG_0003.CR2", ImageFileFormat.Cr2),
|
|
Frame(3, "IMG_0004.CR2", ImageFileFormat.Cr2, sha: null),
|
|
Frame(4, "IMG_0005.CR2", ImageFileFormat.Cr2, flags: FrameFlags.Undecodable),
|
|
Frame(5, "IMG_0006.CR3", ImageFileFormat.Cr3),
|
|
Frame(6, "IMG_0007.HEIC", ImageFileFormat.Heif),
|
|
Frame(7, "IMG_0008.JPG", ImageFileFormat.Jpeg, flags: FrameFlags.Missing),
|
|
Frame(8, "IMG_0009.CR2", ImageFileFormat.Cr2, flags: FrameFlags.PreviewOnly | FrameFlags.Excluded, tier: DecodeTier.EmbeddedPreview),
|
|
];
|
|
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames, Versions = new ManifestVersions { Frames = 7 } };
|
|
// The third photo's file is gone; the preview-only one probes as preview here.
|
|
var present = frames.Where(f => f.I != 2).Select(f => f.File).ToList();
|
|
var decoder = new ImageDecoder([new ProbeAs(DecodeTier.EmbeddedPreview)]);
|
|
var plan = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.LinearDng), decoder, "Roof");
|
|
|
|
Assert.Equal((manifest.Id, "Roof", "2026/roof", 7), (plan.SequenceId, plan.Title, plan.Folder, plan.FramesVersion));
|
|
Assert.SequenceEqual([0], plan.Frames.Select(f => f.Index));
|
|
Assert.SequenceEqual(
|
|
[(ConvertSkip.SameFormat, 1), (ConvertSkip.NoNativeDecoder, 2), (ConvertSkip.PreviewOnly, 1), (ConvertSkip.Undecodable, 1), (ConvertSkip.Missing, 2),
|
|
(ConvertSkip.Unhashed, 1)],
|
|
plan.Skipped.Select(s => (s.Reason, s.Frames)));
|
|
Assert.True(plan.Converts);
|
|
Assert.Equal(1000, plan.BytesBefore);
|
|
Assert.Equal(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 6000, 4000), plan.BytesAfter);
|
|
Assert.Equal(8, plan.SkippedFrames);
|
|
}
|
|
|
|
[Test]
|
|
public void APhotoRecordedAsPreviewOnlyElsewhereConvertsWhereItDecodes()
|
|
{
|
|
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
|
|
var frame = Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2, flags: FrameFlags.PreviewOnly, tier: DecodeTier.EmbeddedPreview);
|
|
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = [frame] };
|
|
var plan = ConvertPlanner.Plan(manifest, folder, [frame.File], new ConvertOptions(ConvertTarget.Tiff16), new ImageDecoder([new ProbeAs(DecodeTier.Native)]), "Roof");
|
|
Assert.Count(1, plan.Frames);
|
|
Assert.Empty(plan.Skipped);
|
|
}
|
|
|
|
[Test]
|
|
public void SizesAreEstimatedFromThePixels()
|
|
{
|
|
const long pixels = 6000L * 4000;
|
|
Assert.Equal((long)Math.Ceiling(pixels * 30 / 8.0), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 6000, 4000));
|
|
Assert.Equal((long)Math.Ceiling(pixels * 26 / 8.0), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 6000, 4000));
|
|
Assert.Equal((long)Math.Ceiling(pixels * ExportEstimator.JpegBitsPerPixel(95) * 1.25 / 8),
|
|
ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg), 6000, 4000));
|
|
Assert.Less(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg, 50), 6000, 4000),
|
|
ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg, 95), 6000, 4000));
|
|
// Unknown size: a 24 MP frame.
|
|
Assert.Equal(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 6000, 4000), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 0, 0));
|
|
Assert.Equal((long)Math.Ceiling(1_000_000_000 * 1.15) + (512L << 20), ConvertPlanner.Required(1_000_000_000));
|
|
Assert.True(new ConvertSpace(100, null).Enough);
|
|
Assert.True(new ConvertSpace(100, 100).Enough);
|
|
Assert.False(new ConvertSpace(100, 99).Enough);
|
|
}
|
|
|
|
[Test]
|
|
public void TheFingerprintFollowsWhatThePlanConverts()
|
|
{
|
|
var id = Guid.NewGuid();
|
|
ConvertSequencePlan Sequence(int version, int frames, params ConvertBlock[] blocked) =>
|
|
new(id, "Roof", "roof", version, [.. Enumerable.Range(0, frames).Select(i => new ConvertFramePlan(Guid.Empty, i, $"f{i}", ImageFileFormat.Cr2, 10, 20))],
|
|
[], blocked);
|
|
ConvertPlan Plan(ConvertOptions options, ConvertMode mode, ConvertSequencePlan sequence) => new(Guid.Empty, options, mode, [sequence], new ConvertSpace(0, null));
|
|
var dng = new ConvertOptions(ConvertTarget.LinearDng);
|
|
var reference = Plan(dng, ConvertMode.Replace, Sequence(3, 2)).Fingerprint();
|
|
Assert.Equal(64, reference.Length);
|
|
Assert.Equal(reference, Plan(dng with { Quality = 40 }, ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
|
|
Assert.NotEqual(reference, Plan(dng, ConvertMode.NewSequence, Sequence(3, 2)).Fingerprint());
|
|
Assert.NotEqual(reference, Plan(new ConvertOptions(ConvertTarget.Jpeg), ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
|
|
Assert.NotEqual(reference, Plan(dng, ConvertMode.Replace, Sequence(4, 2)).Fingerprint());
|
|
Assert.NotEqual(reference, Plan(dng, ConvertMode.Replace, Sequence(3, 1)).Fingerprint());
|
|
var busy = Plan(dng, ConvertMode.Replace, Sequence(3, 2, ConvertBlock.Busy));
|
|
Assert.NotEqual(reference, busy.Fingerprint());
|
|
Assert.Equal((0, 0L), (busy.Frames, busy.BytesAfter));
|
|
Assert.NotEqual(Plan(new ConvertOptions(ConvertTarget.Jpeg, 90), ConvertMode.Replace, Sequence(3, 2)).Fingerprint(),
|
|
Plan(new ConvertOptions(ConvertTarget.Jpeg, 80), ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
|
|
}
|
|
|
|
/// <summary>A provider that reads CR2 natively by format, and probes every file at a fixed tier.</summary>
|
|
private sealed class ProbeAs(DecodeTier tier) : IImageDecoderProvider
|
|
{
|
|
public string Name => "probe";
|
|
|
|
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Cr2 ? DecodeTier.Native : null;
|
|
|
|
public FileTier Probe(string path, ImageFileFormat format) => new(format, tier, Name, null, 0, 0);
|
|
|
|
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
|
|
}
|
|
}
|