Owner review 2, R7 asks for mass conversion after import on the web panel and the desktop, with a choice at every run between Replace and New sequence. This is the engine and the server API both UIs build on. - Core/Conversion: ConvertOptions (JPEG 4:4:4 at quality 1-100, default 95; TIFF-16 sRGB Deflate with the predictor; linear DNG, the archival choice; converted names keep the stem and take the target's extension in the old one's case). FrameConverter decodes at the native tier only, encodes with the in-house writers, writes Exif from the sidecar (capture time to the tick, offset, exposure) and the original (camera, serial, lens, modes, GPS), hashes the bytes as made, writes atomically and verifies: read back, the metadata again, a decode by the server's decoders. ExifWriter now writes the exposure program, metering mode, flash and white balance. ConvertPlanner: convertible frames, skips with reasons, size estimates per pixel, the plan's space and fingerprint. - FrameTrash.ReplaceAsync swaps the files of frames in place: same ids and indices, the old files and their XMP companions into one Replaced trash entry (records, replacement hashes, a copy of analysis.json), tombstones aliasing old content to new, proxies of the swapped frames dropped, under .chronos/replace.pending.json, which the next writer undoes or completes. Restoring a Replaced entry swaps back while every converted frame is unchanged (the converted files become a Replaced entry in turn), else the originals come back as a new sequence. - MassConverter and JobKind.Convert (19): the dry run (busy, offline, unreadable, space against ArchiveServiceOptions.FreeSpace), stage A converting into staging/convert-<job>/ in parallel with each source re-hashed and each staged file appended to journal/convert-<job>.staged, stage B per sequence all or nothing under the lease at the planned frames version: Replace, or a new sequence through the ingest with the source's tags, rating, notes and project. Resumes keep verified staged files; a sequence changed, busy or with a failed photo is left as it was and fails the job at the end, the journal kept for a retry. - POST /api/v1/archive/convert: dry run with the plan, its space line and fingerprint, or a 202 convert job; formats the server cannot read at full resolution are skipped as convert_on_pc; validated, audited (archive_converted), sequenceChanged converted/created. The server measures the space volume for the check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 809210d924cfc3f597ecdea26106fb3ff62d3762) (cherry picked from commit e13d4c0bb1bd9441933b6f3e5b6220e865a7228c)
287 lines
15 KiB
C#
287 lines
15 KiB
C#
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Codecs.Dng;
|
|
using Chronos.Core.Codecs.Jpeg;
|
|
using Chronos.Core.Codecs.Tiff;
|
|
using Chronos.Core.Conversion;
|
|
using Chronos.Core.Imaging;
|
|
using Chronos.Core.Imaging.Decoders;
|
|
using Chronos.Core.Metadata;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Core.Tests.Conversion;
|
|
|
|
/// <summary>
|
|
/// One photo converted (owner review 2, R7): to JPEG, TIFF-16 and linear DNG, decoded at full resolution only, with the
|
|
/// sidecar's capture facts to the tick and the original's camera, lens, modes and GPS, verified (read back, metadata, the
|
|
/// server's decoders) before it counts.
|
|
/// </summary>
|
|
[Category("conversion")]
|
|
public sealed class FrameConverterTests(TestContext context)
|
|
{
|
|
private static readonly DateTime Captured = new DateTime(2026, 9, 28, 18, 0, 5).AddTicks(1_234_567);
|
|
|
|
/// <summary>The camera's file says one second earlier, without sub-seconds: the sidecar wins.</summary>
|
|
private static readonly FrameMetadata Camera = new()
|
|
{
|
|
Make = "Canon",
|
|
Model = "Canon EOS R5",
|
|
BodySerial = "032021001234",
|
|
LensModel = "RF15-35mm F2.8 L IS USM",
|
|
CapturedAt = new DateTime(2026, 9, 28, 18, 0, 4),
|
|
UtcOffsetMinutes = 60,
|
|
ExposureTime = 1 / 250.0,
|
|
FNumber = 8,
|
|
Iso = 100,
|
|
FocalLength = 15,
|
|
FocalLength35 = 15,
|
|
ExposureBias = -2 / 3.0,
|
|
ExposureProgram = 3,
|
|
MeteringMode = 5,
|
|
Flash = 16,
|
|
WhiteBalanceMode = 1,
|
|
Artist = "Alberto",
|
|
Gps = new GpsPosition(45.4642035, 9.1899817, 122.5, null),
|
|
};
|
|
|
|
private static SidecarFrame Frame(string file, ImageFileFormat format, int width, int height) => new()
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
File = "originals/" + file,
|
|
Original = file,
|
|
Sha256 = new string('a', 64),
|
|
CapturedAt = Captured,
|
|
UtcOffset = 120,
|
|
TimeSource = TimestampSource.ExifOriginal,
|
|
Exposure = 1 / 125.0,
|
|
FNumber = 5.6,
|
|
Iso = 200,
|
|
Focal = 24,
|
|
Bias = -1,
|
|
W = width,
|
|
H = height,
|
|
Format = format,
|
|
Flags = FrameFlags.Intervalometer,
|
|
};
|
|
|
|
/// <summary>A gradient with some texture, so every encoder has real work and errors show.</summary>
|
|
private static ImageBuffer Scene(int width, int height)
|
|
{
|
|
var image = new ImageBuffer(width, height);
|
|
for (var y = 0; y < height; y++)
|
|
{
|
|
for (var x = 0; x < width; x++)
|
|
{
|
|
image[0, x, y] = 0.05f + 0.6f * x / width;
|
|
image[1, x, y] = 0.1f + 0.5f * y / height;
|
|
image[2, x, y] = 0.2f + 0.2f * ((x / 8 + y / 8) % 2);
|
|
}
|
|
}
|
|
return image;
|
|
}
|
|
|
|
private string Source(int width = 96, int height = 64, int orientation = 1)
|
|
{
|
|
var path = Path.Combine(context.CreateTempDirectory("source"), "IMG_0001.JPG");
|
|
using var image = Scene(width, height);
|
|
var exif = ExifWriter.BuildJpegApp1(Camera with { Orientation = orientation });
|
|
File.WriteAllBytes(path, JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 97, Subsampling = JpegSubsampling.Yuv444, App1Segments = [exif[4..]] }));
|
|
return path;
|
|
}
|
|
|
|
private string Target(ConvertOptions options) => Path.Combine(context.CreateTempDirectory("staged"), "converted" + options.Extension);
|
|
|
|
[Theory]
|
|
[Case(ConvertTarget.Jpeg)]
|
|
[Case(ConvertTarget.Tiff16)]
|
|
[Case(ConvertTarget.LinearDng)]
|
|
public void TheSidecarsCaptureFactsAndTheCamerasMetadataRoundTrip(ConvertTarget target)
|
|
{
|
|
var options = new ConvertOptions(target);
|
|
var source = Source();
|
|
var frame = Frame("IMG_0001.JPG", ImageFileFormat.Jpeg, 96, 64);
|
|
var file = FrameConverter.Convert(new ConvertSource(source, frame), options, DecoderFactory.CreateServer(), Target(options), cancellationToken: context.Cancellation);
|
|
|
|
Assert.Equal((options.Format, 96, 64), (file.Format, file.Width, file.Height));
|
|
Assert.Equal(new FileInfo(file.Path).Length, file.Size);
|
|
Assert.Equal(file.Sha256, ContentHash.Sha256Async(file.Path).GetAwaiter().GetResult());
|
|
var read = MetadataReader.Read(file.Path);
|
|
Assert.Equal(options.Format, read.Format);
|
|
// The sidecar wins: the capture time to the tick, its offset, exposure, aperture, ISO, focal length and bias.
|
|
Assert.Equal(Captured, read.CapturedAt);
|
|
Assert.Equal(120, read.UtcOffsetMinutes);
|
|
Assert.Equal(TimestampSource.ExifOriginal, read.TimestampSource);
|
|
Assert.Approximately(1 / 125.0, read.ExposureTime!.Value, 1e-12);
|
|
Assert.Approximately(5.6, read.FNumber!.Value, 1e-9);
|
|
Assert.Equal(200, read.Iso);
|
|
Assert.Approximately(24, read.FocalLength!.Value, 1e-9);
|
|
Assert.Approximately(-1, read.ExposureBias!.Value, 1e-4);
|
|
// The original's own: camera, serial, lens, 35 mm focal length, modes, GPS, artist.
|
|
Assert.Equal(("Canon", "Canon EOS R5", "032021001234", "RF15-35mm F2.8 L IS USM"), (read.Make, read.Model, read.BodySerial, read.LensModel));
|
|
Assert.Approximately(15, read.FocalLength35!.Value, 1e-9);
|
|
Assert.Equal((3, 5, 16, 1), (read.ExposureProgram, read.MeteringMode, read.Flash, read.WhiteBalanceMode));
|
|
var gps = Assert.NotNull(read.Gps);
|
|
Assert.Approximately(45.4642035, gps.Latitude, 1e-6);
|
|
Assert.Approximately(9.1899817, gps.Longitude, 1e-6);
|
|
Assert.Equal(1, read.Orientation);
|
|
Assert.Equal((96, 64), read.OrientedSize);
|
|
}
|
|
|
|
[Theory]
|
|
[Case(ConvertTarget.Jpeg)]
|
|
[Case(ConvertTarget.Tiff16)]
|
|
[Case(ConvertTarget.LinearDng)]
|
|
public void TheConvertedFileDecodesAtFullResolutionWithTheServersDecoders(ConvertTarget target)
|
|
{
|
|
var options = new ConvertOptions(target);
|
|
var source = Source();
|
|
var frame = Frame("IMG_0001.JPG", ImageFileFormat.Jpeg, 96, 64);
|
|
var file = FrameConverter.Convert(new ConvertSource(source, frame), options, DecoderFactory.CreateServer(), Target(options), cancellationToken: context.Cancellation);
|
|
|
|
var server = DecoderFactory.CreateServer();
|
|
Assert.Equal(DecodeTier.Native, server.Probe(file.Path, file.Format).Tier);
|
|
using var original = server.Decode(new DecodeRequest(source, AllowPreview: false))!;
|
|
using var converted = Assert.NotNull(server.Decode(new DecodeRequest(file.Path, AllowPreview: false)));
|
|
Assert.Equal(DecodeTier.Native, converted.Tier);
|
|
Assert.Equal((96, 64), (converted.Pixels.Width, converted.Pixels.Height));
|
|
// The same picture: TIFF-16 and DNG keep it to 16 bits, the JPEG within its quantisation.
|
|
var tolerance = target == ConvertTarget.Jpeg ? 0.03 : 0.01;
|
|
var error = 0.0;
|
|
for (var c = 0; c < 3; c++)
|
|
{
|
|
var a = original.Pixels.Plane(c);
|
|
var b = converted.Pixels.Plane(c);
|
|
for (var i = 0; i < a.Length; i++) error += Math.Abs(a[i] - b[i]);
|
|
}
|
|
Assert.Less(error / (3.0 * 96 * 64), tolerance, $"mean error of {target}");
|
|
}
|
|
|
|
[Test]
|
|
public void TheJpegQualityAndFullChromaAreTheOnesAsked()
|
|
{
|
|
var source = Source(160, 120);
|
|
var frame = Frame("IMG_0001.JPG", ImageFileFormat.Jpeg, 160, 120);
|
|
var decoder = DecoderFactory.CreateServer();
|
|
var high = FrameConverter.Convert(new ConvertSource(source, frame), new ConvertOptions(ConvertTarget.Jpeg, 95), decoder,
|
|
Target(new ConvertOptions(ConvertTarget.Jpeg)), cancellationToken: context.Cancellation);
|
|
var low = FrameConverter.Convert(new ConvertSource(source, frame), new ConvertOptions(ConvertTarget.Jpeg, 20), decoder,
|
|
Target(new ConvertOptions(ConvertTarget.Jpeg)), cancellationToken: context.Cancellation);
|
|
Assert.Less(low.Size, high.Size);
|
|
var info = JpegDecoder.ReadInfo(File.ReadAllBytes(high.Path));
|
|
Assert.Equal("4:4:4", info.SamplingNotation);
|
|
Assert.Equal(ConvertOptions.DefaultQuality, new ConvertOptions(ConvertTarget.Jpeg).Quality);
|
|
}
|
|
|
|
[Test]
|
|
public void TiffAndLinearDngAreSixteenBitsAndReadByTheInHouseReaders()
|
|
{
|
|
var source = Source();
|
|
var frame = Frame("IMG_0001.JPG", ImageFileFormat.Jpeg, 96, 64);
|
|
var decoder = DecoderFactory.CreateServer();
|
|
var tiff = FrameConverter.Convert(new ConvertSource(source, frame), new ConvertOptions(ConvertTarget.Tiff16), decoder,
|
|
Target(new ConvertOptions(ConvertTarget.Tiff16)), cancellationToken: context.Cancellation);
|
|
var dng = FrameConverter.Convert(new ConvertSource(source, frame), new ConvertOptions(ConvertTarget.LinearDng), decoder,
|
|
Target(new ConvertOptions(ConvertTarget.LinearDng)), cancellationToken: context.Cancellation);
|
|
|
|
Assert.Equal(16, MetadataReader.Read(tiff.Path).BitsPerSample);
|
|
using (var back = TiffImageReader.Decode(File.ReadAllBytes(tiff.Path)))
|
|
Assert.Equal((96, 64, 3), (back.Width, back.Height, back.Channels));
|
|
Assert.Equal(ImageFileFormat.Dng, MetadataReader.Read(dng.Path).Format);
|
|
using (var back = TiffImageReader.Decode(File.ReadAllBytes(dng.Path)))
|
|
Assert.Equal((96, 64, 3), (back.Width, back.Height, back.Channels));
|
|
}
|
|
|
|
[Test]
|
|
public void APortraitOriginalIsConvertedUpright()
|
|
{
|
|
var source = Source(96, 64, orientation: 6);
|
|
var frame = Frame("IMG_0001.JPG", ImageFileFormat.Jpeg, 96, 64) with { Orientation = 6 };
|
|
var options = new ConvertOptions(ConvertTarget.Jpeg);
|
|
var file = FrameConverter.Convert(new ConvertSource(source, frame), options, DecoderFactory.CreateServer(), Target(options), cancellationToken: context.Cancellation);
|
|
Assert.Equal((64, 96), (file.Width, file.Height));
|
|
var read = MetadataReader.Read(file.Path);
|
|
Assert.Equal((1, 64, 96), (read.Orientation, read.Width, read.Height));
|
|
}
|
|
|
|
[Test]
|
|
public void ACameraAndLensTheFileLacksComeFromTheSequence()
|
|
{
|
|
var path = Path.Combine(context.CreateTempDirectory("source"), "frame.png");
|
|
using (var image = Scene(32, 24)) File.WriteAllBytes(path, Chronos.Core.Codecs.Png.PngEncoder.Encode(image));
|
|
var frame = Frame("frame.png", ImageFileFormat.Png, 32, 24);
|
|
var options = new ConvertOptions(ConvertTarget.Tiff16);
|
|
var file = FrameConverter.Convert(new ConvertSource(path, frame, new SidecarCamera { Make = "Sony", Model = "ILCE-7RM5", Serial = "77" }, "FE 24mm F1.4 GM"),
|
|
options, DecoderFactory.CreateServer(), Target(options), cancellationToken: context.Cancellation);
|
|
var read = MetadataReader.Read(file.Path);
|
|
Assert.Equal(("Sony", "ILCE-7RM5", "77", "FE 24mm F1.4 GM"), (read.Make, read.Model, read.BodySerial, read.LensModel));
|
|
Assert.Equal(Captured, read.CapturedAt);
|
|
}
|
|
|
|
[Test]
|
|
public void APhotoReadOnlyFromItsPreviewIsNotConverted()
|
|
{
|
|
// A DNG the managed decoder reads natively, behind a chain that offers only a preview.
|
|
var path = Path.Combine(context.CreateTempDirectory("source"), "frame.dng");
|
|
using (var image = Scene(32, 24)) File.WriteAllBytes(path, LinearDngWriter.Write(image));
|
|
var frame = Frame("frame.dng", ImageFileFormat.Dng, 32, 24);
|
|
var options = new ConvertOptions(ConvertTarget.Jpeg);
|
|
var target = Target(options);
|
|
Assert.Throws<ConversionException>(() =>
|
|
FrameConverter.Convert(new ConvertSource(path, frame), options, new ImageDecoder([new PreviewOnlyProvider()]), target, cancellationToken: context.Cancellation));
|
|
Assert.FileMissing(target);
|
|
Assert.Throws<ConversionException>(() =>
|
|
FrameConverter.Convert(new ConvertSource(path, frame), options, new ImageDecoder([]), target, cancellationToken: context.Cancellation));
|
|
}
|
|
|
|
[Test]
|
|
public void AFileThatDoesNotReadBackIsRefused()
|
|
{
|
|
var source = Source();
|
|
var frame = Frame("IMG_0001.JPG", ImageFileFormat.Jpeg, 96, 64);
|
|
var options = new ConvertOptions(ConvertTarget.Jpeg);
|
|
var file = FrameConverter.Convert(new ConvertSource(source, frame), options, DecoderFactory.CreateServer(), Target(options), cancellationToken: context.Cancellation);
|
|
var written = FrameConverter.Metadata(MetadataReader.Read(source), new ConvertSource(source, frame));
|
|
Assert.DoesNotThrow(() => FrameConverter.Verify(file, written));
|
|
// A bit flipped on the way to the disk.
|
|
var bytes = File.ReadAllBytes(file.Path);
|
|
bytes[^10] ^= 0x40;
|
|
File.WriteAllBytes(file.Path, bytes);
|
|
Assert.Throws<ConversionException>(() => FrameConverter.Verify(file, written));
|
|
// A capture time that is not the one written.
|
|
File.WriteAllBytes(file.Path, File.ReadAllBytes(source));
|
|
Assert.Throws<ConversionException>(() => FrameConverter.Verify(file with { Sha256 = ContentHash.Sha256Async(file.Path).GetAwaiter().GetResult(), Size = new FileInfo(file.Path).Length }, written));
|
|
}
|
|
|
|
[Test]
|
|
public void ConvertedNamesKeepTheStemAndTheCaseOfTheExtension()
|
|
{
|
|
Assert.Equal("originals/IMG_0001.DNG", new ConvertOptions(ConvertTarget.LinearDng).NameFor("originals/IMG_0001.CR2"));
|
|
Assert.Equal("originals/img_0001.tif", new ConvertOptions(ConvertTarget.Tiff16).NameFor("originals/img_0001.cr3"));
|
|
Assert.Equal("originals/2026-09-28_0001.jpg", new ConvertOptions(ConvertTarget.Jpeg).NameFor("originals/2026-09-28_0001.Nef"));
|
|
Assert.Equal("frame.jpg", new ConvertOptions(ConvertTarget.Jpeg).NameFor("frame"));
|
|
Assert.Equal(ConvertTarget.LinearDng, ConvertOptions.ArchivalTarget);
|
|
Assert.False(new ConvertOptions(ConvertTarget.Jpeg, 0).IsValid);
|
|
Assert.False(new ConvertOptions(ConvertTarget.Jpeg, 101).IsValid);
|
|
Assert.Equal(new ConvertOptions(ConvertTarget.LinearDng, 10).Key, new ConvertOptions(ConvertTarget.LinearDng, 90).Key);
|
|
}
|
|
|
|
/// <summary>A provider that claims DNG natively but only ever gives the embedded-preview tier.</summary>
|
|
private sealed class PreviewOnlyProvider : IImageDecoderProvider
|
|
{
|
|
private readonly ManagedDecoderProvider _managed = new();
|
|
|
|
public string Name => "preview-only";
|
|
|
|
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Dng ? DecodeTier.Native : null;
|
|
|
|
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default)
|
|
{
|
|
var decoded = _managed.Decode(request with { AllowPreview = true }, cancellationToken);
|
|
if (decoded is null) return null;
|
|
var copy = decoded.Pixels.Clone();
|
|
decoded.Dispose();
|
|
return new DecodedImage(copy, DecodeTier.EmbeddedPreview, Name, copy.Width, copy.Height);
|
|
}
|
|
}
|
|
}
|