Files
Chronos/tests/Chronos.Core.Tests/Conversion/ConvertPlannerTests.cs
T
Alby96andClaude Opus 5.5 b471a33bfc fix(convert): a conversion never changes what Chronos reads from a photo
Owner review 2, phase 4b, fix round 3 (codecs).

Chronos's decoders never read an ICC profile: they take code values through
the sRGB curve and float samples as they are. Round 2 wrote a float TIFF's
values with its profile's own curves, so after a Replace the renders,
thumbnails, analyses and deflicker of an Adobe RGB, ProPhoto or gamma-1.0
float TIFF saw other pixels (up to 2.7 EV darker, shadows lifted). Every
converted file now holds the values the decoders read from the original, as
they read them back, and says truthfully what they are:

- A JPEG or TIFF-16 keeps the codes of code values and carries the original's
  profile. Float values are written as sRGB codes with a profile made for
  them: the original's colorants and media white with the sRGB curve. A gray
  profile's codes are kept, its curve carried as an RGB profile (they were
  converted to sRGB before, which changed them too).
- A Linear DNG, whose matrices Chronos develops it with as well, takes only
  values that are sRGB's as the decoders read them; any other profile is
  refused as color_profile instead of converted to colours Chronos would then
  show differently.
- A profile with colorant tags that convert nothing is never sRGB by its name.
- A float file with NaN or infinite samples fails as damaged data instead of
  being written black (and spending a DNG's headroom).

The plan and the decoders behind it:

- TooLarge and WorstBytes answer any size, int.MaxValue sides included,
  without a product that wraps around; the worst case saturates.
- A frame another host flagged unreadable (the server, for a CMYK or CIELab
  TIFF its reader refuses) is asked again with a small decode where the
  conversion has other decoders (WIC on a PC), so a PC still converts it.
  The too_large sentences say "or Chronos reads back" (en, it).
- WIC's TIFF decoder decodes an unknown compression as zeros, does not undo
  the floating-point predictor of Photoshop's 32-bit files and reads 32-bit
  integers as floats, all without an error: such TIFFs are left to the
  managed reader, on the desktop as in conversions, never black or noise.
- A conversion's WIC probe reads the headers (decoder, first frame's oriented
  size, pixel format), so the size bound applies to what only WIC reads; a
  read error stays a read error (inaccessible), never a verdict.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 09:36:00 +01:00

516 lines
33 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Chronos.Core.Archiving;
using Chronos.Core.Codecs.Icc;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Codecs.Raw;
using Chronos.Core.Codecs.Tiff;
using Chronos.Core.Conversion;
using Chronos.Core.Export;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Metadata.Tiff;
using Chronos.Core.Model;
using Chronos.Core.Tests.Raw;
using Chronos.Testing;
namespace Chronos.Core.Tests.Conversion;
/// <summary>The dry run of a conversion (owner review 2, R7): which frames convert, which stay and why, the sizes, the space and the fingerprint.</summary>
[Category("conversion")]
public sealed class ConvertPlannerTests(TestContext context)
{
private static SidecarFrame Frame(int i, string file, ImageFileFormat format, string? sha = "s", FrameFlags flags = FrameFlags.None,
DecodeTier tier = default) => new()
{
Id = Guid.NewGuid(),
I = i,
File = "originals/" + file,
Size = 1000 + i,
Sha256 = sha is null ? null : sha + i,
Format = format,
W = 6000,
H = 4000,
Flags = flags,
Tier = tier,
};
[Test]
public void EachFrameThatCannotConvertSaysWhy()
{
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("2026/roof");
SidecarFrame[] frames =
[
Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2),
Frame(1, "IMG_0002.DNG", ImageFileFormat.Dng),
Frame(2, "IMG_0003.CR2", ImageFileFormat.Cr2),
Frame(3, "IMG_0004.CR2", ImageFileFormat.Cr2, sha: null),
Frame(4, "IMG_0005.CR2", ImageFileFormat.Cr2, flags: FrameFlags.Undecodable),
Frame(5, "IMG_0006.CR3", ImageFileFormat.Cr3),
Frame(6, "IMG_0007.HEIC", ImageFileFormat.Heif),
Frame(7, "IMG_0008.JPG", ImageFileFormat.Jpeg, flags: FrameFlags.Missing),
Frame(8, "IMG_0009.CR2", ImageFileFormat.Cr2, flags: FrameFlags.PreviewOnly | FrameFlags.Excluded, tier: DecodeTier.EmbeddedPreview),
];
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames, Versions = new ManifestVersions { Frames = 7 } };
// The third photo's file is gone; the preview-only one probes as preview here, the first one native.
var present = frames.Where(f => f.I != 2).Select(f => f.File).ToList();
var decoder = new ImageDecoder([new ProbeAs(DecodeTier.EmbeddedPreview, path => path.EndsWith("IMG_0009.CR2", StringComparison.Ordinal))]);
var plan = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.LinearDng), decoder, "Roof");
Assert.Equal((manifest.Id, "Roof", "2026/roof", 7), (plan.SequenceId, plan.Title, plan.Folder, plan.FramesVersion));
Assert.SequenceEqual([0], plan.Frames.Select(f => f.Index));
Assert.Equal("s0", plan.Frames.Single().Sha256);
Assert.SequenceEqual(
[(ConvertSkip.SameFormat, 1), (ConvertSkip.NoNativeDecoder, 2), (ConvertSkip.PreviewOnly, 1), (ConvertSkip.Undecodable, 1), (ConvertSkip.Missing, 2),
(ConvertSkip.Unhashed, 1)],
plan.Skipped.Select(s => (s.Reason, s.Frames)));
Assert.True(plan.Converts);
Assert.Equal(1000, plan.BytesBefore);
Assert.Equal(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 6000, 4000), plan.BytesAfter);
Assert.Equal(8, plan.SkippedFrames);
}
[Test]
public void APhotoRecordedAsPreviewOnlyElsewhereConvertsWhereItDecodes()
{
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
var frame = Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2, flags: FrameFlags.PreviewOnly, tier: DecodeTier.EmbeddedPreview);
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = [frame] };
var plan = ConvertPlanner.Plan(manifest, folder, [frame.File], new ConvertOptions(ConvertTarget.Tiff16), new ImageDecoder([new ProbeAs(DecodeTier.Native)]), "Roof");
Assert.Count(1, plan.Frames);
Assert.Empty(plan.Skipped);
}
/// <summary>
/// A frame whose tier is not known yet (imported, not analysed) or was recorded native by another host with other
/// decoders is asked again: the dry run promises only what the file says here now. A file the decoders here do not read
/// at all, although they read its format, is left to a PC; only a frame flagged unreadable is unreadable.
/// </summary>
[Test]
public void AFrameIsAskedWhateverTierWasRecorded()
{
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
SidecarFrame[] frames = [Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2), Frame(1, "IMG_0002.CR2", ImageFileFormat.Cr2, tier: DecodeTier.Native)];
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
var present = frames.Select(f => f.File).ToList();
var dng = new ConvertOptions(ConvertTarget.LinearDng);
var preview = ConvertPlanner.Plan(manifest, folder, present, dng, new ImageDecoder([new ProbeAs(DecodeTier.EmbeddedPreview)]), "Roof");
Assert.Empty(preview.Frames);
Assert.SequenceEqual([(ConvertSkip.PreviewOnly, 2)], preview.Skipped.Select(s => (s.Reason, s.Frames)));
var unreadable = ConvertPlanner.Plan(manifest, folder, present, dng, new ImageDecoder([new ProbeAs(DecodeTier.Unavailable)]), "Roof");
Assert.SequenceEqual([(ConvertSkip.NoNativeDecoder, 2)], unreadable.Skipped.Select(s => (s.Reason, s.Frames)));
var native = ConvertPlanner.Plan(manifest, folder, present, dng, new ImageDecoder([new ProbeAs(DecodeTier.Native)]), "Roof");
Assert.SequenceEqual([0, 1], native.Frames.Select(f => f.Index));
}
/// <summary>
/// A file the plan cannot read is not called missing unless it is gone: one another program holds, or that access is
/// denied to, could not be opened now (a plan made once it is free converts it), and one shorter than its own headers
/// cannot be decoded.
/// </summary>
[Test]
public void AFileThatCannotBeReadNowSaysWhy()
{
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
Exception[] failures =
[
new FileNotFoundException("gone"), new DirectoryNotFoundException("gone"), new IOException("held by another process"),
new UnauthorizedAccessException("denied"), new EndOfStreamException("short"),
];
SidecarFrame[] frames = [.. failures.Select((_, i) => Frame(i, $"IMG_{i:0000}.CR2", ImageFileFormat.Cr2))];
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
// IMG_0000 fails as the first, IMG_0004 as the last.
var decoder = new ImageDecoder([new ProbeFails(path => failures[Path.GetFileNameWithoutExtension(path)[^1] - '0'])]);
var plan = ConvertPlanner.Plan(manifest, folder, [.. frames.Select(f => f.File)], new ConvertOptions(ConvertTarget.LinearDng), decoder, "Roof");
Assert.Empty(plan.Frames);
Assert.SequenceEqual([(ConvertSkip.Undecodable, 1), (ConvertSkip.Missing, 2), (ConvertSkip.Inaccessible, 2)], plan.Skipped.Select(s => (s.Reason, s.Frames)));
}
/// <summary>A RAW decoder whose probe of a file fails as <paramref name="failure"/> says.</summary>
private sealed class ProbeFails(Func<string, Exception> failure) : IImageDecoderProvider
{
public string Name => "fails";
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Cr2 ? DecodeTier.Native : null;
public FileTier Probe(string path, ImageFileFormat format) => throw failure(path);
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
}
/// <summary>
/// The managed decoder develops every RAW it reads natively (at float precision) even where WIC could; a RAW only WIC's
/// Raw Image Extension develops, at 8 bits, converts to JPEG but not to TIFF-16 or linear DNG.
/// </summary>
[Test]
public void ARawDevelopedAtEightBitsOnlyConvertsToJpegOnly()
{
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(folder.File("originals/x"))!);
using var scene = SyntheticRaw.Smooth(96, 64);
var active = new RawRect(24, 4, 24 + 96, 4 + 64);
var sensor = NativeRawWriters.Sensor(scene, "Canon EOS 5D Mark III", CfaPattern.Bayer("RGGB"), 128, 72, active.Left, active.Top, 2048, 16383);
SidecarFrame[] frames =
[
Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2, tier: DecodeTier.Native),
Frame(1, "IMG_0002.CR2", ImageFileFormat.Cr2, tier: DecodeTier.Native),
Frame(2, "IMG_0003.CR3", ImageFileFormat.Cr3, tier: DecodeTier.Native),
];
File.WriteAllBytes(folder.File(frames[0].File), NativeRawWriters.Cr2(sensor, active));
// A body the managed decoder has no matrix for: only the 8-bit development reads it.
File.WriteAllBytes(folder.File(frames[1].File), NativeRawWriters.Cr2(sensor, active, model: "Canon EOS R99"));
File.WriteAllBytes(folder.File(frames[2].File), [0]);
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
var present = frames.Select(f => f.File).ToList();
var desktop = new ImageDecoder([new EightBitRaw(), new ManagedDecoderProvider()]);
foreach (var target in (ConvertTarget[])[ConvertTarget.Tiff16, ConvertTarget.LinearDng])
{
var plan = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(target), desktop, "Roof");
Assert.SequenceEqual([0], plan.Frames.Select(f => f.Index));
Assert.SequenceEqual([(ConvertSkip.EightBit, 2)], plan.Skipped.Select(s => (s.Reason, s.Frames)));
}
var jpeg = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.Jpeg), desktop, "Roof");
Assert.SequenceEqual([0, 1, 2], jpeg.Frames.Select(f => f.Index));
// The server, without WIC: the unsupported body and the CR3 are to convert on a PC.
var server = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), "Roof");
Assert.SequenceEqual([0], server.Frames.Select(f => f.Index));
Assert.SequenceEqual([(ConvertSkip.NoNativeDecoder, 1), (ConvertSkip.PreviewOnly, 1)], server.Skipped.Select(s => (s.Reason, s.Frames)));
}
/// <summary>A photo whose colour space no target can keep, or which a target cannot hold, is planned out with why.</summary>
[Test]
public void ColourAndSizeTheTargetCannotHoldArePlannedOut()
{
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(folder.File("originals/x"))!);
using var scene = ColorFixtures.Scene();
SidecarFrame[] frames =
[
Frame(0, "plain.jpg", ImageFileFormat.Jpeg),
Frame(1, "p3.jpg", ImageFileFormat.Jpeg),
Frame(2, "unknown.jpg", ImageFileFormat.Jpeg),
Frame(3, "cmyk.jpg", ImageFileFormat.Jpeg),
Frame(4, "panorama.png", ImageFileFormat.Png) with { W = 70_000, H = 2_500 },
Frame(5, "huge.png", ImageFileFormat.Png) with { W = 70_000, H = 12_000 },
];
File.WriteAllBytes(folder.File(frames[0].File), ColorFixtures.Jpeg(scene));
File.WriteAllBytes(folder.File(frames[1].File), ColorFixtures.Jpeg(scene, IccProfile.DisplayP3.Data));
File.WriteAllBytes(folder.File(frames[2].File), ColorFixtures.Jpeg(scene, exifTiff: ColorFixtures.Exif(0xFFFF)));
File.WriteAllBytes(folder.File(frames[3].File), ColorFixtures.Jpeg(scene, ColorFixtures.Cmyk()));
// A panorama's header (the dry run reads no further), and a file the sidecar's size already rules out.
File.WriteAllBytes(folder.File(frames[4].File), ColorFixtures.PngHeader(70_000, 2_500));
File.WriteAllBytes(folder.File(frames[5].File), [0]);
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
var present = frames.Select(f => f.File).ToList();
var decoder = DecoderFactory.CreateServer();
// A TIFF carries the P3 photo's profile; a DNG, which holds sRGB as Chronos reads it, does not take it. Neither takes
// the unknown and CMYK ones, nor 2.5 G samples.
var tiff = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.Tiff16), decoder, "Roof");
Assert.SequenceEqual([0, 1, 4], tiff.Frames.Select(f => f.Index));
Assert.SequenceEqual([(ConvertSkip.ColorProfile, 2), (ConvertSkip.TooLarge, 1)], tiff.Skipped.Select(s => (s.Reason, s.Frames)));
var dng = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.LinearDng), decoder, "Roof");
Assert.SequenceEqual([0, 4], dng.Frames.Select(f => f.Index));
Assert.SequenceEqual([(ConvertSkip.ColorProfile, 3), (ConvertSkip.TooLarge, 1)], dng.Skipped.Select(s => (s.Reason, s.Frames)));
// A JPEG side holds 65535 pixels.
var jpeg = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.Jpeg), decoder, "Roof");
Assert.SequenceEqual([(ConvertSkip.SameFormat, 4), (ConvertSkip.TooLarge, 2)], jpeg.Skipped.Select(s => (s.Reason, s.Frames)));
Assert.True(ConvertPlanner.TooLarge(new ConvertOptions(ConvertTarget.Jpeg), 65_536, 10));
Assert.False(ConvertPlanner.TooLarge(new ConvertOptions(ConvertTarget.Jpeg), 65_535, 10));
Assert.False(ConvertPlanner.TooLarge(new ConvertOptions(ConvertTarget.Tiff16), 0, 0));
}
/// <summary>
/// The size bound is what the server's decoders read back, which every converted file must (owner review 2, R7), never
/// an estimate: a JPEG of 2^28 pixels, a TIFF-16 or DNG (whose raster the same TIFF reader reads) of 2^29 samples, whose
/// headers are read while one row more is refused. A file that size stays within the array it is made in (2 GB) even
/// when its pixels do not compress at all (16-bit noise takes 48 bits a pixel, up to 51 as lossless JPEG codes them),
/// and noise encodes within that bound in every target.
/// </summary>
[Test]
public void TheSizeBoundIsWhatTheServersDecodersReadBack()
{
var jpeg = new ConvertOptions(ConvertTarget.Jpeg, 100);
Assert.False(ConvertPlanner.TooLarge(jpeg, 16_384, 16_384));
Assert.True(ConvertPlanner.TooLarge(jpeg, 16_384, 16_385));
Assert.False(ConvertPlanner.TooLarge(jpeg, 4_096, 65_535));
Assert.True(ConvertPlanner.TooLarge(jpeg, 4_097, 65_535));
var largest = JpegDecoder.ReadInfo(ColorFixtures.JpegHeader(16_384, 16_384));
Assert.Equal((16_384, 16_384), (largest.Width, largest.Height));
Assert.Throws<InvalidDataException>(() => JpegDecoder.ReadInfo(ColorFixtures.JpegHeader(16_384, 16_385)));
foreach (var target in (ConvertTarget[])[ConvertTarget.Tiff16, ConvertTarget.LinearDng])
{
Assert.False(ConvertPlanner.TooLarge(new ConvertOptions(target), 13_377, 13_377), $"{target} of 2^29 samples at most");
Assert.True(ConvertPlanner.TooLarge(new ConvertOptions(target), 13_377, 13_378), $"{target} of one row more");
}
foreach (ushort photometric in (ushort[])[2, 34892])
{
Assert.Equal((13_377, 13_377), TiffImageReader.ReadInfo(new MemoryStream(ColorFixtures.TiffHeader(13_377, 13_377, photometric))));
Assert.Throws<InvalidDataException>(() => TiffImageReader.ReadInfo(new MemoryStream(ColorFixtures.TiffHeader(13_377, 13_378, photometric))));
}
Assert.LessOrEqual(ConvertPlanner.WorstBytes(16_384, 16_384), Array.MaxLength);
Assert.LessOrEqual(ConvertPlanner.WorstBytes(13_377, 13_377), Array.MaxLength);
// 342 MP: its estimate (30 or 26 bits a pixel) would fit, the worst case would not, and nothing reads it back.
foreach (var target in Enum.GetValues<ConvertTarget>())
{
Assert.True(ConvertPlanner.TooLarge(new ConvertOptions(target, 100), 18_500, 18_500), $"{target} at 342 MP");
Assert.Less(ConvertPlanner.EstimateBytes(new ConvertOptions(target, 100), 18_500, 18_500), Array.MaxLength);
}
Assert.Greater(ConvertPlanner.WorstBytes(18_500, 18_500), Array.MaxLength);
using var noise = new ImageBuffer(512, 512);
var random = new Random(7);
foreach (ref var v in noise.Data) v = random.NextSingle();
var metadata = new Chronos.Core.Metadata.FrameMetadata { CapturedAt = new DateTime(2026, 10, 27, 12, 0, 0) };
foreach (var target in Enum.GetValues<ConvertTarget>())
{
var bytes = FrameConverter.Encode(noise, metadata, new ConvertOptions(target, 100));
var bits = bytes.Length * 8.0 / noise.PlaneLength;
context.Log($"{target}: {bits:0.00} bits a pixel of noise");
Assert.LessOrEqual(bits, ConvertPlanner.WorstBitsPerPixel, $"{target} encodes noise within the bound, its headers included");
Assert.LessOrEqual(bytes.Length, ConvertPlanner.WorstBytes(512, 512));
}
}
/// <summary>
/// Any size is answered, a damaged header's included (owner review 2, R7): sides up to <see cref="int.MaxValue"/> are too
/// large for every target, never let through by a product that wrapped around, and the worst case saturates instead of
/// turning negative.
/// </summary>
[Test]
public void AnySizeIsAnsweredWithoutOverflow()
{
(int W, int H)[] absurd =
[
(int.MaxValue, int.MaxValue), (2_000_000_000, 2_000_000_000), (1_800_000_000, 1_800_000_000), (1_000_000_000, 1_000_000_000),
(int.MaxValue, 1), (1, int.MaxValue),
];
foreach (var target in Enum.GetValues<ConvertTarget>())
{
foreach (var (w, h) in absurd) Assert.True(ConvertPlanner.TooLarge(new ConvertOptions(target), w, h), $"{target} at {w}×{h}");
Assert.Greater(ConvertPlanner.EstimateBytes(new ConvertOptions(target), int.MaxValue, int.MaxValue), 0L);
}
Assert.Equal(long.MaxValue, ConvertPlanner.WorstBytes(int.MaxValue, int.MaxValue));
Assert.Greater(ConvertPlanner.WorstBytes(1_800_000_000, 1_800_000_000), Array.MaxLength);
Assert.Greater(ConvertPlanner.WorstBytes(65_535, 65_535), 0L);
Assert.False(ConvertPlanner.TooLarge(new ConvertOptions(ConvertTarget.Tiff16), -5, 10));
}
/// <summary>
/// A frame flagged unreadable by the host that analysed it (the server, whose decoders refuse a CMYK TIFF) is asked again
/// where the conversion's decoders have others for its format (WIC on a PC), with a small decode (owner review 2, R7):
/// converted where they read it, unreadable where they do not. A corrupt frame stays unreadable, and so does a flagged one
/// where only the managed decoders read the format. A file that cannot be read while it is asked is inaccessible, nothing
/// concluded of its content.
/// </summary>
[Test]
public void AFrameFlaggedUnreadableElsewhereIsAskedWhereOtherDecodersReadIt()
{
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(folder.File("originals/x"))!);
SidecarFrame[] frames =
[
Frame(0, "cmyk.tif", ImageFileFormat.Tiff, flags: FrameFlags.Undecodable) with { W = 6, H = 4 },
Frame(1, "corrupt.tif", ImageFileFormat.Tiff, flags: FrameFlags.Corrupt) with { W = 6, H = 4 },
];
File.WriteAllBytes(folder.File(frames[0].File), ColorFixtures.Tiff(6, 4, 5, [0, 255, 255, 0]));
File.WriteAllBytes(folder.File(frames[1].File), ColorFixtures.Tiff(6, 4, 2, [1, 2, 3]));
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
var present = frames.Select(f => f.File).ToList();
var jpeg = new ConvertOptions(ConvertTarget.Jpeg);
IEnumerable<(ConvertSkip, int)> Skipped(ConvertSequencePlan plan) => plan.Skipped.Select(s => (s.Reason, s.Frames));
var server = ConvertPlanner.Plan(manifest, folder, present, jpeg, DecoderFactory.CreateServer(), "Roof");
Assert.Empty(server.Frames);
Assert.SequenceEqual([(ConvertSkip.Undecodable, 2)], Skipped(server));
var reads = ConvertPlanner.Plan(manifest, folder, present, jpeg, new ImageDecoder([new DecodesTiffs(decodes: true), new ManagedDecoderProvider()]), "Roof");
Assert.SequenceEqual([0], reads.Frames.Select(f => f.Index));
Assert.SequenceEqual([(ConvertSkip.Undecodable, 1)], Skipped(reads));
var fails = ConvertPlanner.Plan(manifest, folder, present, jpeg, new ImageDecoder([new DecodesTiffs(decodes: false), new ManagedDecoderProvider()]), "Roof");
Assert.SequenceEqual([(ConvertSkip.Undecodable, 2)], Skipped(fails));
var held = ConvertPlanner.Plan(manifest, folder, present, jpeg,
new ImageDecoder([new DecodesTiffs(decodes: true, failure: new IOException("held by another process")), new ManagedDecoderProvider()]), "Roof");
Assert.Empty(held.Frames);
Assert.SequenceEqual([(ConvertSkip.Undecodable, 1), (ConvertSkip.Inaccessible, 1)], Skipped(held));
}
/// <summary>
/// WIC as a PC has it, for TIFFs: native by format (its probe trusts it), and a decode that reads a CMYK TIFF, finds
/// nothing in it, or fails as <paramref name="failure"/> says.
/// </summary>
private sealed class DecodesTiffs(bool decodes, Exception? failure = null) : IImageDecoderProvider
{
public string Name => "wic";
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Tiff ? DecodeTier.Native : null;
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default)
{
if (failure is not null) throw failure;
if (!decodes) return null;
var red = new ImageBuffer(6, 4);
red.Plane(0).Fill(0.6f);
return new DecodedImage(red, DecodeTier.Native, Name, 6, 4);
}
}
/// <summary>
/// The dry run reads the headers of TIFF, PNG and JPEG files before it promises them (owner review 2, R7): on the server,
/// a CMYK, CIELab, palette or ZSTD TIFF, one with more samples than its decoder takes and an arithmetic-coded JPEG are
/// left to a PC, never planned to fail at every retry; where another decoder reads them (WIC on a PC), they convert. A
/// photo larger than a converted file may be stays as it is everywhere.
/// </summary>
[Test]
public void WhatTheServersDecodersRefuseIsLeftToAPc()
{
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(folder.File("originals/x"))!);
(string File, ImageFileFormat Format, byte[] Bytes, int W, int H)[] files =
[
("rgb.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 2, [200, 30, 10]), 6, 4),
("cmyk.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 5, [0, 255, 255, 0]), 6, 4),
("lab.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 8, [50, 60, 70]), 6, 4),
("palette.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 3, [1]), 6, 4),
("zstd.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 2, [1, 2, 3], d => d.Short(TiffTags.Compression, 50000)), 6, 4),
// RGBA of 576 M samples: beyond the managed reader, while its DNG (432 M) is not.
("stitched.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 2, [1, 2, 3, 4], d =>
{
d.Long(TiffTags.ImageWidth, 12_000);
d.Long(TiffTags.ImageLength, 12_000);
}), 12_000, 12_000),
("arithmetic.jpg", ImageFileFormat.Jpeg, ColorFixtures.JpegHeader(6, 4, sof: 0xC9), 6, 4),
("stitched.png", ImageFileFormat.Png, ColorFixtures.PngHeader(17_000, 17_000), 17_000, 17_000),
];
var frames = files.Select((f, i) => Frame(i, f.File, f.Format) with { W = f.W, H = f.H }).ToArray();
for (var i = 0; i < files.Length; i++) File.WriteAllBytes(folder.File(frames[i].File), files[i].Bytes);
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
var present = frames.Select(f => f.File).ToList();
var dng = new ConvertOptions(ConvertTarget.LinearDng);
var server = ConvertPlanner.Plan(manifest, folder, present, dng, DecoderFactory.CreateServer(), "Roof");
Assert.SequenceEqual([0], server.Frames.Select(f => f.Index));
Assert.SequenceEqual([(ConvertSkip.NoNativeDecoder, 6), (ConvertSkip.TooLarge, 1)], server.Skipped.Select(s => (s.Reason, s.Frames)));
// A PC whose WIC reads them: the managed decoders first, WIC for what they refuse.
var desktop = ConvertPlanner.Plan(manifest, folder, present, dng, new ImageDecoder([new ReadsEverything(), new ManagedDecoderProvider()]), "Roof");
Assert.SequenceEqual(Enumerable.Range(0, 7), desktop.Frames.Select(f => f.Index));
Assert.SequenceEqual([(ConvertSkip.TooLarge, 1)], desktop.Skipped.Select(s => (s.Reason, s.Frames)));
// The managed provider of a conversion reads headers; the import's does not open the files.
Assert.True(FrameConverter.Decoders(DecoderFactory.CreateServer(), dng, ImageFileFormat.Tiff).Providers.OfType<ManagedDecoderProvider>().Single().ReadsHeaders);
Assert.Equal(DecodeTier.Native, DecoderFactory.CreateServer().Probe(folder.File(frames[1].File), ImageFileFormat.Tiff).Tier);
}
/// <summary>A photo whose profile's colorants convert nothing is planned out of a DNG, which converts through them, and carried into a TIFF.</summary>
[Test]
public void AProfileThatConvertsNothingKeepsItsPhotoOutOfADng()
{
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(folder.File("originals/x"))!);
using var scene = ColorFixtures.Scene();
var frame = Frame(0, "odd.jpg", ImageFileFormat.Jpeg);
File.WriteAllBytes(folder.File(frame.File), ColorFixtures.Jpeg(scene, ColorFixtures.Degenerate()));
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = [frame] };
var dng = ConvertPlanner.Plan(manifest, folder, [frame.File], new ConvertOptions(ConvertTarget.LinearDng), DecoderFactory.CreateServer(), "Roof");
Assert.SequenceEqual([(ConvertSkip.ColorProfile, 1)], dng.Skipped.Select(s => (s.Reason, s.Frames)));
var tiff = ConvertPlanner.Plan(manifest, folder, [frame.File], new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), "Roof");
Assert.Count(1, tiff.Frames);
}
[Test]
public void SizesAreEstimatedFromThePixels()
{
const long pixels = 6000L * 4000;
Assert.Equal((long)Math.Ceiling(pixels * 30 / 8.0), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 6000, 4000));
Assert.Equal((long)Math.Ceiling(pixels * 26 / 8.0), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 6000, 4000));
Assert.Equal((long)Math.Ceiling(pixels * ExportEstimator.JpegBitsPerPixel(95) * 1.25 / 8),
ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg), 6000, 4000));
Assert.Less(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg, 50), 6000, 4000),
ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg, 95), 6000, 4000));
// Unknown size: a 24 MP frame.
Assert.Equal(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 6000, 4000), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 0, 0));
Assert.Equal((long)Math.Ceiling(1_000_000_000 * 1.15) + (512L << 20), ConvertPlanner.Required(1_000_000_000));
Assert.True(new ConvertSpace(100, null).Enough);
Assert.True(new ConvertSpace(100, 100).Enough);
Assert.False(new ConvertSpace(100, 99).Enough);
}
[Test]
public void TheFingerprintFollowsWhatThePlanConverts()
{
var id = Guid.NewGuid();
ConvertSequencePlan Sequence(int version, int frames, params ConvertBlock[] blocked) =>
new(id, "Roof", "roof", version, [.. Enumerable.Range(0, frames).Select(i => new ConvertFramePlan(Guid.Empty, i, $"f{i}", ImageFileFormat.Cr2, 10, 20))],
[], blocked);
ConvertPlan Plan(ConvertOptions options, ConvertMode mode, ConvertSequencePlan sequence) => new(Guid.Empty, options, mode, [sequence], new ConvertSpace(0, null));
var dng = new ConvertOptions(ConvertTarget.LinearDng);
var reference = Plan(dng, ConvertMode.Replace, Sequence(3, 2)).Fingerprint();
Assert.Equal(64, reference.Length);
Assert.Equal(reference, Plan(dng with { Quality = 40 }, ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
Assert.NotEqual(reference, Plan(dng, ConvertMode.NewSequence, Sequence(3, 2)).Fingerprint());
Assert.NotEqual(reference, Plan(new ConvertOptions(ConvertTarget.Jpeg), ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
Assert.NotEqual(reference, Plan(dng, ConvertMode.Replace, Sequence(4, 2)).Fingerprint());
Assert.NotEqual(reference, Plan(dng, ConvertMode.Replace, Sequence(3, 1)).Fingerprint());
var busy = Plan(dng, ConvertMode.Replace, Sequence(3, 2, ConvertBlock.Busy));
Assert.NotEqual(reference, busy.Fingerprint());
Assert.Equal((0, 0L), (busy.Frames, busy.BytesAfter));
Assert.NotEqual(Plan(new ConvertOptions(ConvertTarget.Jpeg, 90), ConvertMode.Replace, Sequence(3, 2)).Fingerprint(),
Plan(new ConvertOptions(ConvertTarget.Jpeg, 80), ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
}
[Test]
public void TheDigestFollowsThePhotosConvertedAndTheirContent()
{
var a = Guid.NewGuid();
var b = Guid.NewGuid();
ConvertSequencePlan Sequence(int version, params (Guid Id, string Sha)[] frames) =>
new(Guid.Empty, "Roof", "roof", version, [.. frames.Select((f, i) => new ConvertFramePlan(f.Id, i, $"f{i}", ImageFileFormat.Cr2, 10, 20, f.Sha))], [], []);
var reference = Sequence(3, (a, "aa"), (b, "bb")).Digest();
Assert.Equal(64, reference.Length);
// The frames version and the case of a hash say nothing of what converts.
Assert.Equal(reference, Sequence(4, (a, "AA"), (b, "bb")).Digest());
Assert.Equal(reference, ConvertSequencePlan.DigestOf([(a, "aa"), (b, "bb")]));
// Another content, a photo fewer or more (verified since the plan), the same photos in another order: another digest.
Assert.NotEqual(reference, Sequence(3, (a, "aa"), (b, "cc")).Digest());
Assert.NotEqual(reference, Sequence(3, (a, "aa")).Digest());
Assert.NotEqual(reference, Sequence(3, (a, "aa"), (b, "bb"), (Guid.NewGuid(), "dd")).Digest());
Assert.NotEqual(reference, Sequence(3, (b, "bb"), (a, "aa")).Digest());
// The plan's fingerprint pins the content too: as many photos of the same size holding other content is another plan.
ConvertPlan Plan(ConvertSequencePlan sequence) => new(Guid.Empty, new ConvertOptions(ConvertTarget.LinearDng), ConvertMode.Replace, [sequence], new ConvertSpace(0, null));
Assert.NotEqual(Plan(Sequence(3, (a, "aa"), (b, "bb"))).Fingerprint(), Plan(Sequence(3, (a, "aa"), (b, "cc"))).Fingerprint());
}
/// <summary>A provider that reads CR2 natively by format, and probes the files <paramref name="applies"/> picks (all
/// by default) at a fixed tier, the others native.</summary>
private sealed class ProbeAs(DecodeTier tier, Func<string, bool>? applies = null) : IImageDecoderProvider
{
public string Name => "probe";
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Cr2 ? DecodeTier.Native : null;
public FileTier Probe(string path, ImageFileFormat format) => new(format, applies?.Invoke(path) == false ? DecodeTier.Native : tier, Name, null, 0, 0);
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
}
/// <summary>WIC as a PC with every codec has it: JPEG, PNG and TIFF native, whatever they hold.</summary>
private sealed class ReadsEverything : IImageDecoderProvider
{
public string Name => "wic";
public DecodeTier? TierFor(ImageFileFormat format) => format is ImageFileFormat.Jpeg or ImageFileFormat.Png or ImageFileFormat.Tiff ? DecodeTier.Native : null;
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
}
/// <summary>WIC with the Raw Image Extension, as a conversion sees it: every camera RAW native, developed at 8 bits.</summary>
internal sealed class EightBitRaw : IImageDecoderProvider
{
public string Name => "wic";
public bool DevelopsRawAt8Bits => true;
public DecodeTier? TierFor(ImageFileFormat format) => ImageFormatInfo.Of(format).Family == ImageFamily.CameraRaw ? DecodeTier.Native : null;
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
}
}