CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
516 lines
33 KiB
C#
516 lines
33 KiB
C#
using Chronos.Core.Archiving;
|
||
using Chronos.Core.Codecs.Icc;
|
||
using Chronos.Core.Codecs.Jpeg;
|
||
using Chronos.Core.Codecs.Raw;
|
||
using Chronos.Core.Codecs.Tiff;
|
||
using Chronos.Core.Conversion;
|
||
using Chronos.Core.Export;
|
||
using Chronos.Core.Imaging;
|
||
using Chronos.Core.Imaging.Decoders;
|
||
using Chronos.Core.Metadata.Tiff;
|
||
using Chronos.Core.Model;
|
||
using Chronos.Core.Tests.Raw;
|
||
using Chronos.Testing;
|
||
|
||
namespace Chronos.Core.Tests.Conversion;
|
||
|
||
/// <summary>The dry run of a conversion (owner review 2, R7): which frames convert, which stay and why, the sizes, the space and the fingerprint.</summary>
|
||
[Category("conversion")]
|
||
public sealed class ConvertPlannerTests(TestContext context)
|
||
{
|
||
private static SidecarFrame Frame(int i, string file, ImageFileFormat format, string? sha = "s", FrameFlags flags = FrameFlags.None,
|
||
DecodeTier tier = default) => new()
|
||
{
|
||
Id = Guid.NewGuid(),
|
||
I = i,
|
||
File = "originals/" + file,
|
||
Size = 1000 + i,
|
||
Sha256 = sha is null ? null : sha + i,
|
||
Format = format,
|
||
W = 6000,
|
||
H = 4000,
|
||
Flags = flags,
|
||
Tier = tier,
|
||
};
|
||
|
||
[Test]
|
||
public void EachFrameThatCannotConvertSaysWhy()
|
||
{
|
||
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("2026/roof");
|
||
SidecarFrame[] frames =
|
||
[
|
||
Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2),
|
||
Frame(1, "IMG_0002.DNG", ImageFileFormat.Dng),
|
||
Frame(2, "IMG_0003.CR2", ImageFileFormat.Cr2),
|
||
Frame(3, "IMG_0004.CR2", ImageFileFormat.Cr2, sha: null),
|
||
Frame(4, "IMG_0005.CR2", ImageFileFormat.Cr2, flags: FrameFlags.Undecodable),
|
||
Frame(5, "IMG_0006.CR3", ImageFileFormat.Cr3),
|
||
Frame(6, "IMG_0007.HEIC", ImageFileFormat.Heif),
|
||
Frame(7, "IMG_0008.JPG", ImageFileFormat.Jpeg, flags: FrameFlags.Missing),
|
||
Frame(8, "IMG_0009.CR2", ImageFileFormat.Cr2, flags: FrameFlags.PreviewOnly | FrameFlags.Excluded, tier: DecodeTier.EmbeddedPreview),
|
||
];
|
||
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames, Versions = new ManifestVersions { Frames = 7 } };
|
||
// The third photo's file is gone; the preview-only one probes as preview here, the first one native.
|
||
var present = frames.Where(f => f.I != 2).Select(f => f.File).ToList();
|
||
var decoder = new ImageDecoder([new ProbeAs(DecodeTier.EmbeddedPreview, path => path.EndsWith("IMG_0009.CR2", StringComparison.Ordinal))]);
|
||
var plan = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.LinearDng), decoder, "Roof");
|
||
|
||
Assert.Equal((manifest.Id, "Roof", "2026/roof", 7), (plan.SequenceId, plan.Title, plan.Folder, plan.FramesVersion));
|
||
Assert.SequenceEqual([0], plan.Frames.Select(f => f.Index));
|
||
Assert.Equal("s0", plan.Frames.Single().Sha256);
|
||
Assert.SequenceEqual(
|
||
[(ConvertSkip.SameFormat, 1), (ConvertSkip.NoNativeDecoder, 2), (ConvertSkip.PreviewOnly, 1), (ConvertSkip.Undecodable, 1), (ConvertSkip.Missing, 2),
|
||
(ConvertSkip.Unhashed, 1)],
|
||
plan.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
Assert.True(plan.Converts);
|
||
Assert.Equal(1000, plan.BytesBefore);
|
||
Assert.Equal(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 6000, 4000), plan.BytesAfter);
|
||
Assert.Equal(8, plan.SkippedFrames);
|
||
}
|
||
|
||
[Test]
|
||
public void APhotoRecordedAsPreviewOnlyElsewhereConvertsWhereItDecodes()
|
||
{
|
||
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
|
||
var frame = Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2, flags: FrameFlags.PreviewOnly, tier: DecodeTier.EmbeddedPreview);
|
||
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = [frame] };
|
||
var plan = ConvertPlanner.Plan(manifest, folder, [frame.File], new ConvertOptions(ConvertTarget.Tiff16), new ImageDecoder([new ProbeAs(DecodeTier.Native)]), "Roof");
|
||
Assert.Count(1, plan.Frames);
|
||
Assert.Empty(plan.Skipped);
|
||
}
|
||
|
||
/// <summary>
|
||
/// A frame whose tier is not known yet (imported, not analysed) or was recorded native by another host with other
|
||
/// decoders is asked again: the dry run promises only what the file says here now. A file the decoders here do not read
|
||
/// at all, although they read its format, is left to a PC; only a frame flagged unreadable is unreadable.
|
||
/// </summary>
|
||
[Test]
|
||
public void AFrameIsAskedWhateverTierWasRecorded()
|
||
{
|
||
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
|
||
SidecarFrame[] frames = [Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2), Frame(1, "IMG_0002.CR2", ImageFileFormat.Cr2, tier: DecodeTier.Native)];
|
||
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
|
||
var present = frames.Select(f => f.File).ToList();
|
||
var dng = new ConvertOptions(ConvertTarget.LinearDng);
|
||
|
||
var preview = ConvertPlanner.Plan(manifest, folder, present, dng, new ImageDecoder([new ProbeAs(DecodeTier.EmbeddedPreview)]), "Roof");
|
||
Assert.Empty(preview.Frames);
|
||
Assert.SequenceEqual([(ConvertSkip.PreviewOnly, 2)], preview.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
var unreadable = ConvertPlanner.Plan(manifest, folder, present, dng, new ImageDecoder([new ProbeAs(DecodeTier.Unavailable)]), "Roof");
|
||
Assert.SequenceEqual([(ConvertSkip.NoNativeDecoder, 2)], unreadable.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
var native = ConvertPlanner.Plan(manifest, folder, present, dng, new ImageDecoder([new ProbeAs(DecodeTier.Native)]), "Roof");
|
||
Assert.SequenceEqual([0, 1], native.Frames.Select(f => f.Index));
|
||
}
|
||
|
||
/// <summary>
|
||
/// A file the plan cannot read is not called missing unless it is gone: one another program holds, or that access is
|
||
/// denied to, could not be opened now (a plan made once it is free converts it), and one shorter than its own headers
|
||
/// cannot be decoded.
|
||
/// </summary>
|
||
[Test]
|
||
public void AFileThatCannotBeReadNowSaysWhy()
|
||
{
|
||
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
|
||
Exception[] failures =
|
||
[
|
||
new FileNotFoundException("gone"), new DirectoryNotFoundException("gone"), new IOException("held by another process"),
|
||
new UnauthorizedAccessException("denied"), new EndOfStreamException("short"),
|
||
];
|
||
SidecarFrame[] frames = [.. failures.Select((_, i) => Frame(i, $"IMG_{i:0000}.CR2", ImageFileFormat.Cr2))];
|
||
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
|
||
// IMG_0000 fails as the first, IMG_0004 as the last.
|
||
var decoder = new ImageDecoder([new ProbeFails(path => failures[Path.GetFileNameWithoutExtension(path)[^1] - '0'])]);
|
||
|
||
var plan = ConvertPlanner.Plan(manifest, folder, [.. frames.Select(f => f.File)], new ConvertOptions(ConvertTarget.LinearDng), decoder, "Roof");
|
||
Assert.Empty(plan.Frames);
|
||
Assert.SequenceEqual([(ConvertSkip.Undecodable, 1), (ConvertSkip.Missing, 2), (ConvertSkip.Inaccessible, 2)], plan.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
}
|
||
|
||
/// <summary>A RAW decoder whose probe of a file fails as <paramref name="failure"/> says.</summary>
|
||
private sealed class ProbeFails(Func<string, Exception> failure) : IImageDecoderProvider
|
||
{
|
||
public string Name => "fails";
|
||
|
||
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Cr2 ? DecodeTier.Native : null;
|
||
|
||
public FileTier Probe(string path, ImageFileFormat format) => throw failure(path);
|
||
|
||
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
|
||
}
|
||
|
||
/// <summary>
|
||
/// The managed decoder develops every RAW it reads natively (at float precision) even where WIC could; a RAW only WIC's
|
||
/// Raw Image Extension develops, at 8 bits, converts to JPEG but not to TIFF-16 or linear DNG.
|
||
/// </summary>
|
||
[Test]
|
||
public void ARawDevelopedAtEightBitsOnlyConvertsToJpegOnly()
|
||
{
|
||
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
|
||
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(folder.File("originals/x"))!);
|
||
using var scene = SyntheticRaw.Smooth(96, 64);
|
||
var active = new RawRect(24, 4, 24 + 96, 4 + 64);
|
||
var sensor = NativeRawWriters.Sensor(scene, "Canon EOS 5D Mark III", CfaPattern.Bayer("RGGB"), 128, 72, active.Left, active.Top, 2048, 16383);
|
||
SidecarFrame[] frames =
|
||
[
|
||
Frame(0, "IMG_0001.CR2", ImageFileFormat.Cr2, tier: DecodeTier.Native),
|
||
Frame(1, "IMG_0002.CR2", ImageFileFormat.Cr2, tier: DecodeTier.Native),
|
||
Frame(2, "IMG_0003.CR3", ImageFileFormat.Cr3, tier: DecodeTier.Native),
|
||
];
|
||
File.WriteAllBytes(folder.File(frames[0].File), NativeRawWriters.Cr2(sensor, active));
|
||
// A body the managed decoder has no matrix for: only the 8-bit development reads it.
|
||
File.WriteAllBytes(folder.File(frames[1].File), NativeRawWriters.Cr2(sensor, active, model: "Canon EOS R99"));
|
||
File.WriteAllBytes(folder.File(frames[2].File), [0]);
|
||
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
|
||
var present = frames.Select(f => f.File).ToList();
|
||
var desktop = new ImageDecoder([new EightBitRaw(), new ManagedDecoderProvider()]);
|
||
|
||
foreach (var target in (ConvertTarget[])[ConvertTarget.Tiff16, ConvertTarget.LinearDng])
|
||
{
|
||
var plan = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(target), desktop, "Roof");
|
||
Assert.SequenceEqual([0], plan.Frames.Select(f => f.Index));
|
||
Assert.SequenceEqual([(ConvertSkip.EightBit, 2)], plan.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
}
|
||
var jpeg = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.Jpeg), desktop, "Roof");
|
||
Assert.SequenceEqual([0, 1, 2], jpeg.Frames.Select(f => f.Index));
|
||
// The server, without WIC: the unsupported body and the CR3 are to convert on a PC.
|
||
var server = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), "Roof");
|
||
Assert.SequenceEqual([0], server.Frames.Select(f => f.Index));
|
||
Assert.SequenceEqual([(ConvertSkip.NoNativeDecoder, 1), (ConvertSkip.PreviewOnly, 1)], server.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
}
|
||
|
||
/// <summary>A photo whose colour space no target can keep, or which a target cannot hold, is planned out with why.</summary>
|
||
[Test]
|
||
public void ColourAndSizeTheTargetCannotHoldArePlannedOut()
|
||
{
|
||
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
|
||
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(folder.File("originals/x"))!);
|
||
using var scene = ColorFixtures.Scene();
|
||
SidecarFrame[] frames =
|
||
[
|
||
Frame(0, "plain.jpg", ImageFileFormat.Jpeg),
|
||
Frame(1, "p3.jpg", ImageFileFormat.Jpeg),
|
||
Frame(2, "unknown.jpg", ImageFileFormat.Jpeg),
|
||
Frame(3, "cmyk.jpg", ImageFileFormat.Jpeg),
|
||
Frame(4, "panorama.png", ImageFileFormat.Png) with { W = 70_000, H = 2_500 },
|
||
Frame(5, "huge.png", ImageFileFormat.Png) with { W = 70_000, H = 12_000 },
|
||
];
|
||
File.WriteAllBytes(folder.File(frames[0].File), ColorFixtures.Jpeg(scene));
|
||
File.WriteAllBytes(folder.File(frames[1].File), ColorFixtures.Jpeg(scene, IccProfile.DisplayP3.Data));
|
||
File.WriteAllBytes(folder.File(frames[2].File), ColorFixtures.Jpeg(scene, exifTiff: ColorFixtures.Exif(0xFFFF)));
|
||
File.WriteAllBytes(folder.File(frames[3].File), ColorFixtures.Jpeg(scene, ColorFixtures.Cmyk()));
|
||
// A panorama's header (the dry run reads no further), and a file the sidecar's size already rules out.
|
||
File.WriteAllBytes(folder.File(frames[4].File), ColorFixtures.PngHeader(70_000, 2_500));
|
||
File.WriteAllBytes(folder.File(frames[5].File), [0]);
|
||
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
|
||
var present = frames.Select(f => f.File).ToList();
|
||
var decoder = DecoderFactory.CreateServer();
|
||
|
||
// A TIFF carries the P3 photo's profile; a DNG, which holds sRGB as Chronos reads it, does not take it. Neither takes
|
||
// the unknown and CMYK ones, nor 2.5 G samples.
|
||
var tiff = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.Tiff16), decoder, "Roof");
|
||
Assert.SequenceEqual([0, 1, 4], tiff.Frames.Select(f => f.Index));
|
||
Assert.SequenceEqual([(ConvertSkip.ColorProfile, 2), (ConvertSkip.TooLarge, 1)], tiff.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
var dng = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.LinearDng), decoder, "Roof");
|
||
Assert.SequenceEqual([0, 4], dng.Frames.Select(f => f.Index));
|
||
Assert.SequenceEqual([(ConvertSkip.ColorProfile, 3), (ConvertSkip.TooLarge, 1)], dng.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
// A JPEG side holds 65535 pixels.
|
||
var jpeg = ConvertPlanner.Plan(manifest, folder, present, new ConvertOptions(ConvertTarget.Jpeg), decoder, "Roof");
|
||
Assert.SequenceEqual([(ConvertSkip.SameFormat, 4), (ConvertSkip.TooLarge, 2)], jpeg.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
Assert.True(ConvertPlanner.TooLarge(new ConvertOptions(ConvertTarget.Jpeg), 65_536, 10));
|
||
Assert.False(ConvertPlanner.TooLarge(new ConvertOptions(ConvertTarget.Jpeg), 65_535, 10));
|
||
Assert.False(ConvertPlanner.TooLarge(new ConvertOptions(ConvertTarget.Tiff16), 0, 0));
|
||
}
|
||
|
||
/// <summary>
|
||
/// The size bound is what the server's decoders read back, which every converted file must (owner review 2, R7), never
|
||
/// an estimate: a JPEG of 2^28 pixels, a TIFF-16 or DNG (whose raster the same TIFF reader reads) of 2^29 samples, whose
|
||
/// headers are read while one row more is refused. A file that size stays within the array it is made in (2 GB) even
|
||
/// when its pixels do not compress at all (16-bit noise takes 48 bits a pixel, up to 51 as lossless JPEG codes them),
|
||
/// and noise encodes within that bound in every target.
|
||
/// </summary>
|
||
[Test]
|
||
public void TheSizeBoundIsWhatTheServersDecodersReadBack()
|
||
{
|
||
var jpeg = new ConvertOptions(ConvertTarget.Jpeg, 100);
|
||
Assert.False(ConvertPlanner.TooLarge(jpeg, 16_384, 16_384));
|
||
Assert.True(ConvertPlanner.TooLarge(jpeg, 16_384, 16_385));
|
||
Assert.False(ConvertPlanner.TooLarge(jpeg, 4_096, 65_535));
|
||
Assert.True(ConvertPlanner.TooLarge(jpeg, 4_097, 65_535));
|
||
var largest = JpegDecoder.ReadInfo(ColorFixtures.JpegHeader(16_384, 16_384));
|
||
Assert.Equal((16_384, 16_384), (largest.Width, largest.Height));
|
||
Assert.Throws<InvalidDataException>(() => JpegDecoder.ReadInfo(ColorFixtures.JpegHeader(16_384, 16_385)));
|
||
foreach (var target in (ConvertTarget[])[ConvertTarget.Tiff16, ConvertTarget.LinearDng])
|
||
{
|
||
Assert.False(ConvertPlanner.TooLarge(new ConvertOptions(target), 13_377, 13_377), $"{target} of 2^29 samples at most");
|
||
Assert.True(ConvertPlanner.TooLarge(new ConvertOptions(target), 13_377, 13_378), $"{target} of one row more");
|
||
}
|
||
foreach (ushort photometric in (ushort[])[2, 34892])
|
||
{
|
||
Assert.Equal((13_377, 13_377), TiffImageReader.ReadInfo(new MemoryStream(ColorFixtures.TiffHeader(13_377, 13_377, photometric))));
|
||
Assert.Throws<InvalidDataException>(() => TiffImageReader.ReadInfo(new MemoryStream(ColorFixtures.TiffHeader(13_377, 13_378, photometric))));
|
||
}
|
||
Assert.LessOrEqual(ConvertPlanner.WorstBytes(16_384, 16_384), Array.MaxLength);
|
||
Assert.LessOrEqual(ConvertPlanner.WorstBytes(13_377, 13_377), Array.MaxLength);
|
||
// 342 MP: its estimate (30 or 26 bits a pixel) would fit, the worst case would not, and nothing reads it back.
|
||
foreach (var target in Enum.GetValues<ConvertTarget>())
|
||
{
|
||
Assert.True(ConvertPlanner.TooLarge(new ConvertOptions(target, 100), 18_500, 18_500), $"{target} at 342 MP");
|
||
Assert.Less(ConvertPlanner.EstimateBytes(new ConvertOptions(target, 100), 18_500, 18_500), Array.MaxLength);
|
||
}
|
||
Assert.Greater(ConvertPlanner.WorstBytes(18_500, 18_500), Array.MaxLength);
|
||
using var noise = new ImageBuffer(512, 512);
|
||
var random = new Random(7);
|
||
foreach (ref var v in noise.Data) v = random.NextSingle();
|
||
var metadata = new Chronos.Core.Metadata.FrameMetadata { CapturedAt = new DateTime(2026, 10, 27, 12, 0, 0) };
|
||
foreach (var target in Enum.GetValues<ConvertTarget>())
|
||
{
|
||
var bytes = FrameConverter.Encode(noise, metadata, new ConvertOptions(target, 100));
|
||
var bits = bytes.Length * 8.0 / noise.PlaneLength;
|
||
context.Log($"{target}: {bits:0.00} bits a pixel of noise");
|
||
Assert.LessOrEqual(bits, ConvertPlanner.WorstBitsPerPixel, $"{target} encodes noise within the bound, its headers included");
|
||
Assert.LessOrEqual(bytes.Length, ConvertPlanner.WorstBytes(512, 512));
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Any size is answered, a damaged header's included (owner review 2, R7): sides up to <see cref="int.MaxValue"/> are too
|
||
/// large for every target, never let through by a product that wrapped around, and the worst case saturates instead of
|
||
/// turning negative.
|
||
/// </summary>
|
||
[Test]
|
||
public void AnySizeIsAnsweredWithoutOverflow()
|
||
{
|
||
(int W, int H)[] absurd =
|
||
[
|
||
(int.MaxValue, int.MaxValue), (2_000_000_000, 2_000_000_000), (1_800_000_000, 1_800_000_000), (1_000_000_000, 1_000_000_000),
|
||
(int.MaxValue, 1), (1, int.MaxValue),
|
||
];
|
||
foreach (var target in Enum.GetValues<ConvertTarget>())
|
||
{
|
||
foreach (var (w, h) in absurd) Assert.True(ConvertPlanner.TooLarge(new ConvertOptions(target), w, h), $"{target} at {w}×{h}");
|
||
Assert.Greater(ConvertPlanner.EstimateBytes(new ConvertOptions(target), int.MaxValue, int.MaxValue), 0L);
|
||
}
|
||
Assert.Equal(long.MaxValue, ConvertPlanner.WorstBytes(int.MaxValue, int.MaxValue));
|
||
Assert.Greater(ConvertPlanner.WorstBytes(1_800_000_000, 1_800_000_000), Array.MaxLength);
|
||
Assert.Greater(ConvertPlanner.WorstBytes(65_535, 65_535), 0L);
|
||
Assert.False(ConvertPlanner.TooLarge(new ConvertOptions(ConvertTarget.Tiff16), -5, 10));
|
||
}
|
||
|
||
/// <summary>
|
||
/// A frame flagged unreadable by the host that analysed it (the server, whose decoders refuse a CMYK TIFF) is asked again
|
||
/// where the conversion's decoders have others for its format (WIC on a PC), with a small decode (owner review 2, R7):
|
||
/// converted where they read it, unreadable where they do not. A corrupt frame stays unreadable, and so does a flagged one
|
||
/// where only the managed decoders read the format. A file that cannot be read while it is asked is inaccessible, nothing
|
||
/// concluded of its content.
|
||
/// </summary>
|
||
[Test]
|
||
public void AFrameFlaggedUnreadableElsewhereIsAskedWhereOtherDecodersReadIt()
|
||
{
|
||
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
|
||
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(folder.File("originals/x"))!);
|
||
SidecarFrame[] frames =
|
||
[
|
||
Frame(0, "cmyk.tif", ImageFileFormat.Tiff, flags: FrameFlags.Undecodable) with { W = 6, H = 4 },
|
||
Frame(1, "corrupt.tif", ImageFileFormat.Tiff, flags: FrameFlags.Corrupt) with { W = 6, H = 4 },
|
||
];
|
||
File.WriteAllBytes(folder.File(frames[0].File), ColorFixtures.Tiff(6, 4, 5, [0, 255, 255, 0]));
|
||
File.WriteAllBytes(folder.File(frames[1].File), ColorFixtures.Tiff(6, 4, 2, [1, 2, 3]));
|
||
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
|
||
var present = frames.Select(f => f.File).ToList();
|
||
var jpeg = new ConvertOptions(ConvertTarget.Jpeg);
|
||
IEnumerable<(ConvertSkip, int)> Skipped(ConvertSequencePlan plan) => plan.Skipped.Select(s => (s.Reason, s.Frames));
|
||
|
||
var server = ConvertPlanner.Plan(manifest, folder, present, jpeg, DecoderFactory.CreateServer(), "Roof");
|
||
Assert.Empty(server.Frames);
|
||
Assert.SequenceEqual([(ConvertSkip.Undecodable, 2)], Skipped(server));
|
||
var reads = ConvertPlanner.Plan(manifest, folder, present, jpeg, new ImageDecoder([new DecodesTiffs(decodes: true), new ManagedDecoderProvider()]), "Roof");
|
||
Assert.SequenceEqual([0], reads.Frames.Select(f => f.Index));
|
||
Assert.SequenceEqual([(ConvertSkip.Undecodable, 1)], Skipped(reads));
|
||
var fails = ConvertPlanner.Plan(manifest, folder, present, jpeg, new ImageDecoder([new DecodesTiffs(decodes: false), new ManagedDecoderProvider()]), "Roof");
|
||
Assert.SequenceEqual([(ConvertSkip.Undecodable, 2)], Skipped(fails));
|
||
var held = ConvertPlanner.Plan(manifest, folder, present, jpeg,
|
||
new ImageDecoder([new DecodesTiffs(decodes: true, failure: new IOException("held by another process")), new ManagedDecoderProvider()]), "Roof");
|
||
Assert.Empty(held.Frames);
|
||
Assert.SequenceEqual([(ConvertSkip.Undecodable, 1), (ConvertSkip.Inaccessible, 1)], Skipped(held));
|
||
}
|
||
|
||
/// <summary>
|
||
/// WIC as a PC has it, for TIFFs: native by format (its probe trusts it), and a decode that reads a CMYK TIFF, finds
|
||
/// nothing in it, or fails as <paramref name="failure"/> says.
|
||
/// </summary>
|
||
private sealed class DecodesTiffs(bool decodes, Exception? failure = null) : IImageDecoderProvider
|
||
{
|
||
public string Name => "wic";
|
||
|
||
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Tiff ? DecodeTier.Native : null;
|
||
|
||
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default)
|
||
{
|
||
if (failure is not null) throw failure;
|
||
if (!decodes) return null;
|
||
var red = new ImageBuffer(6, 4);
|
||
red.Plane(0).Fill(0.6f);
|
||
return new DecodedImage(red, DecodeTier.Native, Name, 6, 4);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// The dry run reads the headers of TIFF, PNG and JPEG files before it promises them (owner review 2, R7): on the server,
|
||
/// a CMYK, CIELab, palette or ZSTD TIFF, one with more samples than its decoder takes and an arithmetic-coded JPEG are
|
||
/// left to a PC, never planned to fail at every retry; where another decoder reads them (WIC on a PC), they convert. A
|
||
/// photo larger than a converted file may be stays as it is everywhere.
|
||
/// </summary>
|
||
[Test]
|
||
public void WhatTheServersDecodersRefuseIsLeftToAPc()
|
||
{
|
||
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
|
||
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(folder.File("originals/x"))!);
|
||
(string File, ImageFileFormat Format, byte[] Bytes, int W, int H)[] files =
|
||
[
|
||
("rgb.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 2, [200, 30, 10]), 6, 4),
|
||
("cmyk.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 5, [0, 255, 255, 0]), 6, 4),
|
||
("lab.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 8, [50, 60, 70]), 6, 4),
|
||
("palette.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 3, [1]), 6, 4),
|
||
("zstd.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 2, [1, 2, 3], d => d.Short(TiffTags.Compression, 50000)), 6, 4),
|
||
// RGBA of 576 M samples: beyond the managed reader, while its DNG (432 M) is not.
|
||
("stitched.tif", ImageFileFormat.Tiff, ColorFixtures.Tiff(6, 4, 2, [1, 2, 3, 4], d =>
|
||
{
|
||
d.Long(TiffTags.ImageWidth, 12_000);
|
||
d.Long(TiffTags.ImageLength, 12_000);
|
||
}), 12_000, 12_000),
|
||
("arithmetic.jpg", ImageFileFormat.Jpeg, ColorFixtures.JpegHeader(6, 4, sof: 0xC9), 6, 4),
|
||
("stitched.png", ImageFileFormat.Png, ColorFixtures.PngHeader(17_000, 17_000), 17_000, 17_000),
|
||
];
|
||
var frames = files.Select((f, i) => Frame(i, f.File, f.Format) with { W = f.W, H = f.H }).ToArray();
|
||
for (var i = 0; i < files.Length; i++) File.WriteAllBytes(folder.File(frames[i].File), files[i].Bytes);
|
||
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames };
|
||
var present = frames.Select(f => f.File).ToList();
|
||
var dng = new ConvertOptions(ConvertTarget.LinearDng);
|
||
|
||
var server = ConvertPlanner.Plan(manifest, folder, present, dng, DecoderFactory.CreateServer(), "Roof");
|
||
Assert.SequenceEqual([0], server.Frames.Select(f => f.Index));
|
||
Assert.SequenceEqual([(ConvertSkip.NoNativeDecoder, 6), (ConvertSkip.TooLarge, 1)], server.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
// A PC whose WIC reads them: the managed decoders first, WIC for what they refuse.
|
||
var desktop = ConvertPlanner.Plan(manifest, folder, present, dng, new ImageDecoder([new ReadsEverything(), new ManagedDecoderProvider()]), "Roof");
|
||
Assert.SequenceEqual(Enumerable.Range(0, 7), desktop.Frames.Select(f => f.Index));
|
||
Assert.SequenceEqual([(ConvertSkip.TooLarge, 1)], desktop.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
// The managed provider of a conversion reads headers; the import's does not open the files.
|
||
Assert.True(FrameConverter.Decoders(DecoderFactory.CreateServer(), dng, ImageFileFormat.Tiff).Providers.OfType<ManagedDecoderProvider>().Single().ReadsHeaders);
|
||
Assert.Equal(DecodeTier.Native, DecoderFactory.CreateServer().Probe(folder.File(frames[1].File), ImageFileFormat.Tiff).Tier);
|
||
}
|
||
|
||
/// <summary>A photo whose profile's colorants convert nothing is planned out of a DNG, which converts through them, and carried into a TIFF.</summary>
|
||
[Test]
|
||
public void AProfileThatConvertsNothingKeepsItsPhotoOutOfADng()
|
||
{
|
||
var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof");
|
||
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(folder.File("originals/x"))!);
|
||
using var scene = ColorFixtures.Scene();
|
||
var frame = Frame(0, "odd.jpg", ImageFileFormat.Jpeg);
|
||
File.WriteAllBytes(folder.File(frame.File), ColorFixtures.Jpeg(scene, ColorFixtures.Degenerate()));
|
||
var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = [frame] };
|
||
var dng = ConvertPlanner.Plan(manifest, folder, [frame.File], new ConvertOptions(ConvertTarget.LinearDng), DecoderFactory.CreateServer(), "Roof");
|
||
Assert.SequenceEqual([(ConvertSkip.ColorProfile, 1)], dng.Skipped.Select(s => (s.Reason, s.Frames)));
|
||
var tiff = ConvertPlanner.Plan(manifest, folder, [frame.File], new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), "Roof");
|
||
Assert.Count(1, tiff.Frames);
|
||
}
|
||
|
||
[Test]
|
||
public void SizesAreEstimatedFromThePixels()
|
||
{
|
||
const long pixels = 6000L * 4000;
|
||
Assert.Equal((long)Math.Ceiling(pixels * 30 / 8.0), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 6000, 4000));
|
||
Assert.Equal((long)Math.Ceiling(pixels * 26 / 8.0), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 6000, 4000));
|
||
Assert.Equal((long)Math.Ceiling(pixels * ExportEstimator.JpegBitsPerPixel(95) * 1.25 / 8),
|
||
ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg), 6000, 4000));
|
||
Assert.Less(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg, 50), 6000, 4000),
|
||
ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Jpeg, 95), 6000, 4000));
|
||
// Unknown size: a 24 MP frame.
|
||
Assert.Equal(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 6000, 4000), ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 0, 0));
|
||
Assert.Equal((long)Math.Ceiling(1_000_000_000 * 1.15) + (512L << 20), ConvertPlanner.Required(1_000_000_000));
|
||
Assert.True(new ConvertSpace(100, null).Enough);
|
||
Assert.True(new ConvertSpace(100, 100).Enough);
|
||
Assert.False(new ConvertSpace(100, 99).Enough);
|
||
}
|
||
|
||
[Test]
|
||
public void TheFingerprintFollowsWhatThePlanConverts()
|
||
{
|
||
var id = Guid.NewGuid();
|
||
ConvertSequencePlan Sequence(int version, int frames, params ConvertBlock[] blocked) =>
|
||
new(id, "Roof", "roof", version, [.. Enumerable.Range(0, frames).Select(i => new ConvertFramePlan(Guid.Empty, i, $"f{i}", ImageFileFormat.Cr2, 10, 20))],
|
||
[], blocked);
|
||
ConvertPlan Plan(ConvertOptions options, ConvertMode mode, ConvertSequencePlan sequence) => new(Guid.Empty, options, mode, [sequence], new ConvertSpace(0, null));
|
||
var dng = new ConvertOptions(ConvertTarget.LinearDng);
|
||
var reference = Plan(dng, ConvertMode.Replace, Sequence(3, 2)).Fingerprint();
|
||
Assert.Equal(64, reference.Length);
|
||
Assert.Equal(reference, Plan(dng with { Quality = 40 }, ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
|
||
Assert.NotEqual(reference, Plan(dng, ConvertMode.NewSequence, Sequence(3, 2)).Fingerprint());
|
||
Assert.NotEqual(reference, Plan(new ConvertOptions(ConvertTarget.Jpeg), ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
|
||
Assert.NotEqual(reference, Plan(dng, ConvertMode.Replace, Sequence(4, 2)).Fingerprint());
|
||
Assert.NotEqual(reference, Plan(dng, ConvertMode.Replace, Sequence(3, 1)).Fingerprint());
|
||
var busy = Plan(dng, ConvertMode.Replace, Sequence(3, 2, ConvertBlock.Busy));
|
||
Assert.NotEqual(reference, busy.Fingerprint());
|
||
Assert.Equal((0, 0L), (busy.Frames, busy.BytesAfter));
|
||
Assert.NotEqual(Plan(new ConvertOptions(ConvertTarget.Jpeg, 90), ConvertMode.Replace, Sequence(3, 2)).Fingerprint(),
|
||
Plan(new ConvertOptions(ConvertTarget.Jpeg, 80), ConvertMode.Replace, Sequence(3, 2)).Fingerprint());
|
||
}
|
||
|
||
[Test]
|
||
public void TheDigestFollowsThePhotosConvertedAndTheirContent()
|
||
{
|
||
var a = Guid.NewGuid();
|
||
var b = Guid.NewGuid();
|
||
ConvertSequencePlan Sequence(int version, params (Guid Id, string Sha)[] frames) =>
|
||
new(Guid.Empty, "Roof", "roof", version, [.. frames.Select((f, i) => new ConvertFramePlan(f.Id, i, $"f{i}", ImageFileFormat.Cr2, 10, 20, f.Sha))], [], []);
|
||
var reference = Sequence(3, (a, "aa"), (b, "bb")).Digest();
|
||
Assert.Equal(64, reference.Length);
|
||
// The frames version and the case of a hash say nothing of what converts.
|
||
Assert.Equal(reference, Sequence(4, (a, "AA"), (b, "bb")).Digest());
|
||
Assert.Equal(reference, ConvertSequencePlan.DigestOf([(a, "aa"), (b, "bb")]));
|
||
// Another content, a photo fewer or more (verified since the plan), the same photos in another order: another digest.
|
||
Assert.NotEqual(reference, Sequence(3, (a, "aa"), (b, "cc")).Digest());
|
||
Assert.NotEqual(reference, Sequence(3, (a, "aa")).Digest());
|
||
Assert.NotEqual(reference, Sequence(3, (a, "aa"), (b, "bb"), (Guid.NewGuid(), "dd")).Digest());
|
||
Assert.NotEqual(reference, Sequence(3, (b, "bb"), (a, "aa")).Digest());
|
||
// The plan's fingerprint pins the content too: as many photos of the same size holding other content is another plan.
|
||
ConvertPlan Plan(ConvertSequencePlan sequence) => new(Guid.Empty, new ConvertOptions(ConvertTarget.LinearDng), ConvertMode.Replace, [sequence], new ConvertSpace(0, null));
|
||
Assert.NotEqual(Plan(Sequence(3, (a, "aa"), (b, "bb"))).Fingerprint(), Plan(Sequence(3, (a, "aa"), (b, "cc"))).Fingerprint());
|
||
}
|
||
|
||
/// <summary>A provider that reads CR2 natively by format, and probes the files <paramref name="applies"/> picks (all
|
||
/// by default) at a fixed tier, the others native.</summary>
|
||
private sealed class ProbeAs(DecodeTier tier, Func<string, bool>? applies = null) : IImageDecoderProvider
|
||
{
|
||
public string Name => "probe";
|
||
|
||
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Cr2 ? DecodeTier.Native : null;
|
||
|
||
public FileTier Probe(string path, ImageFileFormat format) => new(format, applies?.Invoke(path) == false ? DecodeTier.Native : tier, Name, null, 0, 0);
|
||
|
||
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
|
||
}
|
||
|
||
/// <summary>WIC as a PC with every codec has it: JPEG, PNG and TIFF native, whatever they hold.</summary>
|
||
private sealed class ReadsEverything : IImageDecoderProvider
|
||
{
|
||
public string Name => "wic";
|
||
|
||
public DecodeTier? TierFor(ImageFileFormat format) => format is ImageFileFormat.Jpeg or ImageFileFormat.Png or ImageFileFormat.Tiff ? DecodeTier.Native : null;
|
||
|
||
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
|
||
}
|
||
|
||
/// <summary>WIC with the Raw Image Extension, as a conversion sees it: every camera RAW native, developed at 8 bits.</summary>
|
||
internal sealed class EightBitRaw : IImageDecoderProvider
|
||
{
|
||
public string Name => "wic";
|
||
|
||
public bool DevelopsRawAt8Bits => true;
|
||
|
||
public DecodeTier? TierFor(ImageFileFormat format) => ImageFormatInfo.Of(format).Family == ImageFamily.CameraRaw ? DecodeTier.Native : null;
|
||
|
||
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
|
||
}
|
||
}
|