using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using Chronos.Core.Archiving;
using Chronos.Core.Codecs.Icc;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Codecs.Tiff;
using Chronos.Core.Export;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Model;
namespace Chronos.Core.Conversion;
/// Why a frame of a sequence is not converted (owner review 2, R7); nothing of it is touched.
public enum ConvertSkip
{
/// The photo is in the target's format already (a TIFF of 8 bits included).
SameFormat,
///
/// This host has no full-resolution decoder for the photo's format (a server and a CR3 or HEIC), or its decoders do not
/// read this file at full resolution (a CMYK, CIELab or palette TIFF, a compression or size they do not take, a RAW body
/// they do not develop and without a preview): a PC may have one.
///
NoNativeDecoder,
/// This host reads the photo only from its embedded preview (a camera body its RAW decoder does not support).
PreviewOnly,
///
/// The photo is flagged unreadable, damaged or corrupt: no decoder produced its pixels. A flag set where other decoders
/// read the file (the server's, for a CMYK TIFF that WIC reads on a PC) holds only where a decode says so too.
///
Undecodable,
/// The photo's file is not there.
Missing,
/// The photo was never hashed (adopted, not verified yet): its replacement could not name it. Verify it first.
Unhashed,
/// A 16-bit target, and this host develops the RAW at 8 bits only (WIC's Raw Image Extension): JPEG converts it.
EightBit,
/// The photo's colour profile can be neither carried into the target nor converted to it (CMYK, an unknown space…).
ColorProfile,
///
/// The photo is larger than the target holds (a JPEG side above 65535) or than Chronos reads back: the server's decoders,
/// which every converted file must decode with ().
///
TooLarge,
///
/// The photo's file is there but could not be opened when the plan was made: another program holds it without sharing
/// (a sync client, an editor), or access is denied. A plan made once it is free converts it.
///
Inaccessible,
}
/// Why a whole sequence is not converted now; nothing of it is touched.
public enum ConvertBlock
{
/// A running or paused job works on the sequence or its whole archive, or a conversion of it waits in the queue.
Busy,
/// The archive is offline or read-only.
Offline,
/// The sequence's sidecar cannot be read, or belongs to another sequence.
Unreadable,
///
/// The photos the sequence converts are no longer those of the plan confirmed (found by the job, which then leaves it as
/// it is).
///
FramesChanged,
}
/// Frames of a sequence that are not converted, and why.
public sealed record ConvertSkipped(ConvertSkip Reason, int Frames);
///
/// One frame a conversion converts: its id and index, its file and format, its size, the converted file's estimated size,
/// and the SHA-256 of the content converted.
///
public sealed record ConvertFramePlan(Guid FrameId, int Index, string File, ImageFileFormat Format, long Size, long EstimatedBytes, string? Sha256 = null);
///
/// One sequence of a conversion plan: the frames converted, those skipped with why, the frames version the plan was made at,
/// and why the whole sequence cannot be converted now ().
///
public sealed record ConvertSequencePlan(Guid SequenceId, string Title, string Folder, int FramesVersion, IReadOnlyList Frames,
IReadOnlyList Skipped, IReadOnlyList Blocked)
{
/// Whether the sequence is converted: something to convert and nothing blocking it.
public bool Converts => Blocked.Count == 0 && Frames.Count > 0;
/// Size of the originals converted.
public long BytesBefore => Frames.Sum(f => f.Size);
/// Estimated size of the converted files.
public long BytesAfter => Frames.Sum(f => f.EstimatedBytes);
public int SkippedFrames => Skipped.Sum(s => s.Frames);
///
/// A digest of the photos the sequence converts: each frame in order with the SHA-256 of the content converted. A
/// conversion job converts the sequence only while its plan there still has the digest the user confirmed.
///
public string Digest() => DigestOf(Frames.Select(f => (f.FrameId, f.Sha256)));
/// The digest of frames, each with the SHA-256 of its content, in order ().
public static string DigestOf(IEnumerable<(Guid FrameId, string? Sha256)> frames)
{
ArgumentNullException.ThrowIfNull(frames);
var text = new StringBuilder();
foreach (var (id, sha) in frames) text.Append(CultureInfo.InvariantCulture, $"{id:N}|{sha?.ToLowerInvariant()}\n");
return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString())));
}
}
///
/// The free space a conversion needs on the archive's volume (: the estimate with a margin and a
/// reserve, ) and what it has; null when it cannot be measured.
///
public sealed record ConvertSpace(long Required, long? Free)
{
public bool Enough => Free is not { } free || free >= Required;
}
///
/// The dry run of a mass conversion (owner review 2, R7): the options and mode, each sequence's plan, the space it needs,
/// and the fingerprint the confirmation sends back ().
///
public sealed record ConvertPlan(Guid ArchiveId, ConvertOptions Options, ConvertMode Mode, IReadOnlyList Sequences, ConvertSpace Space)
{
/// The sequences that are converted.
public IEnumerable Converted => Sequences.Where(s => s.Converts);
public int Frames => Converted.Sum(s => s.Frames.Count);
public long BytesBefore => Converted.Sum(s => s.BytesBefore);
public long BytesAfter => Converted.Sum(s => s.BytesAfter);
public int Skipped => Sequences.Sum(s => s.SkippedFrames);
///
/// A digest of what the plan converts (options, mode, and each sequence in order with its frames version, the photos
/// converted with their content (), those skipped, and what blocks it), the
/// same for two plans of the same archive state.
///
public string Fingerprint()
{
var text = new StringBuilder().Append(Options.Key).Append('|').Append(Mode).Append('\n');
foreach (var s in Sequences)
{
text.Append(CultureInfo.InvariantCulture, $"{s.SequenceId:N}|{s.FramesVersion}|{s.Frames.Count}|{s.BytesBefore}|{s.Digest()}|");
foreach (var skipped in s.Skipped) text.Append(CultureInfo.InvariantCulture, $"{skipped.Reason}:{skipped.Frames};");
foreach (var blocked in s.Blocked) text.Append(blocked).Append(';');
text.Append('\n');
}
return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString())));
}
}
///
/// Plans the conversion of a sequence's frames (owner review 2, R7): a frame is converted when it is not in the target's
/// format already, its file is there, it was hashed, it is not flagged unreadable (or another host's decoders flagged it and
/// this host's decode it), its size is one the target holds and the server's decoders read back (),
/// this host decodes it at full resolution with the decoders the conversion uses (: its
/// format has a native decoder here and the file itself probes native now, whatever tier another host or an earlier pass
/// recorded, its headers saying what those decoders take; for a 16-bit target, not through a RAW development at 8 bits),
/// and its colour profile can be kept (). The plan asks the files what the conversion
/// will ask them, reading headers only (and decoding small the few frames flagged elsewhere), several at once; a file it
/// cannot read now is inaccessible, never a verdict on its content. Sizes after are estimated from the pixels
/// ().
///
public static class ConvertPlanner
{
/// Files asked at once while a sequence is planned: header reads, bound by the disk more than the cores.
private const int ProbesAtOnce = 8;
/// Long edge of the decode that asks a frame flagged unreadable elsewhere whether this host's decoders read it.
private const int ConfirmEdge = 64;
/// Margin over the estimate the free space must hold (estimates of compressed files vary with the content).
public const double SpaceMargin = 1.15;
/// Space left free on the volume after the conversion, for the index, sidecars and everything else writing there.
public const long SpaceReserve = 512L << 20;
/// Pixels assumed for a frame whose size is not known (a 24 MP sensor).
public const long UnknownPixels = 24_000_000;
///
/// Bits per pixel of a 4:4:4 JPEG over a 4:2:0 one of the same quality: the chroma is coded at full resolution, about a
/// quarter more bits on photographs.
///
private const double FullChroma = 1.25;
///
/// Bits per pixel of pixels that do not compress at all (noise) in the costliest target: three 16-bit samples, 48 bits
/// as Deflate stores them, and less than a bit more each as lossless JPEG codes them (an optimal Huffman code exceeds
/// the entropy of what it codes by less than one bit); a 4:4:4 JPEG of noise at quality 100 takes about 22.
///
public const int WorstBitsPerPixel = 51;
///
/// Margin on : Deflate's stored blocks and zlib framing per strip, lossless JPEG's byte
/// stuffing (a 0xFF byte in 256 of noise) and headers per tile stay below 1 %.
///
private const double WorstMargin = 1.01;
/// What a converted file holds beyond its pixels at most: an ICC profile carried whole, the Exif block, the directory and its strip or tile tables.
private const long WorstOverhead = IccProfile.MaxBytes + (4L << 20);
/// The plan of one sequence from its sidecar.
/// Every file in the folders of the sequence's frames, relative to the sequence folder
/// (): the frames whose file is not among them are missing.
/// This host's decoders.
public static ConvertSequencePlan Plan(SequenceManifest manifest, SequencePaths folder, IReadOnlyCollection present, ConvertOptions options,
ImageDecoder decoder, string title)
{
ArgumentNullException.ThrowIfNull(manifest);
ArgumentNullException.ThrowIfNull(folder);
ArgumentNullException.ThrowIfNull(present);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(decoder);
var onDisk = present as IReadOnlySet ?? new HashSet(present, StringComparer.OrdinalIgnoreCase);
var reasons = new ConvertSkip?[manifest.Frames.Count];
Parallel.For(0, reasons.Length, new ParallelOptions { MaxDegreeOfParallelism = ProbesAtOnce },
i => reasons[i] = Skip(manifest.Frames[i], folder, onDisk, options, decoder));
var skipped = new Dictionary();
var frames = new List();
for (var i = 0; i < reasons.Length; i++)
{
var frame = manifest.Frames[i];
if (reasons[i] is { } reason)
{
skipped[reason] = skipped.GetValueOrDefault(reason) + 1;
continue;
}
frames.Add(new ConvertFramePlan(frame.Id, frame.I, frame.File, frame.Format, frame.Size, EstimateBytes(options, frame.W, frame.H), frame.Sha256));
}
return new ConvertSequencePlan(manifest.Id, title, folder.RelativeFolder, manifest.Versions.Frames, frames,
[.. skipped.OrderBy(s => s.Key).Select(s => new ConvertSkipped(s.Key, s.Value))], []);
}
/// Why is not converted, or null when it is.
/// This host's decoders; the conversion's are made from them ().
public static ConvertSkip? Skip(SidecarFrame frame, SequencePaths folder, IReadOnlySet present, ConvertOptions options, ImageDecoder decoder)
{
ArgumentNullException.ThrowIfNull(frame);
ArgumentNullException.ThrowIfNull(folder);
ArgumentNullException.ThrowIfNull(present);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(decoder);
if (frame.Format == options.Format) return ConvertSkip.SameFormat;
if ((frame.Flags & FrameFlags.Missing) != 0 || !present.Contains(frame.File)) return ConvertSkip.Missing;
if (frame.Sha256 is null) return ConvertSkip.Unhashed;
if ((frame.Flags & FrameFlags.Corrupt) != 0) return ConvertSkip.Undecodable;
var decoders = FrameConverter.Decoders(decoder, options, frame.Format);
// Flagged unreadable by the host that analysed it last: on a server, whose managed decoders refuse what they do not
// read (a CMYK or CIELab TIFF), that holds for a PC only where its own decoders (WIC) do not read it either.
var flagged = (frame.Flags & FrameFlags.Undecodable) != 0;
if (flagged && !OthersRead(decoders, frame.Format)) return ConvertSkip.Undecodable;
if (TooLarge(options, frame.W, frame.H)) return ConvertSkip.TooLarge;
var eightBit = FrameConverter.OnlyAt8Bits(decoder, options, frame.Format);
if (decoders.TierFor(frame.Format) != DecodeTier.Native) return eightBit ? ConvertSkip.EightBit : ConvertSkip.NoNativeDecoder;
// Whatever tier was recorded (unknown before the light analysis, or by another host with other decoders, or for a
// body this host's RAW decoder does not support), the file itself is asked: headers only, and nothing at all for the
// formats every file of which decodes alike.
var path = folder.File(frame.File);
try
{
// Not read at full resolution here, although the format is: what the file holds (its encoding, its size, a RAW
// body) is beyond these decoders, and only a preview, or nothing, is left.
var probe = decoders.Probe(path, frame.Format);
if (probe.Tier != DecodeTier.Native)
{
return flagged ? ConvertSkip.Undecodable
: eightBit ? ConvertSkip.EightBit
: probe.Tier == DecodeTier.EmbeddedPreview ? ConvertSkip.PreviewOnly
: ConvertSkip.NoNativeDecoder;
}
if (TooLarge(options, probe.Width, probe.Height)) return ConvertSkip.TooLarge;
// The flag was another host's verdict, whose decoders failed where the headers say nothing: a decode here says
// whether these read its pixels.
if (flagged && !Decodes(decoders, path)) return ConvertSkip.Undecodable;
return FrameConverter.Treatment(SourceColor.Read(path, frame.Format), options.Target) == ColorTreatment.Refuse ? ConvertSkip.ColorProfile : null;
}
catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
{
// Gone since the folder was listed.
return ConvertSkip.Missing;
}
catch (EndOfStreamException)
{
// Opened, and shorter than its own headers say: no decoder gets its pixels.
return ConvertSkip.Undecodable;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// There, but held by another program or denied, or a read failed: not missing, and not planned (a conversion job
// converts only the photos its plan listed), so the dry run says why. Nothing is concluded of its content.
return ConvertSkip.Inaccessible;
}
}
///
/// Whether have, for , a native decoder besides the managed ones
/// every host has (WIC on a PC): a frame those flagged unreadable may still decode here.
///
private static bool OthersRead(ImageDecoder decoders, ImageFileFormat format) =>
decoders.Providers.Any(p => p is not ManagedDecoderProvider && p.TierFor(format) == DecodeTier.Native);
///
/// Whether produce the pixels of the file at at full-resolution tier,
/// asked with a small decode.
///
/// The file cannot be read.
private static bool Decodes(ImageDecoder decoders, string path)
{
using var decoded = decoders.Decode(new DecodeRequest(path, ConfirmEdge, DecodePurpose.Thumbnail, ApplyOrientation: false, AllowPreview: false));
return decoded is { Tier: DecodeTier.Native, ByChoice: false };
}
///
/// Whether a frame of × (unknown: 0) cannot be converted to
/// , a bound and never an estimate: the target does not hold it (a JPEG side above
/// ), Chronos does not read it back (the server's decoders, which every converted
/// file must decode with, : a JPEG of more pixels than
/// , a TIFF-16 or DNG of more samples than ),
/// or its file could outgrow the array it is made in (, which those sizes keep below it). What it
/// lets through fits one image buffer, its file the array it is made in, and the server's decoders read it back. Any
/// size is answered, a damaged header's included: nothing is multiplied beyond what a long holds.
///
public static bool TooLarge(ConvertOptions options, int width, int height)
{
ArgumentNullException.ThrowIfNull(options);
if (width <= 0 || height <= 0) return false;
// At most (2^31 - 1)^2, below 2^62.
var pixels = (long)width * height;
var readBack = options.Target == ConvertTarget.Jpeg
? Math.Max(width, height) <= FrameConverter.MaxJpegSide && pixels <= JpegDecodeOptions.Default.MaxPixels
: pixels <= TiffDecodeOptions.Default.MaxSamples / 3;
return !readBack || WorstBytes(width, height) > Array.MaxLength;
}
///
/// The largest a converted file of × pixels can be when its pixels do
/// not compress at all, in any target: with its margin, and the most a file holds beyond
/// its pixels; for sizes beyond it.
///
public static long WorstBytes(int width, int height) =>
Saturated(Math.Ceiling((double)Math.Max(0, width) * Math.Max(0, height) * WorstBitsPerPixel / 8 * WorstMargin) + WorstOverhead);
/// (whole, not negative) as a long, when beyond it.
private static long Saturated(double bytes) => bytes >= long.MaxValue ? long.MaxValue : (long)bytes;
///
/// Estimated size of a converted file of × pixels (unknown:
/// ), from the bits per pixel the Export tab estimates with ():
/// JPEG by quality (×1.25 for 4:4:4 chroma), TIFF-16 30, linear DNG 26.
///
public static long EstimateBytes(ConvertOptions options, int width, int height)
{
ArgumentNullException.ThrowIfNull(options);
var pixels = width > 0 && height > 0 ? (double)width * height : UnknownPixels;
var bits = options.Target switch
{
ConvertTarget.Jpeg => ExportEstimator.JpegBitsPerPixel(options.Quality) * FullChroma,
ConvertTarget.Tiff16 => 30,
_ => 26,
};
return Saturated(Math.Ceiling(pixels * bits / 8));
}
/// The free space a conversion whose files are estimated at bytes needs.
public static long Required(long estimated) => (long)Math.Ceiling(Math.Max(0, estimated) * SpaceMargin) + SpaceReserve;
}