From bb7bc6490dfd1b969a1edacca4a0b8846d62e6fc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 09:24:38 +0000 Subject: [PATCH 1/9] feat(video): add bitstream readers for Annex-B, H.264/HEVC SPS and AV1 OBUs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../Video/Bitstream/AnnexBParser.cs | 214 ++++++++++++ .../Video/Bitstream/Av1ObuParser.cs | 321 +++++++++++++++++ src/Titano.Core/Video/Bitstream/BitReader.cs | 113 ++++++ src/Titano.Core/Video/Bitstream/BitWriter.cs | 80 +++++ .../Video/Bitstream/H264SpsParser.cs | 230 +++++++++++++ .../Video/Bitstream/HevcSpsParser.cs | 120 +++++++ .../Titano.Core.Tests/Video/BitstreamTests.cs | 229 +++++++++++++ .../Video/SyntheticStreams.cs | 323 ++++++++++++++++++ 8 files changed, 1630 insertions(+) create mode 100644 src/Titano.Core/Video/Bitstream/AnnexBParser.cs create mode 100644 src/Titano.Core/Video/Bitstream/Av1ObuParser.cs create mode 100644 src/Titano.Core/Video/Bitstream/BitReader.cs create mode 100644 src/Titano.Core/Video/Bitstream/BitWriter.cs create mode 100644 src/Titano.Core/Video/Bitstream/H264SpsParser.cs create mode 100644 src/Titano.Core/Video/Bitstream/HevcSpsParser.cs create mode 100644 tests/Titano.Core.Tests/Video/BitstreamTests.cs create mode 100644 tests/Titano.Core.Tests/Video/SyntheticStreams.cs diff --git a/src/Titano.Core/Video/Bitstream/AnnexBParser.cs b/src/Titano.Core/Video/Bitstream/AnnexBParser.cs new file mode 100644 index 00000000..3d43665e --- /dev/null +++ b/src/Titano.Core/Video/Bitstream/AnnexBParser.cs @@ -0,0 +1,214 @@ +using System.Buffers.Binary; + +namespace Titano.Core.Video.Bitstream; + +/// H.264 nal_unit_type values (ITU-T H.264 Table 7-1). +public enum H264NalUnitType : byte +{ + Unspecified = 0, + NonIdrSlice = 1, + SliceDataPartitionA = 2, + SliceDataPartitionB = 3, + SliceDataPartitionC = 4, + IdrSlice = 5, + Sei = 6, + Sps = 7, + Pps = 8, + AccessUnitDelimiter = 9, + EndOfSequence = 10, + EndOfStream = 11, + FillerData = 12, + SpsExtension = 13, + PrefixNal = 14, + SubsetSps = 15, +} + +/// HEVC nal_unit_type values (ITU-T H.265 Table 7-1). +public enum HevcNalUnitType : byte +{ + TrailN = 0, + TrailR = 1, + TsaN = 2, + TsaR = 3, + StsaN = 4, + StsaR = 5, + RadlN = 6, + RadlR = 7, + RaslN = 8, + RaslR = 9, + BlaWLp = 16, + BlaWRadl = 17, + BlaNLp = 18, + IdrWRadl = 19, + IdrNLp = 20, + Cra = 21, + Vps = 32, + Sps = 33, + Pps = 34, + AccessUnitDelimiter = 35, + EndOfSequence = 36, + EndOfBitstream = 37, + FillerData = 38, + PrefixSei = 39, + SuffixSei = 40, +} + +/// Location of one NAL unit (header included, start code or length prefix excluded) inside a buffer. +public readonly record struct NalUnit(int Offset, int Length) +{ + public ReadOnlySpan Slice(ReadOnlySpan buffer) => buffer.Slice(Offset, Length); +} + +/// Annex-B byte streams and ISO/IEC 14496-15 length-prefixed samples of H.264/HEVC NAL units. +public static class AnnexBParser +{ + private static ReadOnlySpan StartCode => [0, 0, 1]; + + /// True when the data begins with a 3- or 4-byte start code. + public static bool StartsWithStartCode(ReadOnlySpan data) => + data.Length >= 3 && data[0] == 0 && data[1] == 0 && (data[2] == 1 || (data.Length >= 4 && data[2] == 0 && data[3] == 1)); + + /// + /// Splits an Annex-B stream at 3/4-byte start codes. Bytes before the first start code are ignored and + /// trailing zero bytes (trailing_zero_8bits, the leading zero of a 4-byte start code) are not part of a unit. + /// + public static List Split(ReadOnlySpan data) + { + var units = new List(); + var first = data.IndexOf(StartCode); + if (first < 0) return units; + var start = first + 3; + while (true) + { + var next = data[start..].IndexOf(StartCode); + var end = next < 0 ? data.Length : start + next; + var nalEnd = end; + while (nalEnd > start && data[nalEnd - 1] == 0) nalEnd--; + if (nalEnd > start) units.Add(new NalUnit(start, nalEnd - start)); + if (next < 0) return units; + start = end + 3; + } + } + + /// + /// Splits big-endian length-prefixed NAL units. Returns false (with the units read so far) when a length + /// is zero, runs past the end or the buffer does not end exactly after the last unit. + /// + public static bool TrySplitLengthPrefixed(ReadOnlySpan data, int lengthSize, out List units) + { + if (lengthSize is not (1 or 2 or 4)) throw new ArgumentOutOfRangeException(nameof(lengthSize)); + units = []; + var pos = 0; + while (pos < data.Length) + { + if (data.Length - pos < lengthSize) return false; + var length = lengthSize switch + { + 1 => data[pos], + 2 => BinaryPrimitives.ReadUInt16BigEndian(data[pos..]), + _ => (long)BinaryPrimitives.ReadUInt32BigEndian(data[pos..]), + }; + pos += lengthSize; + if (length == 0 || length > data.Length - pos) return false; + units.Add(new NalUnit(pos, (int)length)); + pos += (int)length; + } + return units.Count > 0; + } + + /// + /// Splits one access unit given either with 4-byte length prefixes or as Annex-B. The length-prefixed reading + /// wins only when it covers the buffer exactly with valid NAL headers, which an Annex-B stream practically never does. + /// + public static List SplitAccessUnit(ReadOnlySpan data, out bool wasAnnexB) + { + if (TrySplitLengthPrefixed(data, 4, out var units) && HeadersValid(data, units)) + { + wasAnnexB = false; + return units; + } + wasAnnexB = true; + return Split(data); + } + + private static bool HeadersValid(ReadOnlySpan data, List units) + { + foreach (var u in units) + { + if ((data[u.Offset] & 0x80) != 0) return false; // forbidden_zero_bit + } + return true; + } + + /// Removes emulation_prevention_three_byte (00 00 03 → 00 00) to obtain the RBSP. + public static byte[] RemoveEmulationPrevention(ReadOnlySpan nal) + { + var output = new byte[nal.Length]; + var n = 0; + var zeros = 0; + foreach (var b in nal) + { + if (zeros >= 2 && b == 3) + { + zeros = 0; + continue; + } + output[n++] = b; + zeros = b == 0 ? zeros + 1 : 0; + } + return n == output.Length ? output : output.AsSpan(0, n).ToArray(); + } + + /// Inserts emulation prevention bytes so the payload contains no start-code prefix. + public static byte[] AddEmulationPrevention(ReadOnlySpan rbsp) + { + var output = new List(rbsp.Length + rbsp.Length / 64 + 2); + var zeros = 0; + foreach (var b in rbsp) + { + if (zeros >= 2 && b <= 3) + { + output.Add(3); + zeros = 0; + } + output.Add(b); + zeros = b == 0 ? zeros + 1 : 0; + } + // A payload ending in 0x00 (cabac_zero_word) gets a final 0x03 (H.264 7.4.1). + if (rbsp.Length > 0 && rbsp[^1] == 0) output.Add(3); + return [.. output]; + } + + public static H264NalUnitType H264Type(byte header) => (H264NalUnitType)(header & 0x1F); + + public static HevcNalUnitType HevcType(byte header) => (HevcNalUnitType)((header >> 1) & 0x3F); + + /// Converts an Annex-B stream to 4-byte big-endian length prefixes. + public static byte[] ToLengthPrefixed(ReadOnlySpan annexB) + { + var units = Split(annexB); + var total = 0; + foreach (var u in units) total += 4 + u.Length; + var output = new byte[total]; + var pos = 0; + foreach (var u in units) + { + BinaryPrimitives.WriteInt32BigEndian(output.AsSpan(pos), u.Length); + annexB.Slice(u.Offset, u.Length).CopyTo(output.AsSpan(pos + 4)); + pos += 4 + u.Length; + } + return output; + } + + /// Concatenates NAL units with 4-byte start codes. + public static byte[] ToAnnexB(IEnumerable nalUnits) + { + using var ms = new MemoryStream(); + foreach (var nal in nalUnits) + { + ms.Write([0, 0, 0, 1]); + ms.Write(nal); + } + return ms.ToArray(); + } +} diff --git a/src/Titano.Core/Video/Bitstream/Av1ObuParser.cs b/src/Titano.Core/Video/Bitstream/Av1ObuParser.cs new file mode 100644 index 00000000..ad5a2c8f --- /dev/null +++ b/src/Titano.Core/Video/Bitstream/Av1ObuParser.cs @@ -0,0 +1,321 @@ +namespace Titano.Core.Video.Bitstream; + +/// AV1 obu_type values (AV1 specification 6.2.2). +public enum Av1ObuType : byte +{ + SequenceHeader = 1, + TemporalDelimiter = 2, + FrameHeader = 3, + TileGroup = 4, + Metadata = 5, + Frame = 6, + RedundantFrameHeader = 7, + TileList = 8, + Padding = 15, +} + +/// One OBU inside a buffer: header (with extension and size field) followed by the payload. +public readonly record struct Av1Obu(Av1ObuType Type, int Offset, int HeaderLength, int PayloadLength, bool HasSizeField, bool HasExtension, byte TemporalId, byte SpatialId) +{ + public int TotalLength => HeaderLength + PayloadLength; + + public ReadOnlySpan Payload(ReadOnlySpan buffer) => buffer.Slice(Offset + HeaderLength, PayloadLength); +} + +/// Sequence header fields needed by av1C, level checks and diagnostics. +public sealed record Av1SequenceHeader +{ + public required int SeqProfile { get; init; } + public required bool StillPicture { get; init; } + public required bool ReducedStillPictureHeader { get; init; } + public required int SeqLevelIdx { get; init; } + public required bool SeqTier { get; init; } + public required bool HighBitDepth { get; init; } + public required bool TwelveBit { get; init; } + public required int BitDepth { get; init; } + public required bool MonoChrome { get; init; } + public required bool ChromaSubsamplingX { get; init; } + public required bool ChromaSubsamplingY { get; init; } + public required int ChromaSamplePosition { get; init; } + public required int MaxFrameWidth { get; init; } + public required int MaxFrameHeight { get; init; } + public required int ColorPrimaries { get; init; } + public required int TransferCharacteristics { get; init; } + public required int MatrixCoefficients { get; init; } + public required bool FullRange { get; init; } + + /// initial_display_delay_minus_1 + 1 of operating point 0, when signalled. + public int? InitialDisplayDelay { get; init; } +} + +/// AV1 low-overhead bitstream (OBU) parsing. Never throws on malformed input. +public static class Av1ObuParser +{ + /// Reads the OBU at ; one without obu_size extends to the end of the buffer. + public static bool TryReadObu(ReadOnlySpan data, int offset, out Av1Obu obu) + { + obu = default; + if (offset < 0 || offset >= data.Length) return false; + var header = data[offset]; + if ((header & 0x80) != 0) return false; // obu_forbidden_bit + var type = (Av1ObuType)((header >> 3) & 0x0F); + var hasExtension = (header & 0x04) != 0; + var hasSize = (header & 0x02) != 0; + var pos = offset + 1; + byte temporalId = 0, spatialId = 0; + if (hasExtension) + { + if (pos >= data.Length) return false; + temporalId = (byte)(data[pos] >> 5); + spatialId = (byte)((data[pos] >> 3) & 3); + pos++; + } + long payload; + if (hasSize) + { + if (!TryReadLeb128(data[pos..], out var size, out var sizeLength)) return false; + pos += sizeLength; + payload = (long)Math.Min(size, int.MaxValue); + } + else + { + payload = data.Length - pos; + } + if (payload > data.Length - pos) return false; + obu = new Av1Obu(type, offset, pos - offset, (int)payload, hasSize, hasExtension, temporalId, spatialId); + return true; + } + + /// Splits a buffer into OBUs, stopping at the first malformed one. + public static List Split(ReadOnlySpan data) + { + var list = new List(); + var pos = 0; + while (pos < data.Length && TryReadObu(data, pos, out var obu)) + { + list.Add(obu); + pos += obu.TotalLength; + } + return list; + } + + /// Splits a low-overhead stream into temporal units, each starting at a temporal delimiter OBU. + public static List<(int Offset, int Length)> SplitTemporalUnits(ReadOnlySpan stream) + { + var units = new List<(int Offset, int Length)>(); + var start = -1; + var end = 0; + foreach (var obu in Split(stream)) + { + if (obu.Type == Av1ObuType.TemporalDelimiter) + { + if (start >= 0) units.Add((start, obu.Offset - start)); + start = obu.Offset; + } + else if (start < 0) + { + start = obu.Offset; // stream without a leading delimiter + } + end = obu.Offset + obu.TotalLength; + } + if (start >= 0 && end > start) units.Add((start, end - start)); + return units; + } + + public static bool TryReadLeb128(ReadOnlySpan data, out ulong value, out int length) + { + value = 0; + for (length = 0; length < 8 && length < data.Length; length++) + { + var b = data[length]; + value |= (ulong)(b & 0x7F) << (7 * length); + if ((b & 0x80) == 0) + { + length++; + return value <= uint.MaxValue; + } + } + return false; + } + + public static int Leb128Size(ulong value) + { + var n = 1; + while ((value >>= 7) != 0) n++; + return n; + } + + public static int WriteLeb128(Span destination, ulong value) + { + var n = 0; + do + { + var b = (byte)(value & 0x7F); + value >>= 7; + destination[n++] = value != 0 ? (byte)(b | 0x80) : b; + } + while (value != 0); + return n; + } + + /// Parses a sequence_header_obu() payload (AV1 5.5). + public static bool TryParseSequenceHeader(ReadOnlySpan payload, out Av1SequenceHeader header) + { + header = null!; + var r = new BitReader(payload); + var profile = (int)r.ReadBits(3); + if (profile > 2) return false; + var still = r.ReadFlag(); + var reduced = r.ReadFlag(); + int levelIdx; + var tier = false; + int? initialDelay = null; + if (reduced) + { + levelIdx = (int)r.ReadBits(5); + } + else + { + var decoderModelInfo = false; + var bufferDelayLength = 0; + if (r.ReadFlag()) + { + r.ReadBits(32); // num_units_in_display_tick + r.ReadBits(32); // time_scale + if (r.ReadFlag()) r.ReadUvlc(); // equal_picture_interval, num_ticks_per_picture_minus_1 + decoderModelInfo = r.ReadFlag(); + if (decoderModelInfo) + { + bufferDelayLength = (int)r.ReadBits(5) + 1; + r.ReadBits(32); // num_units_in_decoding_tick + r.ReadBits(5); // buffer_removal_time_length_minus_1 + r.ReadBits(5); // frame_presentation_time_length_minus_1 + } + } + var initialDisplayDelayPresent = r.ReadFlag(); + var operatingPoints = (int)r.ReadBits(5) + 1; + levelIdx = 0; + for (var i = 0; i < operatingPoints && !r.IsOverrun; i++) + { + r.ReadBits(12); // operating_point_idc + var level = (int)r.ReadBits(5); + var opTier = level > 7 && r.ReadFlag(); + if (decoderModelInfo && r.ReadFlag()) + { + r.Skip(bufferDelayLength * 2L + 1); // decoder/encoder buffer delay, low_delay_mode_flag + } + int? delay = null; + if (initialDisplayDelayPresent && r.ReadFlag()) delay = (int)r.ReadBits(4) + 1; + if (i == 0) + { + levelIdx = level; + tier = opTier; + initialDelay = delay; + } + } + } + + var widthBits = (int)r.ReadBits(4) + 1; + var heightBits = (int)r.ReadBits(4) + 1; + var maxWidth = (int)r.ReadBits(widthBits) + 1; + var maxHeight = (int)r.ReadBits(heightBits) + 1; + if (!reduced && r.ReadFlag()) + { + r.ReadBits(4); // delta_frame_id_length_minus_2 + r.ReadBits(3); // additional_frame_id_length_minus_1 + } + r.ReadBits(3); // use_128x128_superblock, enable_filter_intra, enable_intra_edge_filter + if (!reduced) + { + r.ReadBits(4); // interintra, masked compound, warped motion, dual filter + var orderHint = r.ReadFlag(); + if (orderHint) r.ReadBits(2); // enable_jnt_comp, enable_ref_frame_mvs + var forceScreenContent = r.ReadFlag() ? 2 : (int)r.ReadBits(1); + if (forceScreenContent > 0 && !r.ReadFlag()) r.ReadBits(1); // seq_force_integer_mv + if (orderHint) r.ReadBits(3); // order_hint_bits_minus_1 + } + r.ReadBits(3); // enable_superres, enable_cdef, enable_restoration + + // color_config() + var highBitDepth = r.ReadFlag(); + var twelveBit = false; + var bitDepth = 8; + if (profile == 2 && highBitDepth) + { + twelveBit = r.ReadFlag(); + bitDepth = twelveBit ? 12 : 10; + } + else if (highBitDepth) + { + bitDepth = 10; + } + var mono = profile != 1 && r.ReadFlag(); + int primaries = 2, transfer = 2, matrix = 2; + if (r.ReadFlag()) + { + primaries = (int)r.ReadBits(8); + transfer = (int)r.ReadBits(8); + matrix = (int)r.ReadBits(8); + } + bool fullRange, subX, subY; + var samplePosition = 0; + if (mono) + { + fullRange = r.ReadFlag(); + subX = subY = true; + } + else if (primaries == 1 && transfer == 13 && matrix == 0) + { + fullRange = true; + subX = subY = false; + } + else + { + fullRange = r.ReadFlag(); + if (profile == 0) + { + subX = subY = true; + } + else if (profile == 1) + { + subX = subY = false; + } + else if (bitDepth == 12) + { + subX = r.ReadFlag(); + subY = subX && r.ReadFlag(); + } + else + { + subX = true; + subY = false; + } + if (subX && subY) samplePosition = (int)r.ReadBits(2); + } + if (r.IsOverrun) return false; + + header = new Av1SequenceHeader + { + SeqProfile = profile, + StillPicture = still, + ReducedStillPictureHeader = reduced, + SeqLevelIdx = levelIdx, + SeqTier = tier, + HighBitDepth = highBitDepth, + TwelveBit = twelveBit, + BitDepth = bitDepth, + MonoChrome = mono, + ChromaSubsamplingX = subX, + ChromaSubsamplingY = subY, + ChromaSamplePosition = samplePosition, + MaxFrameWidth = maxWidth, + MaxFrameHeight = maxHeight, + ColorPrimaries = primaries, + TransferCharacteristics = transfer, + MatrixCoefficients = matrix, + FullRange = fullRange, + InitialDisplayDelay = initialDelay, + }; + return true; + } +} diff --git a/src/Titano.Core/Video/Bitstream/BitReader.cs b/src/Titano.Core/Video/Bitstream/BitReader.cs new file mode 100644 index 00000000..db70c16a --- /dev/null +++ b/src/Titano.Core/Video/Bitstream/BitReader.cs @@ -0,0 +1,113 @@ +namespace Titano.Core.Video.Bitstream; + +/// +/// MSB-first bit reader over an RBSP (emulation prevention already removed). Reading past the end yields zero +/// bits and sets , so parsers finish a pass over truncated input and reject it afterwards +/// instead of throwing. +/// +public ref struct BitReader +{ + private readonly ReadOnlySpan _data; + private long _position; + + public BitReader(ReadOnlySpan data) + { + _data = data; + _position = 0; + IsOverrun = false; + } + + /// Bit offset from the start of the data. + public readonly long Position => _position; + + public readonly long BitsRemaining => Math.Max(0, (long)_data.Length * 8 - _position); + + public readonly bool IsByteAligned => (_position & 7) == 0; + + /// True once a read went past the end or an Exp-Golomb code exceeded 32 bits. + public bool IsOverrun { get; private set; } + + public uint ReadBit() + { + var index = _position >> 3; + if (index >= _data.Length) + { + IsOverrun = true; + _position++; + return 0; + } + var bit = (uint)(_data[(int)index] >> (7 - (int)(_position & 7))) & 1; + _position++; + return bit; + } + + public bool ReadFlag() => ReadBit() != 0; + + /// Reads 0–32 bits, most significant first. + public uint ReadBits(int count) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan((uint)count, 32u, nameof(count)); + uint value = 0; + // Whole bytes when aligned keep long fields (32-bit timing values) cheap. + while (count >= 8 && IsByteAligned && (_position >> 3) < _data.Length) + { + value = (value << 8) | _data[(int)(_position >> 3)]; + _position += 8; + count -= 8; + } + for (var i = 0; i < count; i++) value = (value << 1) | ReadBit(); + return value; + } + + /// Reads 0–64 bits, most significant first. + public ulong ReadBits64(int count) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan((uint)count, 64u, nameof(count)); + if (count <= 32) return ReadBits(count); + var high = (ulong)ReadBits(count - 32); + return (high << 32) | ReadBits(32); + } + + public void Skip(long bits) + { + _position += bits; + if (_position > (long)_data.Length * 8) IsOverrun = true; + } + + public void AlignToByte() => _position = (_position + 7) & ~7L; + + /// Unsigned Exp-Golomb ue(v). + public uint ReadUe() + { + var zeros = 0; + while (ReadBit() == 0) + { + if (++zeros > 31 || IsOverrun) + { + IsOverrun = true; + return 0; + } + } + return zeros == 0 ? 0 : (uint)((1UL << zeros) - 1 + ReadBits(zeros)); + } + + /// Signed Exp-Golomb se(v). + public int ReadSe() + { + var k = ReadUe(); + return (k & 1) != 0 ? (int)((k + 1) / 2) : -(int)(k / 2); + } + + /// AV1 uvlc(): returns for 32 or more leading zeros. + public uint ReadUvlc() + { + var zeros = 0; + while (ReadBit() == 0) + { + if (IsOverrun) return 0; + zeros++; + } + if (zeros >= 32) return uint.MaxValue; + return ReadBits(zeros) + (uint)((1UL << zeros) - 1); + } +} diff --git a/src/Titano.Core/Video/Bitstream/BitWriter.cs b/src/Titano.Core/Video/Bitstream/BitWriter.cs new file mode 100644 index 00000000..258a8731 --- /dev/null +++ b/src/Titano.Core/Video/Bitstream/BitWriter.cs @@ -0,0 +1,80 @@ +using System.Numerics; + +namespace Titano.Core.Video.Bitstream; + +/// MSB-first bit writer with Exp-Golomb and AV1 leb128 helpers (used to synthesise parameter sets). +public sealed class BitWriter +{ + private byte[] _buffer = new byte[64]; + private long _bitLength; + + public long BitLength => _bitLength; + + public bool IsByteAligned => (_bitLength & 7) == 0; + + public void WriteBit(bool bit) => WriteBits(bit ? 1UL : 0UL, 1); + + /// Writes the low bits (0–64) of , most significant first. + public void WriteBits(ulong value, int count) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan((uint)count, 64u, nameof(count)); + EnsureCapacity(_bitLength + count); + for (var i = count - 1; i >= 0; i--) + { + if (((value >> i) & 1) != 0) _buffer[_bitLength >> 3] |= (byte)(0x80 >> (int)(_bitLength & 7)); + _bitLength++; + } + } + + /// Unsigned Exp-Golomb ue(v); values up to 2³² − 2 are readable by . + public void WriteUe(uint value) + { + var v = (ulong)value + 1; + var bits = 64 - BitOperations.LeadingZeroCount(v); + WriteBits(0, bits - 1); + WriteBits(v, bits); + } + + /// Signed Exp-Golomb se(v). + public void WriteSe(int value) => WriteUe(value > 0 ? (uint)(2L * value - 1) : (uint)(-2L * value)); + + /// rbsp_trailing_bits(): a stop bit followed by zero bits up to the byte boundary. + public void WriteTrailingBits() + { + WriteBit(true); + AlignWithZeros(); + } + + public void AlignWithZeros() + { + var pad = (int)((8 - (_bitLength & 7)) & 7); + WriteBits(0, pad); + } + + public void WriteBytes(ReadOnlySpan bytes) + { + foreach (var b in bytes) WriteBits(b, 8); + } + + /// AV1 leb128() (byte aligned). + public void WriteLeb128(ulong value) + { + do + { + var b = (byte)(value & 0x7F); + value >>= 7; + WriteBits(value != 0 ? b | 0x80u : b, 8); + } + while (value != 0); + } + + /// The written bytes; a partial last byte is zero padded. + public byte[] ToArray() => _buffer.AsSpan(0, (int)((_bitLength + 7) >> 3)).ToArray(); + + private void EnsureCapacity(long bits) + { + var bytes = (bits + 7) >> 3; + if (bytes <= _buffer.Length) return; + Array.Resize(ref _buffer, (int)Math.Max(bytes, _buffer.Length * 2L)); + } +} diff --git a/src/Titano.Core/Video/Bitstream/H264SpsParser.cs b/src/Titano.Core/Video/Bitstream/H264SpsParser.cs new file mode 100644 index 00000000..87924f4d --- /dev/null +++ b/src/Titano.Core/Video/Bitstream/H264SpsParser.cs @@ -0,0 +1,230 @@ +namespace Titano.Core.Video.Bitstream; + +/// Fields of an H.264 sequence parameter set needed for containers and diagnostics. +public sealed record H264Sps +{ + public required int ProfileIdc { get; init; } + + /// constraint_set0..5 flags plus the two reserved bits, as the byte stored in avcC. + public required byte ConstraintFlags { get; init; } + + public required int LevelIdc { get; init; } + public required int SpsId { get; init; } + public required int ChromaFormatIdc { get; init; } + public required int BitDepthLuma { get; init; } + public required int BitDepthChroma { get; init; } + + /// Displayed width after frame cropping. + public required int Width { get; init; } + + /// Displayed height after frame cropping. + public required int Height { get; init; } + + public required int CodedWidth { get; init; } + public required int CodedHeight { get; init; } + public required bool FrameMbsOnly { get; init; } + public required int PicOrderCntType { get; init; } + public required int MaxNumRefFrames { get; init; } + public bool HasVui { get; init; } + public int SarWidth { get; init; } = 1; + public int SarHeight { get; init; } = 1; + public bool? FullRange { get; init; } + public int? ColourPrimaries { get; init; } + public int? TransferCharacteristics { get; init; } + public int? MatrixCoefficients { get; init; } + + /// VUI num_units_in_tick (0 when timing info is absent). + public uint NumUnitsInTick { get; init; } + + /// VUI time_scale (0 when timing info is absent). + public uint TimeScale { get; init; } + + public bool FixedFrameRate { get; init; } + + /// Frame rate from VUI timing (time_scale / 2·num_units_in_tick), or null. + public double? FrameRate => NumUnitsInTick > 0 && TimeScale > 0 ? TimeScale / (2.0 * NumUnitsInTick) : null; +} + +/// Parses H.264 SPS NAL units (ITU-T H.264 7.3.2.1.1). Never throws on malformed input. +public static class H264SpsParser +{ + /// Parses an SPS NAL unit (header byte included, emulation prevention present). + public static bool TryParse(ReadOnlySpan nal, out H264Sps sps) + { + sps = null!; + if (nal.Length < 4 || AnnexBParser.H264Type(nal[0]) != H264NalUnitType.Sps) return false; + var rbsp = AnnexBParser.RemoveEmulationPrevention(nal[1..]); + var r = new BitReader(rbsp); + + var profile = (int)r.ReadBits(8); + var constraints = (byte)r.ReadBits(8); + var level = (int)r.ReadBits(8); + var spsId = r.ReadUe(); + if (spsId > 31) return false; + + int chroma = 1, depthLuma = 8, depthChroma = 8; + var separateColourPlane = false; + if (profile is 100 or 110 or 122 or 244 or 44 or 83 or 86 or 118 or 128 or 138 or 139 or 134 or 135) + { + chroma = (int)r.ReadUe(); + if (chroma > 3) return false; + if (chroma == 3) separateColourPlane = r.ReadFlag(); + depthLuma = 8 + (int)r.ReadUe(); + depthChroma = 8 + (int)r.ReadUe(); + if (depthLuma > 14 || depthChroma > 14) return false; + r.ReadFlag(); // qpprime_y_zero_transform_bypass_flag + if (r.ReadFlag()) + { + var lists = chroma != 3 ? 8 : 12; + for (var i = 0; i < lists && !r.IsOverrun; i++) + { + if (r.ReadFlag()) SkipScalingList(ref r, i < 6 ? 16 : 64); + } + } + } + + r.ReadUe(); // log2_max_frame_num_minus4 + var pocType = (int)r.ReadUe(); + if (pocType == 0) + { + r.ReadUe(); // log2_max_pic_order_cnt_lsb_minus4 + } + else if (pocType == 1) + { + r.ReadFlag(); // delta_pic_order_always_zero_flag + r.ReadSe(); // offset_for_non_ref_pic + r.ReadSe(); // offset_for_top_to_bottom_field + var cycle = r.ReadUe(); + if (cycle > 255) return false; + for (var i = 0; i < cycle; i++) r.ReadSe(); + } + else if (pocType != 2) + { + return false; + } + + var maxRefFrames = (int)r.ReadUe(); + r.ReadFlag(); // gaps_in_frame_num_value_allowed_flag + var widthMbs = r.ReadUe() + 1L; + var heightMapUnits = r.ReadUe() + 1L; + var frameMbsOnly = r.ReadFlag(); + if (!frameMbsOnly) r.ReadFlag(); // mb_adaptive_frame_field_flag + r.ReadFlag(); // direct_8x8_inference_flag + long cropLeft = 0, cropRight = 0, cropTop = 0, cropBottom = 0; + if (r.ReadFlag()) + { + cropLeft = r.ReadUe(); + cropRight = r.ReadUe(); + cropTop = r.ReadUe(); + cropBottom = r.ReadUe(); + } + + var codedWidth = widthMbs * 16; + var codedHeight = (2 - (frameMbsOnly ? 1 : 0)) * heightMapUnits * 16; + var chromaArrayType = separateColourPlane ? 0 : chroma; + var cropUnitX = chromaArrayType == 0 ? 1 : (chromaArrayType == 3 ? 1 : 2); + var cropUnitY = (chromaArrayType == 0 ? 1 : (chromaArrayType == 1 ? 2 : 1)) * (frameMbsOnly ? 1 : 2); + var width = codedWidth - cropUnitX * (cropLeft + cropRight); + var height = codedHeight - cropUnitY * (cropTop + cropBottom); + if (r.IsOverrun || width <= 0 || height <= 0 || codedWidth > 65536 || codedHeight > 65536) return false; + + var result = new H264Sps + { + ProfileIdc = profile, + ConstraintFlags = constraints, + LevelIdc = level, + SpsId = (int)spsId, + ChromaFormatIdc = chroma, + BitDepthLuma = depthLuma, + BitDepthChroma = depthChroma, + Width = (int)width, + Height = (int)height, + CodedWidth = (int)codedWidth, + CodedHeight = (int)codedHeight, + FrameMbsOnly = frameMbsOnly, + PicOrderCntType = pocType, + MaxNumRefFrames = maxRefFrames, + }; + + if (r.ReadFlag()) result = ParseVui(ref r, result); + // VUI is optional detail: a truncated VUI still yields the core fields. + sps = result; + return true; + } + + private static H264Sps ParseVui(ref BitReader r, H264Sps sps) + { + int sarW = 1, sarH = 1; + if (r.ReadFlag()) + { + var idc = (int)r.ReadBits(8); + if (idc == 255) + { + sarW = (int)r.ReadBits(16); + sarH = (int)r.ReadBits(16); + } + else if (idc > 0 && idc < SarTable.Length / 2) + { + sarW = SarTable[idc * 2]; + sarH = SarTable[idc * 2 + 1]; + } + } + if (r.ReadFlag()) r.ReadFlag(); // overscan_info_present_flag, overscan_appropriate_flag + bool? fullRange = null; + int? primaries = null, transfer = null, matrix = null; + if (r.ReadFlag()) + { + r.ReadBits(3); // video_format + fullRange = r.ReadFlag(); + if (r.ReadFlag()) + { + primaries = (int)r.ReadBits(8); + transfer = (int)r.ReadBits(8); + matrix = (int)r.ReadBits(8); + } + } + if (r.ReadFlag()) + { + r.ReadUe(); // chroma_sample_loc_type_top_field + r.ReadUe(); // chroma_sample_loc_type_bottom_field + } + uint units = 0, scale = 0; + var fixedRate = false; + if (r.ReadFlag()) + { + units = r.ReadBits(32); + scale = r.ReadBits(32); + fixedRate = r.ReadFlag(); + } + if (r.IsOverrun) return sps with { HasVui = true }; + return sps with + { + HasVui = true, + SarWidth = sarW == 0 ? 1 : sarW, + SarHeight = sarH == 0 ? 1 : sarH, + FullRange = fullRange, + ColourPrimaries = primaries, + TransferCharacteristics = transfer, + MatrixCoefficients = matrix, + NumUnitsInTick = units, + TimeScale = scale, + FixedFrameRate = fixedRate, + }; + } + + private static void SkipScalingList(ref BitReader r, int size) + { + int last = 8, next = 8; + for (var j = 0; j < size && !r.IsOverrun; j++) + { + if (next != 0) next = (last + r.ReadSe() + 256) % 256; + last = next == 0 ? last : next; + } + } + + // Table E-1 sample aspect ratios for aspect_ratio_idc 1..16 (index 0 unused). + private static ReadOnlySpan SarTable => + [ + 0, 0, 1, 1, 12, 11, 10, 11, 16, 11, 40, 33, 24, 11, 20, 11, 32, 11, 80, 33, 18, 11, 15, 11, 64, 33, 160, 99, 4, 3, 3, 2, 2, 1, + ]; +} diff --git a/src/Titano.Core/Video/Bitstream/HevcSpsParser.cs b/src/Titano.Core/Video/Bitstream/HevcSpsParser.cs new file mode 100644 index 00000000..5305a521 --- /dev/null +++ b/src/Titano.Core/Video/Bitstream/HevcSpsParser.cs @@ -0,0 +1,120 @@ +namespace Titano.Core.Video.Bitstream; + +/// general_profile_tier_level fields of an HEVC SPS (ITU-T H.265 7.3.3), as stored in hvcC. +public sealed record HevcProfileTierLevel( + int ProfileSpace, + bool HighTier, + int ProfileIdc, + uint ProfileCompatibilityFlags, + ulong ConstraintIndicatorFlags, + int LevelIdc); + +/// Fields of an HEVC sequence parameter set needed for containers and diagnostics. +public sealed record HevcSps +{ + public required int VpsId { get; init; } + public required int MaxSubLayers { get; init; } + public required bool TemporalIdNesting { get; init; } + public required HevcProfileTierLevel ProfileTierLevel { get; init; } + public required int SpsId { get; init; } + public required int ChromaFormatIdc { get; init; } + public required int BitDepthLuma { get; init; } + public required int BitDepthChroma { get; init; } + public required int CodedWidth { get; init; } + public required int CodedHeight { get; init; } + + /// Width after the conformance window. + public required int Width { get; init; } + + /// Height after the conformance window. + public required int Height { get; init; } +} + +/// Parses HEVC SPS NAL units up to the bit depths. Never throws on malformed input. +public static class HevcSpsParser +{ + /// Parses an SPS NAL unit (2-byte header included, emulation prevention present). + public static bool TryParse(ReadOnlySpan nal, out HevcSps sps) + { + sps = null!; + if (nal.Length < 6 || AnnexBParser.HevcType(nal[0]) != HevcNalUnitType.Sps) return false; + var rbsp = AnnexBParser.RemoveEmulationPrevention(nal[2..]); + var r = new BitReader(rbsp); + + var vpsId = (int)r.ReadBits(4); + var maxSubLayersMinus1 = (int)r.ReadBits(3); + if (maxSubLayersMinus1 > 6) return false; + var nesting = r.ReadFlag(); + var ptl = ReadProfileTierLevel(ref r, maxSubLayersMinus1); + var spsId = r.ReadUe(); + if (spsId > 15) return false; + var chroma = (int)r.ReadUe(); + if (chroma > 3) return false; + if (chroma == 3) r.ReadFlag(); // separate_colour_plane_flag + var codedWidth = (long)r.ReadUe(); + var codedHeight = (long)r.ReadUe(); + long left = 0, right = 0, top = 0, bottom = 0; + if (r.ReadFlag()) + { + left = r.ReadUe(); + right = r.ReadUe(); + top = r.ReadUe(); + bottom = r.ReadUe(); + } + var depthLuma = 8 + (int)r.ReadUe(); + var depthChroma = 8 + (int)r.ReadUe(); + + var subWidth = chroma is 1 or 2 ? 2 : 1; + var subHeight = chroma == 1 ? 2 : 1; + var width = codedWidth - subWidth * (left + right); + var height = codedHeight - subHeight * (top + bottom); + if (r.IsOverrun || depthLuma > 16 || depthChroma > 16 || width <= 0 || height <= 0 || codedWidth > 65536 || codedHeight > 65536) + return false; + + sps = new HevcSps + { + VpsId = vpsId, + MaxSubLayers = maxSubLayersMinus1 + 1, + TemporalIdNesting = nesting, + ProfileTierLevel = ptl, + SpsId = (int)spsId, + ChromaFormatIdc = chroma, + BitDepthLuma = depthLuma, + BitDepthChroma = depthChroma, + CodedWidth = (int)codedWidth, + CodedHeight = (int)codedHeight, + Width = (int)width, + Height = (int)height, + }; + return true; + } + + /// profile_tier_level(1, maxNumSubLayersMinus1); sub-layer fields are skipped. + internal static HevcProfileTierLevel ReadProfileTierLevel(ref BitReader r, int maxSubLayersMinus1) + { + var space = (int)r.ReadBits(2); + var tier = r.ReadFlag(); + var profile = (int)r.ReadBits(5); + var compatibility = r.ReadBits(32); + var constraints = r.ReadBits64(48); + var level = (int)r.ReadBits(8); + + Span profilePresent = stackalloc bool[8]; + Span levelPresent = stackalloc bool[8]; + for (var i = 0; i < maxSubLayersMinus1; i++) + { + profilePresent[i] = r.ReadFlag(); + levelPresent[i] = r.ReadFlag(); + } + if (maxSubLayersMinus1 > 0) + { + for (var i = maxSubLayersMinus1; i < 8; i++) r.ReadBits(2); // reserved_zero_2bits + } + for (var i = 0; i < maxSubLayersMinus1; i++) + { + if (profilePresent[i]) r.Skip(88); + if (levelPresent[i]) r.Skip(8); + } + return new HevcProfileTierLevel(space, tier, profile, compatibility, constraints, level); + } +} diff --git a/tests/Titano.Core.Tests/Video/BitstreamTests.cs b/tests/Titano.Core.Tests/Video/BitstreamTests.cs new file mode 100644 index 00000000..f184c3c2 --- /dev/null +++ b/tests/Titano.Core.Tests/Video/BitstreamTests.cs @@ -0,0 +1,229 @@ +using Titano.Core.Video.Bitstream; +using Titano.Testing; + +namespace Titano.Core.Tests.Video; + +public sealed class BitstreamTests +{ + [Test] + public void ExpGolombRoundTrip() + { + uint[] unsigned = [0, 1, 2, 3, 7, 8, 254, 255, 256, 65535, 1_000_000, uint.MaxValue - 1]; + int[] signed = [0, 1, -1, 2, -2, 100, -100, int.MaxValue, -int.MaxValue]; + var w = new BitWriter(); + foreach (var v in unsigned) w.WriteUe(v); + foreach (var v in signed) w.WriteSe(v); + w.WriteBits(0b101, 3); + var r = new BitReader(w.ToArray()); + foreach (var v in unsigned) Assert.Equal(v, r.ReadUe()); + foreach (var v in signed) Assert.Equal(v, r.ReadSe()); + Assert.Equal(0b101u, r.ReadBits(3)); + Assert.False(r.IsOverrun); + } + + [Test] + public void ReadingPastTheEndSetsOverrunWithoutThrowing() + { + var r = new BitReader([0xFF]); + Assert.Equal(0xFFu, r.ReadBits(8)); + Assert.Equal(0u, r.ReadBits(16)); + Assert.True(r.IsOverrun); + var zeros = new BitReader(new byte[8]); + Assert.Equal(0u, zeros.ReadUe()); + Assert.True(zeros.IsOverrun); + } + + [Test] + public void Leb128RoundTrip() + { + Span buffer = stackalloc byte[8]; + foreach (var v in new ulong[] { 0, 1, 127, 128, 16383, 16384, uint.MaxValue }) + { + var n = Av1ObuParser.WriteLeb128(buffer, v); + Assert.Equal(Av1ObuParser.Leb128Size(v), n); + Assert.True(Av1ObuParser.TryReadLeb128(buffer[..n], out var back, out var length)); + Assert.Equal(v, back); + Assert.Equal(n, length); + } + } + + [Test] + public void SplitsThreeAndFourByteStartCodes() + { + byte[] stream = [0xAA, 0, 0, 0, 1, 0x67, 1, 2, 0, 0, 1, 0x68, 3, 0, 0, 0, 0, 1, 0x65, 4, 5, 6, 0, 0]; + var units = AnnexBParser.Split(stream); + Assert.Count(3, units); + Assert.BytesEqual([0x67, 1, 2], units[0].Slice(stream)); + Assert.BytesEqual([0x68, 3], units[1].Slice(stream)); + Assert.BytesEqual([0x65, 4, 5, 6], units[2].Slice(stream)); + Assert.Equal(H264NalUnitType.Sps, AnnexBParser.H264Type(stream[units[0].Offset])); + Assert.Equal(H264NalUnitType.IdrSlice, AnnexBParser.H264Type(stream[units[2].Offset])); + } + + [Test] + public void ConvertsAnnexBToLengthPrefixedAndDetectsBoth() + { + var annexB = AnnexBParser.ToAnnexB([SyntheticStreams.H264Sps(), SyntheticStreams.H264Pps(), SyntheticStreams.H264Slice(true, 300, 1)]); + var prefixed = AnnexBParser.ToLengthPrefixed(annexB); + var fromAnnexB = AnnexBParser.SplitAccessUnit(annexB, out var wasAnnexB); + var fromPrefixed = AnnexBParser.SplitAccessUnit(prefixed, out var wasAnnexB2); + Assert.True(wasAnnexB); + Assert.False(wasAnnexB2); + Assert.Equal(3, fromAnnexB.Count); + Assert.Equal(3, fromPrefixed.Count); + for (var i = 0; i < 3; i++) + Assert.BytesEqual(fromAnnexB[i].Slice(annexB), fromPrefixed[i].Slice(prefixed)); + } + + [Test] + public void LengthPrefixedSplitRejectsInconsistentLengths() + { + Assert.False(AnnexBParser.TrySplitLengthPrefixed([0, 0, 0, 9, 1, 2], 4, out _)); + Assert.False(AnnexBParser.TrySplitLengthPrefixed([0, 0, 0, 0], 4, out _)); + Assert.True(AnnexBParser.TrySplitLengthPrefixed([0, 2, 0x41, 9, 0, 1, 0x41], 2, out var units)); + Assert.Count(2, units); + } + + [Test] + public void EmulationPreventionRoundTrip() + { + var rng = new Random(7); + for (var trial = 0; trial < 200; trial++) + { + var data = new byte[rng.Next(1, 200)]; + for (var i = 0; i < data.Length; i++) data[i] = rng.Next(4) == 0 ? (byte)rng.Next(4) : (byte)0; + var escaped = AnnexBParser.AddEmulationPrevention(data); + for (var i = 2; i < escaped.Length; i++) + Assert.False(escaped[i - 2] == 0 && escaped[i - 1] == 0 && escaped[i] <= 2, "start code prefix survived escaping"); + var back = AnnexBParser.RemoveEmulationPrevention(escaped); + // After a lone trailing 0x00 the appended 0x03 is not preceded by two zeros, so it stays. + if (back.Length == data.Length + 1) back = back[..^1]; + Assert.BytesEqual(data, back); + } + Assert.BytesEqual([0, 0, 3, 1, 0, 0, 3, 0], AnnexBParser.AddEmulationPrevention([0, 0, 1, 0, 0, 0]).AsSpan(0, 8)); + } + + [Test] + public void ParsesH264HighProfileSps() + { + var nal = SyntheticStreams.H264Sps(1920, 1080, 100, 41); + Assert.True(H264SpsParser.TryParse(nal, out var sps)); + Assert.Equal(100, sps.ProfileIdc); + Assert.Equal(41, sps.LevelIdc); + Assert.Equal(1, sps.ChromaFormatIdc); + Assert.Equal(8, sps.BitDepthLuma); + Assert.Equal(1920, sps.Width); + Assert.Equal(1080, sps.Height); + Assert.Equal(1088, sps.CodedHeight); + Assert.Equal(1, sps.ColourPrimaries); + Assert.Equal(false, sps.FullRange); + Assert.Approximately(29.97, sps.FrameRate!.Value, 0.01); + } + + [Theory] + [Case(1280, 720, 77, 31)] + [Case(3840, 2160, 100, 51)] + [Case(642, 482, 66, 30)] + public void ParsesH264SizesWithCropping(int width, int height, int profile, int level) + { + Assert.True(H264SpsParser.TryParse(SyntheticStreams.H264Sps(width, height, profile, level), out var sps)); + Assert.Equal(width, sps.Width); + Assert.Equal(height, sps.Height); + Assert.Equal(profile, sps.ProfileIdc); + } + + [Test] + public void ParsesHevcMain10Sps() + { + Assert.True(HevcSpsParser.TryParse(SyntheticStreams.HevcSps(1920, 1080, 10, 153), out var sps)); + Assert.Equal(1920, sps.Width); + Assert.Equal(1080, sps.Height); + Assert.Equal(1080, sps.CodedHeight); + Assert.Equal(10, sps.BitDepthLuma); + Assert.Equal(10, sps.BitDepthChroma); + Assert.Equal(1, sps.ChromaFormatIdc); + Assert.Equal(2, sps.ProfileTierLevel.ProfileIdc); + Assert.Equal(153, sps.ProfileTierLevel.LevelIdc); + Assert.Equal(0x2000_0000u, sps.ProfileTierLevel.ProfileCompatibilityFlags); + Assert.Equal(0x9000_0000_0000UL, sps.ProfileTierLevel.ConstraintIndicatorFlags); + Assert.False(sps.ProfileTierLevel.HighTier); + Assert.True(sps.TemporalIdNesting); + } + + [Test] + public void AppliesHevcConformanceWindow() + { + Assert.True(HevcSpsParser.TryParse(SyntheticStreams.HevcSps(1916, 1076, 8, 120), out var sps)); + Assert.Equal(1920, sps.CodedWidth); + Assert.Equal(1080, sps.CodedHeight); + Assert.Equal(1916, sps.Width); + Assert.Equal(1076, sps.Height); + Assert.Equal(1, sps.ProfileTierLevel.ProfileIdc); + } + + [Test] + public void ParsesAv1SequenceHeader() + { + var obuBytes = SyntheticStreams.Av1SequenceHeaderObu(3840, 2160, 12, tenBit: true); + Assert.True(Av1ObuParser.TryReadObu(obuBytes, 0, out var obu)); + Assert.Equal(Av1ObuType.SequenceHeader, obu.Type); + Assert.True(obu.HasSizeField); + Assert.Equal(obuBytes.Length, obu.TotalLength); + Assert.True(Av1ObuParser.TryParseSequenceHeader(obu.Payload(obuBytes), out var header)); + Assert.Equal(3840, header.MaxFrameWidth); + Assert.Equal(2160, header.MaxFrameHeight); + Assert.Equal(12, header.SeqLevelIdx); + Assert.Equal(10, header.BitDepth); + Assert.True(header.ChromaSubsamplingX && header.ChromaSubsamplingY); + Assert.Equal(1, header.ColorPrimaries); + Assert.False(header.FullRange); + } + + [Test] + public void ReadsObuWithoutSizeFieldAndSplitsTemporalUnits() + { + byte[] lastWithoutSize = [(byte)Av1ObuType.Frame << 3, 9, 8, 7]; + Assert.True(Av1ObuParser.TryReadObu(lastWithoutSize, 0, out var obu)); + Assert.False(obu.HasSizeField); + Assert.Equal(3, obu.PayloadLength); + + var stream = SyntheticStreams.Concat( + SyntheticStreams.Av1TemporalDelimiter(), SyntheticStreams.Av1SequenceHeaderObu(), SyntheticStreams.Av1Frame(50, 1), + SyntheticStreams.Av1TemporalDelimiter(), SyntheticStreams.Av1Frame(40, 2), + SyntheticStreams.Av1TemporalDelimiter(), SyntheticStreams.Av1Frame(30, 3)); + var units = Av1ObuParser.SplitTemporalUnits(stream); + Assert.Count(3, units); + Assert.Equal(0, units[0].Offset); + Assert.Equal(stream.Length, units[2].Offset + units[2].Length); + Assert.Equal(units[0].Length, units[1].Offset); + } + + [Test] + public void ParsersSurviveTruncationAndGarbage() + { + byte[][] inputs = [SyntheticStreams.H264Sps(), SyntheticStreams.HevcSps(), SyntheticStreams.Av1SequenceHeaderObu()]; + var rng = new Random(3); + foreach (var input in inputs) + { + for (var length = 0; length < input.Length; length++) + { + var slice = input.AsSpan(0, length); + H264SpsParser.TryParse(slice, out _); + HevcSpsParser.TryParse(slice, out _); + Av1ObuParser.Split(slice); + if (length > 2) Av1ObuParser.TryParseSequenceHeader(slice[2..], out _); + } + for (var trial = 0; trial < 300; trial++) + { + var corrupt = (byte[])input.Clone(); + corrupt[rng.Next(corrupt.Length)] ^= (byte)(1 << rng.Next(8)); + H264SpsParser.TryParse(corrupt, out _); + HevcSpsParser.TryParse(corrupt, out _); + Av1ObuParser.SplitTemporalUnits(corrupt); + Av1ObuParser.TryParseSequenceHeader(corrupt, out _); + AnnexBParser.SplitAccessUnit(corrupt, out _); + } + } + Assert.False(H264SpsParser.TryParse(SyntheticStreams.H264Sps().AsSpan(0, 5), out _)); + } +} diff --git a/tests/Titano.Core.Tests/Video/SyntheticStreams.cs b/tests/Titano.Core.Tests/Video/SyntheticStreams.cs new file mode 100644 index 00000000..ef8b8177 --- /dev/null +++ b/tests/Titano.Core.Tests/Video/SyntheticStreams.cs @@ -0,0 +1,323 @@ +using Titano.Core.Video.Bitstream; + +namespace Titano.Core.Tests.Video; + +/// Builds syntactically valid H.264/HEVC parameter sets, AV1 sequence headers and slice stubs. +internal static class SyntheticStreams +{ + /// H.264 SPS NAL (header included) for 4:2:0 8-bit with cropping to the requested size and VUI timing. + public static byte[] H264Sps(int width = 1920, int height = 1080, int profileIdc = 100, int levelIdc = 41, uint timeScale = 60000, uint unitsInTick = 1001) + { + var w = new BitWriter(); + w.WriteBits((ulong)profileIdc, 8); + w.WriteBits(0, 8); // constraint flags + w.WriteBits((ulong)levelIdc, 8); + w.WriteUe(0); // seq_parameter_set_id + if (profileIdc == 100) + { + w.WriteUe(1); // chroma_format_idc + w.WriteUe(0); // bit_depth_luma_minus8 + w.WriteUe(0); // bit_depth_chroma_minus8 + w.WriteBit(false); // qpprime_y_zero_transform_bypass_flag + w.WriteBit(false); // seq_scaling_matrix_present_flag + } + w.WriteUe(0); // log2_max_frame_num_minus4 + w.WriteUe(0); // pic_order_cnt_type + w.WriteUe(2); // log2_max_pic_order_cnt_lsb_minus4 + w.WriteUe(4); // max_num_ref_frames + w.WriteBit(false); // gaps_in_frame_num_value_allowed_flag + var mbW = (width + 15) / 16; + var mbH = (height + 15) / 16; + w.WriteUe((uint)mbW - 1); + w.WriteUe((uint)mbH - 1); + w.WriteBit(true); // frame_mbs_only_flag + w.WriteBit(true); // direct_8x8_inference_flag + var cropRight = (mbW * 16 - width) / 2; + var cropBottom = (mbH * 16 - height) / 2; + var crop = cropRight > 0 || cropBottom > 0; + w.WriteBit(crop); + if (crop) + { + w.WriteUe(0); + w.WriteUe((uint)cropRight); + w.WriteUe(0); + w.WriteUe((uint)cropBottom); + } + w.WriteBit(true); // vui_parameters_present_flag + w.WriteBit(true); // aspect_ratio_info_present_flag + w.WriteBits(1, 8); // aspect_ratio_idc 1:1 + w.WriteBit(false); // overscan_info_present_flag + w.WriteBit(true); // video_signal_type_present_flag + w.WriteBits(5, 3); // video_format + w.WriteBit(false); // video_full_range_flag + w.WriteBit(true); // colour_description_present_flag + w.WriteBits(1, 8); + w.WriteBits(1, 8); + w.WriteBits(1, 8); + w.WriteBit(false); // chroma_loc_info_present_flag + w.WriteBit(true); // timing_info_present_flag + w.WriteBits(unitsInTick, 32); + w.WriteBits(timeScale, 32); + w.WriteBit(true); // fixed_frame_rate_flag + w.WriteBit(false); // nal_hrd_parameters_present_flag + w.WriteBit(false); // vcl_hrd_parameters_present_flag + w.WriteBit(false); // pic_struct_present_flag + w.WriteBit(false); // bitstream_restriction_flag + w.WriteTrailingBits(); + return Nal([0x67], w.ToArray()); + } + + public static byte[] H264Pps() + { + var w = new BitWriter(); + w.WriteUe(0); // pic_parameter_set_id + w.WriteUe(0); // seq_parameter_set_id + w.WriteBit(true); // entropy_coding_mode_flag + w.WriteBit(false); // bottom_field_pic_order_in_frame_present_flag + w.WriteUe(0); // num_slice_groups_minus1 + w.WriteUe(0); // num_ref_idx_l0_default_active_minus1 + w.WriteUe(0); // num_ref_idx_l1_default_active_minus1 + w.WriteBit(false); // weighted_pred_flag + w.WriteBits(0, 2); // weighted_bipred_idc + w.WriteSe(0); // pic_init_qp_minus26 + w.WriteSe(0); // pic_init_qs_minus26 + w.WriteSe(0); // chroma_qp_index_offset + w.WriteBit(true); // deblocking_filter_control_present_flag + w.WriteBit(false); // constrained_intra_pred_flag + w.WriteBit(false); // redundant_pic_cnt_present_flag + w.WriteBit(true); // transform_8x8_mode_flag + w.WriteBit(false); // pic_scaling_matrix_present_flag + w.WriteSe(0); // second_chroma_qp_index_offset + w.WriteTrailingBits(); + return Nal([0x68], w.ToArray()); + } + + /// A slice NAL with a plausible header and pseudo-random payload. + public static byte[] H264Slice(bool idr, int payloadBytes, int seed) + { + var w = new BitWriter(); + w.WriteUe(0); // first_mb_in_slice + w.WriteUe(idr ? 7u : 5u); // slice_type I / P + w.WriteUe(0); // pic_parameter_set_id + w.WriteBytes(Random(payloadBytes, seed)); + w.WriteTrailingBits(); + return Nal([idr ? (byte)0x65 : (byte)0x41], w.ToArray()); + } + + public static byte[] H264AccessUnitDelimiter() => [0x09, 0xF0]; + + public static byte[] HevcVps() + { + var w = new BitWriter(); + w.WriteBits(0, 4); // vps_video_parameter_set_id + w.WriteBit(true); // vps_base_layer_internal_flag + w.WriteBit(true); // vps_base_layer_available_flag + w.WriteBits(0, 6); // vps_max_layers_minus1 + w.WriteBits(0, 3); // vps_max_sub_layers_minus1 + w.WriteBit(true); // vps_temporal_id_nesting_flag + w.WriteBits(0xFFFF, 16); + WriteHevcProfileTierLevel(w, 2, 153); + w.WriteBit(true); // vps_sub_layer_ordering_info_present_flag + w.WriteUe(4); + w.WriteUe(0); + w.WriteUe(0); + w.WriteBits(0, 6); // vps_max_layer_id + w.WriteUe(0); // vps_num_layer_sets_minus1 + w.WriteBit(false); // vps_timing_info_present_flag + w.WriteBit(false); // vps_extension_flag + w.WriteTrailingBits(); + return Nal([0x40, 0x01], w.ToArray()); + } + + /// HEVC SPS for 4:2:0 with the given bit depth, coded in 8×8 units with a conformance window. + public static byte[] HevcSps(int width = 1920, int height = 1080, int bitDepth = 10, int levelIdc = 153) + { + var w = new BitWriter(); + w.WriteBits(0, 4); // sps_video_parameter_set_id + w.WriteBits(0, 3); // sps_max_sub_layers_minus1 + w.WriteBit(true); // sps_temporal_id_nesting_flag + WriteHevcProfileTierLevel(w, bitDepth == 10 ? 2 : 1, levelIdc); + w.WriteUe(0); // sps_seq_parameter_set_id + w.WriteUe(1); // chroma_format_idc + var codedW = (width + 7) / 8 * 8; + var codedH = (height + 7) / 8 * 8; + w.WriteUe((uint)codedW); + w.WriteUe((uint)codedH); + var window = codedW != width || codedH != height; + w.WriteBit(window); + if (window) + { + w.WriteUe(0); + w.WriteUe((uint)(codedW - width) / 2); + w.WriteUe(0); + w.WriteUe((uint)(codedH - height) / 2); + } + w.WriteUe((uint)bitDepth - 8); + w.WriteUe((uint)bitDepth - 8); + w.WriteUe(4); // log2_max_pic_order_cnt_lsb_minus4 + w.WriteBit(true); // sps_sub_layer_ordering_info_present_flag + w.WriteUe(4); + w.WriteUe(0); + w.WriteUe(0); + w.WriteUe(0); // log2_min_luma_coding_block_size_minus3 + w.WriteUe(3); // log2_diff_max_min_luma_coding_block_size + w.WriteUe(0); // log2_min_luma_transform_block_size_minus2 + w.WriteUe(3); // log2_diff_max_min_luma_transform_block_size + w.WriteUe(1); // max_transform_hierarchy_depth_inter + w.WriteUe(1); // max_transform_hierarchy_depth_intra + w.WriteBit(false); // scaling_list_enabled_flag + w.WriteBit(true); // amp_enabled_flag + w.WriteBit(true); // sample_adaptive_offset_enabled_flag + w.WriteBit(false); // pcm_enabled_flag + w.WriteUe(0); // num_short_term_ref_pic_sets + w.WriteBit(false); // long_term_ref_pics_present_flag + w.WriteBit(true); // sps_temporal_mvp_enabled_flag + w.WriteBit(true); // strong_intra_smoothing_enabled_flag + w.WriteBit(false); // vui_parameters_present_flag + w.WriteBit(false); // sps_extension_present_flag + w.WriteTrailingBits(); + return Nal([0x42, 0x01], w.ToArray()); + } + + public static byte[] HevcPps() + { + var w = new BitWriter(); + w.WriteUe(0); // pps_pic_parameter_set_id + w.WriteUe(0); // pps_seq_parameter_set_id + w.WriteBit(false); // dependent_slice_segments_enabled_flag + w.WriteBit(false); // output_flag_present_flag + w.WriteBits(0, 3); // num_extra_slice_header_bits + w.WriteBit(false); // sign_data_hiding_enabled_flag + w.WriteBit(true); // cabac_init_present_flag + w.WriteUe(0); + w.WriteUe(0); + w.WriteSe(0); // init_qp_minus26 + w.WriteBit(false); // constrained_intra_pred_flag + w.WriteBit(false); // transform_skip_enabled_flag + w.WriteBit(false); // cu_qp_delta_enabled_flag + w.WriteSe(0); + w.WriteSe(0); + w.WriteBit(false); // pps_slice_chroma_qp_offsets_present_flag + w.WriteBit(false); // weighted_pred_flag + w.WriteBit(false); // weighted_bipred_flag + w.WriteBit(false); // transquant_bypass_enabled_flag + w.WriteBit(false); // tiles_enabled_flag + w.WriteBit(false); // entropy_coding_sync_enabled_flag + w.WriteBit(true); // pps_loop_filter_across_slices_enabled_flag + w.WriteBit(false); // deblocking_filter_control_present_flag + w.WriteBit(false); // pps_scaling_list_data_present_flag + w.WriteBit(false); // lists_modification_present_flag + w.WriteUe(0); // log2_parallel_merge_level_minus2 + w.WriteBit(false); // slice_segment_header_extension_present_flag + w.WriteBit(false); // pps_extension_present_flag + w.WriteTrailingBits(); + return Nal([0x44, 0x01], w.ToArray()); + } + + public static byte[] HevcSlice(bool idr, int payloadBytes, int seed) + { + var type = idr ? HevcNalUnitType.IdrWRadl : HevcNalUnitType.TrailR; + var w = new BitWriter(); + w.WriteBit(true); // first_slice_segment_in_pic_flag + w.WriteBytes(Random(payloadBytes, seed)); + w.WriteTrailingBits(); + return Nal([(byte)((byte)type << 1), 0x01], w.ToArray()); + } + + /// AV1 sequence header OBU (with size field) for profile 0 4:2:0. + public static byte[] Av1SequenceHeaderObu(int width = 1920, int height = 1080, int levelIdx = 8, bool tenBit = false) + { + var w = new BitWriter(); + w.WriteBits(0, 3); // seq_profile + w.WriteBit(false); // still_picture + w.WriteBit(false); // reduced_still_picture_header + w.WriteBit(false); // timing_info_present_flag + w.WriteBit(false); // initial_display_delay_present_flag + w.WriteBits(0, 5); // operating_points_cnt_minus_1 + w.WriteBits(0, 12); // operating_point_idc[0] + w.WriteBits((ulong)levelIdx, 5); + if (levelIdx > 7) w.WriteBit(false); // seq_tier[0] + const int bits = 16; + w.WriteBits(bits - 1, 4); + w.WriteBits(bits - 1, 4); + w.WriteBits((ulong)width - 1, bits); + w.WriteBits((ulong)height - 1, bits); + w.WriteBit(false); // frame_id_numbers_present_flag + w.WriteBit(false); // use_128x128_superblock + w.WriteBit(true); // enable_filter_intra + w.WriteBit(true); // enable_intra_edge_filter + w.WriteBits(0, 4); // interintra, masked, warped, dual filter + w.WriteBit(true); // enable_order_hint + w.WriteBits(0, 2); // enable_jnt_comp, enable_ref_frame_mvs + w.WriteBit(true); // seq_choose_screen_content_tools + w.WriteBit(true); // seq_choose_integer_mv + w.WriteBits(6, 3); // order_hint_bits_minus_1 + w.WriteBits(0, 3); // superres, cdef, restoration + w.WriteBit(tenBit); // high_bitdepth + w.WriteBit(false); // mono_chrome + w.WriteBit(true); // color_description_present_flag + w.WriteBits(1, 8); + w.WriteBits(1, 8); + w.WriteBits(1, 8); + w.WriteBit(false); // color_range + w.WriteBits(0, 2); // chroma_sample_position + w.WriteBit(false); // separate_uv_delta_q + w.WriteBit(false); // film_grain_params_present + w.WriteTrailingBits(); + return Obu(Av1ObuType.SequenceHeader, w.ToArray()); + } + + public static byte[] Av1TemporalDelimiter() => Obu(Av1ObuType.TemporalDelimiter, []); + + public static byte[] Av1Frame(int payloadBytes, int seed) => Obu(Av1ObuType.Frame, Random(payloadBytes, seed)); + + /// An OBU with obu_has_size_field set. + public static byte[] Obu(Av1ObuType type, ReadOnlySpan payload) + { + var w = new BitWriter(); + w.WriteBits((ulong)((byte)type << 3 | 0x02), 8); + w.WriteLeb128((ulong)payload.Length); + w.WriteBytes(payload); + return w.ToArray(); + } + + public static byte[] Concat(params byte[][] parts) + { + var total = 0; + foreach (var p in parts) total += p.Length; + var result = new byte[total]; + var pos = 0; + foreach (var p in parts) + { + p.CopyTo(result, pos); + pos += p.Length; + } + return result; + } + + public static byte[] Random(int length, int seed) + { + var bytes = new byte[length]; + new Random(seed).NextBytes(bytes); + return bytes; + } + + private static void WriteHevcProfileTierLevel(BitWriter w, int profileIdc, int levelIdc) + { + w.WriteBits(0, 2); // general_profile_space + w.WriteBit(false); // general_tier_flag + w.WriteBits((ulong)profileIdc, 5); + w.WriteBits(1UL << (31 - profileIdc), 32); // general_profile_compatibility_flag[profileIdc] + w.WriteBits(0x9000_0000_0000UL, 48); // progressive_source + frame_only_constraint + w.WriteBits((ulong)levelIdc, 8); + } + + private static byte[] Nal(ReadOnlySpan header, byte[] rbsp) + { + var escaped = AnnexBParser.AddEmulationPrevention(rbsp); + var nal = new byte[header.Length + escaped.Length]; + header.CopyTo(nal); + escaped.CopyTo(nal, header.Length); + return nal; + } +} From 648f5ca7b0b95e9cfc37a5222694f94fe1efdd64 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 09:33:30 +0000 Subject: [PATCH 2/9] feat(video): add streaming MP4 muxer, decoder configuration records and MP4 inspector Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Video/Mp4/BoxWriter.cs | 106 +++ .../Video/Mp4/DecoderConfiguration.cs | 251 +++++++ src/Titano.Core/Video/Mp4/Mp4Muxer.cs | 662 +++++++++++++++++ src/Titano.Core/Video/Mp4Inspector.cs | 683 ++++++++++++++++++ .../Titano.Core.Tests/Video/Mp4MuxerTests.cs | 240 ++++++ tests/Titano.Core.Tests/Video/SparseStream.cs | 76 ++ 6 files changed, 2018 insertions(+) create mode 100644 src/Titano.Core/Video/Mp4/BoxWriter.cs create mode 100644 src/Titano.Core/Video/Mp4/DecoderConfiguration.cs create mode 100644 src/Titano.Core/Video/Mp4/Mp4Muxer.cs create mode 100644 src/Titano.Core/Video/Mp4Inspector.cs create mode 100644 tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs create mode 100644 tests/Titano.Core.Tests/Video/SparseStream.cs diff --git a/src/Titano.Core/Video/Mp4/BoxWriter.cs b/src/Titano.Core/Video/Mp4/BoxWriter.cs new file mode 100644 index 00000000..80542462 --- /dev/null +++ b/src/Titano.Core/Video/Mp4/BoxWriter.cs @@ -0,0 +1,106 @@ +using System.Buffers.Binary; + +namespace Titano.Core.Video.Mp4; + +/// +/// Big-endian ISO-BMFF box builder over a growable in-memory buffer. Boxes nest with +/// /; sizes (32-bit) are patched when a box closes. +/// +public sealed class BoxWriter +{ + private readonly Stack _open = new(); + private byte[] _buffer; + private int _length; + + public BoxWriter(int capacity = 4096) => _buffer = new byte[Math.Max(capacity, 64)]; + + public int Length => _length; + + public ReadOnlySpan WrittenSpan => _buffer.AsSpan(0, _length); + + public void BeginBox(string type) + { + _open.Push(_length); + WriteUInt32(0); + WriteFourCc(type); + } + + public void BeginFullBox(string type, byte version, uint flags) + { + BeginBox(type); + WriteUInt32((uint)version << 24 | (flags & 0xFFFFFF)); + } + + public void EndBox() + { + var start = _open.Pop(); + BinaryPrimitives.WriteUInt32BigEndian(_buffer.AsSpan(start), (uint)(_length - start)); + } + + public void WriteByte(int value) => Grow(1)[0] = (byte)value; + + public void WriteUInt16(int value) => BinaryPrimitives.WriteUInt16BigEndian(Grow(2), (ushort)value); + + public void WriteInt16(short value) => BinaryPrimitives.WriteInt16BigEndian(Grow(2), value); + + public void WriteUInt24(uint value) + { + var span = Grow(3); + span[0] = (byte)(value >> 16); + span[1] = (byte)(value >> 8); + span[2] = (byte)value; + } + + public void WriteUInt32(uint value) => BinaryPrimitives.WriteUInt32BigEndian(Grow(4), value); + + public void WriteInt32(int value) => BinaryPrimitives.WriteInt32BigEndian(Grow(4), value); + + public void WriteUInt48(ulong value) + { + WriteUInt16((int)(value >> 32)); + WriteUInt32((uint)value); + } + + public void WriteUInt64(ulong value) => BinaryPrimitives.WriteUInt64BigEndian(Grow(8), value); + + public void WriteInt64(long value) => BinaryPrimitives.WriteInt64BigEndian(Grow(8), value); + + public void WriteFourCc(string fourCc) + { + if (fourCc.Length != 4) throw new ArgumentException("A four-character code is required", nameof(fourCc)); + var span = Grow(4); + for (var i = 0; i < 4; i++) span[i] = (byte)fourCc[i]; + } + + public void WriteBytes(ReadOnlySpan bytes) => bytes.CopyTo(Grow(bytes.Length)); + + public void WriteZeros(int count) => Grow(count).Clear(); + + /// The identity transformation matrix of mvhd/tkhd. + public void WriteUnityMatrix() + { + WriteUInt32(0x00010000); + WriteUInt32(0); + WriteUInt32(0); + WriteUInt32(0); + WriteUInt32(0x00010000); + WriteUInt32(0); + WriteUInt32(0); + WriteUInt32(0); + WriteUInt32(0x40000000); + } + + public byte[] ToArray() + { + if (_open.Count != 0) throw new InvalidOperationException("Unclosed box"); + return WrittenSpan.ToArray(); + } + + private Span Grow(int count) + { + if (_length + count > _buffer.Length) Array.Resize(ref _buffer, Math.Max(_buffer.Length * 2, _length + count)); + var span = _buffer.AsSpan(_length, count); + _length += count; + return span; + } +} diff --git a/src/Titano.Core/Video/Mp4/DecoderConfiguration.cs b/src/Titano.Core/Video/Mp4/DecoderConfiguration.cs new file mode 100644 index 00000000..a9df9b77 --- /dev/null +++ b/src/Titano.Core/Video/Mp4/DecoderConfiguration.cs @@ -0,0 +1,251 @@ +using System.Buffers.Binary; +using System.Globalization; +using Titano.Core.Video.Bitstream; + +namespace Titano.Core.Video.Mp4; + +/// +/// A decoder configuration record (avcC, hvcC or av1C payload) with the stream properties it declares. +/// +public sealed record DecoderConfiguration( + VideoCodec Codec, + string BoxType, + byte[] Payload, + int Width, + int Height, + int BitDepth, + string CodecString) +{ + /// Builds avcC (ISO/IEC 14496-15 5.3.3) from SPS and PPS NAL units; the first SPS describes the stream. + public static DecoderConfiguration ForAvc(IReadOnlyList sps, IReadOnlyList pps) + { + if (sps.Count is 0 or > 31 || pps.Count is 0 or > 255) throw new ArgumentException("avcC needs 1–31 SPS and 1–255 PPS"); + if (!H264SpsParser.TryParse(sps[0], out var info)) throw new ArgumentException("Unparseable H.264 SPS", nameof(sps)); + var w = new BoxWriter(256); + w.WriteByte(1); + w.WriteByte(info.ProfileIdc); + w.WriteByte(info.ConstraintFlags); + w.WriteByte(info.LevelIdc); + w.WriteByte(0xFF); // reserved + lengthSizeMinusOne = 3 + w.WriteByte(0xE0 | sps.Count); + foreach (var s in sps) + { + w.WriteUInt16(s.Length); + w.WriteBytes(s); + } + w.WriteByte(pps.Count); + foreach (var p in pps) + { + w.WriteUInt16(p.Length); + w.WriteBytes(p); + } + if (info.ProfileIdc is 100 or 110 or 122 or 144) + { + w.WriteByte(0xFC | info.ChromaFormatIdc); + w.WriteByte(0xF8 | (info.BitDepthLuma - 8)); + w.WriteByte(0xF8 | (info.BitDepthChroma - 8)); + w.WriteByte(0); // numOfSequenceParameterSetExt + } + return new DecoderConfiguration(VideoCodec.H264, "avcC", w.ToArray(), info.Width, info.Height, info.BitDepthLuma, AvcCodecString(info)); + } + + /// Builds hvcC (ISO/IEC 14496-15 8.3.3) with complete VPS/SPS/PPS arrays; the first SPS describes the stream. + public static DecoderConfiguration ForHevc(IReadOnlyList vps, IReadOnlyList sps, IReadOnlyList pps) + { + if (vps.Count == 0 || sps.Count == 0 || pps.Count == 0) throw new ArgumentException("hvcC needs VPS, SPS and PPS"); + if (!HevcSpsParser.TryParse(sps[0], out var info)) throw new ArgumentException("Unparseable HEVC SPS", nameof(sps)); + var ptl = info.ProfileTierLevel; + var w = new BoxWriter(256); + w.WriteByte(1); + w.WriteByte(ptl.ProfileSpace << 6 | (ptl.HighTier ? 0x20 : 0) | ptl.ProfileIdc); + w.WriteUInt32(ptl.ProfileCompatibilityFlags); + w.WriteUInt48(ptl.ConstraintIndicatorFlags); + w.WriteByte(ptl.LevelIdc); + w.WriteUInt16(0xF000); // min_spatial_segmentation_idc = 0 + w.WriteByte(0xFC); // parallelismType = 0 (unknown) + w.WriteByte(0xFC | info.ChromaFormatIdc); + w.WriteByte(0xF8 | (info.BitDepthLuma - 8)); + w.WriteByte(0xF8 | (info.BitDepthChroma - 8)); + w.WriteUInt16(0); // avgFrameRate unspecified + w.WriteByte((info.MaxSubLayers & 7) << 3 | (info.TemporalIdNesting ? 4 : 0) | 3); + w.WriteByte(3); + WriteArray(w, HevcNalUnitType.Vps, vps); + WriteArray(w, HevcNalUnitType.Sps, sps); + WriteArray(w, HevcNalUnitType.Pps, pps); + return new DecoderConfiguration(VideoCodec.Hevc, "hvcC", w.ToArray(), info.Width, info.Height, info.BitDepthLuma, HevcCodecString(ptl)); + + static void WriteArray(BoxWriter w, HevcNalUnitType type, IReadOnlyList nals) + { + w.WriteByte(0x80 | (int)type); // array_completeness = 1: no parameter sets in-band (hvc1) + w.WriteUInt16(nals.Count); + foreach (var nal in nals) + { + w.WriteUInt16(nal.Length); + w.WriteBytes(nal); + } + } + } + + /// Builds av1C (AV1-ISOBMFF 2.3) around a sequence header OBU written with obu_size. + public static DecoderConfiguration ForAv1(byte[] sequenceHeaderObu) + { + if (!Av1ObuParser.TryReadObu(sequenceHeaderObu, 0, out var obu) || obu.Type != Av1ObuType.SequenceHeader || !obu.HasSizeField + || !Av1ObuParser.TryParseSequenceHeader(obu.Payload(sequenceHeaderObu), out var seq)) + throw new ArgumentException("A sized sequence header OBU is required", nameof(sequenceHeaderObu)); + var w = new BoxWriter(64); + w.WriteByte(0x81); // marker + version 1 + w.WriteByte(seq.SeqProfile << 5 | seq.SeqLevelIdx); + w.WriteByte((seq.SeqTier ? 0x80 : 0) | (seq.HighBitDepth ? 0x40 : 0) | (seq.TwelveBit ? 0x20 : 0) | (seq.MonoChrome ? 0x10 : 0) + | (seq.ChromaSubsamplingX ? 0x08 : 0) | (seq.ChromaSubsamplingY ? 0x04 : 0) | seq.ChromaSamplePosition); + w.WriteByte(seq.InitialDisplayDelay is { } delay ? 0x10 | ((delay - 1) & 0x0F) : 0); + w.WriteBytes(sequenceHeaderObu.AsSpan(0, obu.TotalLength)); + return new DecoderConfiguration(VideoCodec.Av1, "av1C", w.ToArray(), seq.MaxFrameWidth, seq.MaxFrameHeight, seq.BitDepth, Av1CodecString(seq)); + } + + /// Parses an avcC/hvcC/av1C payload back; explains a failure. + public static bool TryParse(string boxType, ReadOnlySpan payload, out DecoderConfiguration? configuration, out string? error) + { + configuration = null; + error = null; + try + { + configuration = boxType switch + { + "avcC" => ParseAvc(payload, out error), + "hvcC" => ParseHevc(payload, out error), + "av1C" => ParseAv1(payload, out error), + _ => null, + }; + } + catch (ArgumentException ex) + { + error = ex.Message; + configuration = null; + } + error ??= configuration is null ? $"unsupported configuration box '{boxType}'" : null; + return configuration is not null; + } + + private static DecoderConfiguration? ParseAvc(ReadOnlySpan p, out string? error) + { + error = null; + if (p.Length < 7 || p[0] != 1) + { + error = "avcC header invalid"; + return null; + } + if ((p[4] & 3) != 3) error = "avcC lengthSizeMinusOne is not 3"; + var pos = 5; + var sps = ReadNalList(p, ref pos, p[5] & 0x1F, 1); + var pps = pos < p.Length ? ReadNalList(p, ref pos, p[pos], 1) : null; + if (sps is null || pps is null || sps.Count == 0 || pps.Count == 0) + { + error = "avcC parameter set lists truncated or empty"; + return null; + } + var config = ForAvc(sps, pps); + if (config.Payload[1] != p[1] || config.Payload[3] != p[3]) error = "avcC profile/level differ from its SPS"; + return config; + } + + private static DecoderConfiguration? ParseHevc(ReadOnlySpan p, out string? error) + { + error = null; + if (p.Length < 23 || p[0] != 1) + { + error = "hvcC header invalid"; + return null; + } + if ((p[21] & 3) != 3) error = "hvcC lengthSizeMinusOne is not 3"; + var arrays = p[22]; + var pos = 23; + List vps = [], sps = [], pps = []; + for (var a = 0; a < arrays; a++) + { + if (pos + 3 > p.Length) + { + error = "hvcC arrays truncated"; + return null; + } + var type = (HevcNalUnitType)(p[pos] & 0x3F); + var count = BinaryPrimitives.ReadUInt16BigEndian(p[(pos + 1)..]); + pos += 3; + var list = ReadNalList(p, ref pos, count, 0); + if (list is null) + { + error = "hvcC NAL units truncated"; + return null; + } + (type switch { HevcNalUnitType.Vps => vps, HevcNalUnitType.Sps => sps, HevcNalUnitType.Pps => pps, _ => [] }).AddRange(list); + } + if (vps.Count == 0 || sps.Count == 0 || pps.Count == 0) + { + error = "hvcC lacks a VPS, SPS or PPS array"; + return null; + } + var config = ForHevc(vps, sps, pps); + if (!config.Payload.AsSpan(1, 12).SequenceEqual(p.Slice(1, 12))) error = "hvcC profile/tier/level differ from its SPS"; + return config; + } + + private static DecoderConfiguration? ParseAv1(ReadOnlySpan p, out string? error) + { + error = null; + if (p.Length < 4 || p[0] != 0x81) + { + error = "av1C marker/version invalid"; + return null; + } + var obus = p[4..]; + foreach (var obu in Av1ObuParser.Split(obus)) + { + if (obu.Type != Av1ObuType.SequenceHeader) continue; + var config = ForAv1(obus.Slice(obu.Offset, obu.TotalLength).ToArray()); + if (config.Payload[1] != p[1] || config.Payload[2] != p[2]) error = "av1C fields differ from its sequence header"; + return config; + } + error = "av1C has no sequence header OBU"; + return null; + } + + private static List? ReadNalList(ReadOnlySpan p, ref int pos, int count, int countBytes) + { + pos += countBytes; + var list = new List(count); + for (var i = 0; i < count; i++) + { + if (pos + 2 > p.Length) return null; + var length = BinaryPrimitives.ReadUInt16BigEndian(p[pos..]); + pos += 2; + if (length == 0 || pos + length > p.Length) return null; + list.Add(p.Slice(pos, length).ToArray()); + pos += length; + } + return list; + } + + private static string AvcCodecString(H264Sps sps) => + string.Create(CultureInfo.InvariantCulture, $"avc1.{sps.ProfileIdc:x2}{sps.ConstraintFlags:x2}{sps.LevelIdc:x2}"); + + private static string HevcCodecString(HevcProfileTierLevel ptl) + { + var space = ptl.ProfileSpace == 0 ? string.Empty : ((char)('A' + ptl.ProfileSpace - 1)).ToString(); + var compatibility = ReverseBits(ptl.ProfileCompatibilityFlags); + var s = string.Create(CultureInfo.InvariantCulture, + $"hvc1.{space}{ptl.ProfileIdc}.{compatibility:X}.{(ptl.HighTier ? 'H' : 'L')}{ptl.LevelIdc}"); + var last = 5; + while (last >= 0 && ((ptl.ConstraintIndicatorFlags >> (8 * (5 - last))) & 0xFF) == 0) last--; + for (var i = 0; i <= last; i++) s += "." + ((ptl.ConstraintIndicatorFlags >> (8 * (5 - i))) & 0xFF).ToString("X", CultureInfo.InvariantCulture); + return s; + } + + private static string Av1CodecString(Av1SequenceHeader seq) => + string.Create(CultureInfo.InvariantCulture, $"av01.{seq.SeqProfile}.{seq.SeqLevelIdx:D2}{(seq.SeqTier ? 'H' : 'M')}.{seq.BitDepth:D2}"); + + private static uint ReverseBits(uint v) + { + uint r = 0; + for (var i = 0; i < 32; i++, v >>= 1) r = (r << 1) | (v & 1); + return r; + } +} diff --git a/src/Titano.Core/Video/Mp4/Mp4Muxer.cs b/src/Titano.Core/Video/Mp4/Mp4Muxer.cs new file mode 100644 index 00000000..2e1f3e74 --- /dev/null +++ b/src/Titano.Core/Video/Mp4/Mp4Muxer.cs @@ -0,0 +1,662 @@ +using System.Buffers.Binary; +using Titano.Core.Diagnostics; +using Titano.Core.Video.Bitstream; + +namespace Titano.Core.Video.Mp4; + +/// Parameters of the single video track written by . +public sealed record VideoTrackConfig +{ + /// , or . + public required VideoCodec Codec { get; init; } + + /// Display width (tkhd); the sample entry uses the size declared by the bitstream. + public required int Width { get; init; } + + public required int Height { get; init; } + + /// Moves moov in front of mdat when completing (progressive playback); the stream must be readable. + public bool FastStart { get; init; } + + /// Signalled in colr; Titano encodes limited (video) range. + public bool FullRange { get; init; } + + public DateTime CreationTime { get; init; } = DateTime.UtcNow; +} + +/// +/// Streaming MP4 writer for one video track: ftyp, then samples straight into mdat, then moov at the end +/// (optionally moved to the front). Timescale 90 kHz; parameter sets are taken from the bitstream. +/// The output stream is not disposed. +/// +public sealed class Mp4Muxer +{ + /// Media and movie timescale (ticks per second). + public const int TimeScale = 90_000; + + private const long ChunkTicks = TimeScale; + private const long ChunkBytes = 4 << 20; + private const string LogCategory = "Mp4Muxer"; + + private readonly Stream _stream; + private readonly VideoTrackConfig _config; + private readonly long _mdatHeader; + private readonly List _sizes = []; + private readonly List _durations = []; + private readonly List _compositionOffsets = []; + private readonly List _syncSamples = []; + private readonly List _chunkOffsets = []; + private readonly List _chunkSampleCounts = []; + private readonly SortedDictionary _vps = []; + private readonly SortedDictionary _sps = []; + private readonly SortedDictionary _pps = []; + private byte[]? _av1SequenceHeader; + private long _position; + private long _chunkTicks; + private long _chunkBytes; + private long _totalTicks; + private long _totalBytes; + private bool _warnedInBandChange; + private bool _completed; + + public Mp4Muxer(Stream output, VideoTrackConfig config) + { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(config); + if (config.Codec is not (VideoCodec.H264 or VideoCodec.Hevc or VideoCodec.Av1)) + throw new ArgumentException($"MP4 muxing supports H.264, HEVC and AV1, not {config.Codec}", nameof(config)); + if (config.Width is <= 0 or > ushort.MaxValue || config.Height is <= 0 or > ushort.MaxValue) + throw new ArgumentOutOfRangeException(nameof(config), "Frame size out of range"); + if (!output.CanSeek || !output.CanWrite) throw new ArgumentException("The output stream must be writable and seekable", nameof(output)); + if (config.FastStart && !output.CanRead) throw new ArgumentException("FastStart needs a readable stream", nameof(output)); + + _stream = output; + _config = config; + var ftyp = new BoxWriter(64); + ftyp.BeginBox("ftyp"); + ftyp.WriteFourCc("isom"); + ftyp.WriteUInt32(0x200); + ftyp.WriteFourCc("isom"); + ftyp.WriteFourCc("iso6"); + ftyp.WriteFourCc("mp41"); + if (config.Codec == VideoCodec.Av1) ftyp.WriteFourCc("av01"); + ftyp.EndBox(); + _stream.Write(ftyp.WrittenSpan); + _mdatHeader = _stream.Position; + // 16 bytes: 'free' + 32-bit 'mdat' header, or one 64-bit 'mdat' header when the payload exceeds 4 GiB. + _stream.Write(stackalloc byte[16]); + _position = _mdatHeader + 16; + } + + public int SampleCount => _sizes.Count; + + /// Sum of the sample durations written so far, in 90 kHz ticks. + public long Duration90k => _totalTicks; + + /// + /// Records parameter sets (H.264/HEVC NAL units, Annex-B or length-prefixed) or an AV1 sequence header OBU + /// delivered outside the samples, e.g. an encoder's MF_MT_MPEG_SEQUENCE_HEADER. + /// + public void AddParameterSets(ReadOnlySpan data) + { + if (_config.Codec == VideoCodec.Av1) + { + foreach (var obu in Av1ObuParser.Split(data)) + { + if (obu.Type == Av1ObuType.SequenceHeader) _av1SequenceHeader ??= SizedObu(data, obu); + } + return; + } + foreach (var unit in AnnexBParser.SplitAccessUnit(data, out _)) RecordNal(unit.Slice(data)); + } + + /// + /// Appends one access unit (H.264/HEVC: Annex-B or 4-byte length-prefixed NAL units; AV1: a temporal unit of OBUs) + /// in decode order. is presentation minus decode time and may be negative. + /// + public void WriteSample(ReadOnlySpan data, long duration90k, bool isSync, long compositionOffset90k = 0) + { + if (_completed) throw new InvalidOperationException("The muxer is complete"); + if (duration90k <= 0 || duration90k > uint.MaxValue) throw new ArgumentOutOfRangeException(nameof(duration90k)); + if (Math.Abs(compositionOffset90k) > int.MaxValue / 2) throw new ArgumentOutOfRangeException(nameof(compositionOffset90k)); + + var start = _position; + if (_stream.Position != start) _stream.Position = start; + var size = _config.Codec == VideoCodec.Av1 ? WriteObus(data) : WriteNalUnits(data); + if (size == 0) throw new ArgumentException("The sample contains no picture data", nameof(data)); + if (_sizes.Count == 0 && !isSync) Log.Warning(LogCategory, "First sample is not a sync sample; players may not start decoding"); + + if (_chunkSampleCounts.Count == 0 || _chunkTicks >= ChunkTicks || _chunkBytes >= ChunkBytes) + { + _chunkOffsets.Add(start); + _chunkSampleCounts.Add(0); + _chunkTicks = 0; + _chunkBytes = 0; + } + _chunkSampleCounts[^1]++; + _chunkTicks += duration90k; + _chunkBytes += size; + _sizes.Add((uint)size); + _durations.Add((uint)duration90k); + _compositionOffsets.Add(compositionOffset90k); + if (isSync) _syncSamples.Add((uint)_sizes.Count); + _totalTicks += duration90k; + _totalBytes += size; + } + + /// Patches mdat, writes moov (and relocates it when is set) and flushes. + public void Complete() + { + if (_completed) throw new InvalidOperationException("The muxer is already complete"); + if (_sizes.Count == 0) throw new InvalidOperationException("No samples were written"); + var configuration = BuildDecoderConfiguration(); + if (configuration.Width != _config.Width || configuration.Height != _config.Height) + { + Log.Warning(LogCategory, $"Bitstream size {configuration.Width}×{configuration.Height} differs from the track size {_config.Width}×{_config.Height}"); + } + + WriteMdatHeader(); + var moov = BuildMoov(configuration, 0); + _stream.Position = _position; + _stream.Write(moov); + if (_config.FastStart) MoveMoovToFront(configuration, moov.Length); + else _position += moov.Length; + _stream.Flush(); + _completed = true; + } + + private long WriteNalUnits(ReadOnlySpan data) + { + Span length = stackalloc byte[4]; + long size = 0; + var picture = false; + foreach (var unit in AnnexBParser.SplitAccessUnit(data, out _)) + { + var nal = unit.Slice(data); + if (!RecordNal(nal)) continue; + BinaryPrimitives.WriteInt32BigEndian(length, nal.Length); + _stream.Write(length); + _stream.Write(nal); + size += 4 + nal.Length; + picture |= IsPicture(nal[0]); + } + _position += size; + return picture ? size : 0; + } + + private long WriteObus(ReadOnlySpan data) + { + var obus = Av1ObuParser.Split(data); + if (obus.Count == 0 || obus[^1].Offset + obus[^1].TotalLength != data.Length) + throw new ArgumentException("Malformed AV1 temporal unit", nameof(data)); + Span header = stackalloc byte[2 + 8]; + long size = 0; + var picture = false; + foreach (var obu in obus) + { + if (obu.Type is Av1ObuType.TemporalDelimiter or Av1ObuType.Padding) continue; + if (obu.Type == Av1ObuType.SequenceHeader) _av1SequenceHeader ??= SizedObu(data, obu); + picture |= obu.Type is Av1ObuType.Frame or Av1ObuType.FrameHeader or Av1ObuType.TileGroup; + if (obu.HasSizeField) + { + _stream.Write(data.Slice(obu.Offset, obu.TotalLength)); + size += obu.TotalLength; + continue; + } + // Samples carry every OBU with obu_size (AV1-ISOBMFF 2.4). + var n = WriteSizedHeader(header, data, obu); + _stream.Write(header[..n]); + _stream.Write(obu.Payload(data)); + size += n + obu.PayloadLength; + } + _position += size; + return picture ? size : 0; + } + + private static int WriteSizedHeader(Span header, ReadOnlySpan data, Av1Obu obu) + { + header[0] = (byte)(data[obu.Offset] | 0x02); + var n = 1; + if (obu.HasExtension) header[n++] = data[obu.Offset + 1]; + return n + Av1ObuParser.WriteLeb128(header[n..], (ulong)obu.PayloadLength); + } + + private static byte[] SizedObu(ReadOnlySpan data, Av1Obu obu) + { + Span header = stackalloc byte[2 + 8]; + var n = WriteSizedHeader(header, data, obu); + var result = new byte[n + obu.PayloadLength]; + header[..n].CopyTo(result); + obu.Payload(data).CopyTo(result.AsSpan(n)); + return result; + } + + private bool IsPicture(byte header) => _config.Codec == VideoCodec.H264 + ? AnnexBParser.H264Type(header) is >= H264NalUnitType.NonIdrSlice and <= H264NalUnitType.IdrSlice + : (int)AnnexBParser.HevcType(header) < 32; + + /// Stores parameter sets for the sample entry; returns whether the NAL unit belongs in the sample. + private bool RecordNal(ReadOnlySpan nal) + { + if (nal.Length < 2) return true; + SortedDictionary table; + int id; + if (_config.Codec == VideoCodec.H264) + { + switch (AnnexBParser.H264Type(nal[0])) + { + case H264NalUnitType.AccessUnitDelimiter or H264NalUnitType.FillerData: + return false; + case H264NalUnitType.Sps when H264SpsParser.TryParse(nal, out var sps): + (table, id) = (_sps, sps.SpsId); + break; + case H264NalUnitType.Pps: + (table, id) = (_pps, FirstUe(nal[1..])); + break; + default: + return true; + } + } + else + { + switch (AnnexBParser.HevcType(nal[0])) + { + case HevcNalUnitType.AccessUnitDelimiter or HevcNalUnitType.FillerData: + return false; + case HevcNalUnitType.Vps when nal.Length > 2: + (table, id) = (_vps, nal[2] >> 4); + break; + case HevcNalUnitType.Sps when HevcSpsParser.TryParse(nal, out var sps): + (table, id) = (_sps, sps.SpsId); + break; + case HevcNalUnitType.Pps: + (table, id) = (_pps, FirstUe(nal[2..])); + break; + default: + return true; + } + } + if (id < 0) return true; + if (!table.TryGetValue(id, out var known)) + { + table[id] = nal.ToArray(); + return false; + } + if (nal.SequenceEqual(known)) return false; + // A redefinition keeps working in-band, but the sample entry describes only the first definition. + if (!_warnedInBandChange) + { + Log.Warning(LogCategory, "A parameter set changed mid-stream; keeping the new one in-band"); + _warnedInBandChange = true; + } + return true; + } + + private static int FirstUe(ReadOnlySpan payload) + { + var rbsp = AnnexBParser.RemoveEmulationPrevention(payload[..Math.Min(payload.Length, 8)]); + var r = new BitReader(rbsp); + var v = r.ReadUe(); + return r.IsOverrun || v > 255 ? -1 : (int)v; + } + + private DecoderConfiguration BuildDecoderConfiguration() => _config.Codec switch + { + VideoCodec.H264 when _sps.Count > 0 && _pps.Count > 0 => DecoderConfiguration.ForAvc([.. _sps.Values], [.. _pps.Values]), + VideoCodec.Hevc when _vps.Count > 0 && _sps.Count > 0 && _pps.Count > 0 => + DecoderConfiguration.ForHevc([.. _vps.Values], [.. _sps.Values], [.. _pps.Values]), + VideoCodec.Av1 when _av1SequenceHeader is not null => DecoderConfiguration.ForAv1(_av1SequenceHeader), + _ => throw new InvalidOperationException("The bitstream carried no parameter sets / sequence header"), + }; + + private void WriteMdatHeader() + { + Span header = stackalloc byte[16]; + var compact = _position - (_mdatHeader + 8); + if (compact <= uint.MaxValue) + { + BinaryPrimitives.WriteUInt32BigEndian(header, 8); + "free"u8.CopyTo(header[4..]); + BinaryPrimitives.WriteUInt32BigEndian(header[8..], (uint)compact); + "mdat"u8.CopyTo(header[12..]); + } + else + { + BinaryPrimitives.WriteUInt32BigEndian(header, 1); + "mdat"u8.CopyTo(header[4..]); + BinaryPrimitives.WriteUInt64BigEndian(header[8..], (ulong)(_position - _mdatHeader)); + } + _stream.Position = _mdatHeader; + _stream.Write(header); + } + + private void MoveMoovToFront(DecoderConfiguration configuration, int moovAtEnd) + { + // Chunk offsets grow by the relocated moov size, which itself grows if they now need co64. + var delta = moovAtEnd; + byte[] moov; + while (true) + { + moov = BuildMoov(configuration, delta); + if (moov.Length == delta) break; + delta = moov.Length; + } + + var buffer = new byte[1 << 20]; + var remaining = _position - _mdatHeader; + while (remaining > 0) + { + var n = (int)Math.Min(buffer.Length, remaining); + var source = _mdatHeader + remaining - n; + _stream.Position = source; + _stream.ReadExactly(buffer, 0, n); + _stream.Position = source + delta; + _stream.Write(buffer, 0, n); + remaining -= n; + } + _stream.Position = _mdatHeader; + _stream.Write(moov); + _position += delta; + } + + private byte[] BuildMoov(DecoderConfiguration configuration, long chunkOffsetDelta) + { + var count = _sizes.Count; + long minOffset = 0; + foreach (var o in _compositionOffsets) minOffset = Math.Min(minOffset, o); + var shift = -minOffset; + + // Presentation span: earliest composition time to the latest composition end. + long decode = 0, minCts = long.MaxValue, maxEnd = 0; + var anyOffset = false; + for (var i = 0; i < count; i++) + { + var cts = decode + _compositionOffsets[i] + shift; + minCts = Math.Min(minCts, cts); + maxEnd = Math.Max(maxEnd, cts + _durations[i]); + anyOffset |= _compositionOffsets[i] + shift != 0; + decode += _durations[i]; + } + var edit = minCts > 0; + var presentation = edit ? Math.Max(1, maxEnd - minCts) : _totalTicks; + var wide = presentation > uint.MaxValue || _totalTicks > uint.MaxValue; + var created = (ulong)Math.Max(0, (_config.CreationTime.ToUniversalTime() - new DateTime(1904, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalSeconds); + + var w = new BoxWriter(4096 + count * 16); + w.BeginBox("moov"); + + w.BeginFullBox("mvhd", wide ? (byte)1 : (byte)0, 0); + WriteTimes(w, wide, created, presentation); + w.WriteUInt32(0x00010000); // rate 1.0 + w.WriteUInt16(0x0100); // volume 1.0 + w.WriteZeros(10); + w.WriteUnityMatrix(); + w.WriteZeros(24); + w.WriteUInt32(2); // next_track_ID + w.EndBox(); + + w.BeginBox("trak"); + w.BeginFullBox("tkhd", wide ? (byte)1 : (byte)0, 3); // enabled | in movie + if (wide) + { + w.WriteUInt64(created); + w.WriteUInt64(created); + w.WriteUInt32(1); + w.WriteUInt32(0); + w.WriteUInt64((ulong)presentation); + } + else + { + w.WriteUInt32((uint)created); + w.WriteUInt32((uint)created); + w.WriteUInt32(1); + w.WriteUInt32(0); + w.WriteUInt32((uint)presentation); + } + w.WriteZeros(8); + w.WriteZeros(8); // layer, alternate_group, volume, reserved + w.WriteUnityMatrix(); + w.WriteUInt32((uint)_config.Width << 16); + w.WriteUInt32((uint)_config.Height << 16); + w.EndBox(); + + if (edit) + { + w.BeginBox("edts"); + w.BeginFullBox("elst", wide ? (byte)1 : (byte)0, 0); + w.WriteUInt32(1); + if (wide) + { + w.WriteUInt64((ulong)presentation); + w.WriteInt64(minCts); + } + else + { + w.WriteUInt32((uint)presentation); + w.WriteInt32((int)minCts); + } + w.WriteUInt32(0x00010000); // media_rate 1.0 + w.EndBox(); + w.EndBox(); + } + + w.BeginBox("mdia"); + w.BeginFullBox("mdhd", wide ? (byte)1 : (byte)0, 0); + WriteTimes(w, wide, created, _totalTicks); + w.WriteUInt16(0x55C4); // 'und' + w.WriteUInt16(0); + w.EndBox(); + + w.BeginFullBox("hdlr", 0, 0); + w.WriteUInt32(0); + w.WriteFourCc("vide"); + w.WriteZeros(12); + w.WriteBytes("VideoHandler\0"u8); + w.EndBox(); + + w.BeginBox("minf"); + w.BeginFullBox("vmhd", 0, 1); + w.WriteZeros(8); + w.EndBox(); + w.BeginBox("dinf"); + w.BeginFullBox("dref", 0, 0); + w.WriteUInt32(1); + w.BeginFullBox("url ", 0, 1); // media data in this file + w.EndBox(); + w.EndBox(); + w.EndBox(); + + w.BeginBox("stbl"); + WriteSampleDescription(w, configuration); + WriteTimeToSample(w); + if (anyOffset) WriteCompositionOffsets(w, shift); + if (_syncSamples.Count != count) + { + w.BeginFullBox("stss", 0, 0); + w.WriteUInt32((uint)_syncSamples.Count); + foreach (var s in _syncSamples) w.WriteUInt32(s); + w.EndBox(); + } + WriteSampleToChunk(w); + WriteSampleSizes(w); + WriteChunkOffsets(w, chunkOffsetDelta); + w.EndBox(); // stbl + w.EndBox(); // minf + w.EndBox(); // mdia + w.EndBox(); // trak + w.EndBox(); // moov + return w.ToArray(); + } + + private static void WriteTimes(BoxWriter w, bool wide, ulong created, long duration) + { + if (wide) + { + w.WriteUInt64(created); + w.WriteUInt64(created); + w.WriteUInt32(TimeScale); + w.WriteUInt64((ulong)duration); + } + else + { + w.WriteUInt32((uint)created); + w.WriteUInt32((uint)created); + w.WriteUInt32(TimeScale); + w.WriteUInt32((uint)duration); + } + } + + private void WriteSampleDescription(BoxWriter w, DecoderConfiguration configuration) + { + w.BeginFullBox("stsd", 0, 0); + w.WriteUInt32(1); + w.BeginBox(_config.Codec switch { VideoCodec.H264 => "avc1", VideoCodec.Hevc => "hvc1", _ => "av01" }); + w.WriteZeros(6); + w.WriteUInt16(1); // data_reference_index + w.WriteZeros(16); // pre_defined, reserved + w.WriteUInt16(configuration.Width); + w.WriteUInt16(configuration.Height); + w.WriteUInt32(0x00480000); // 72 dpi + w.WriteUInt32(0x00480000); + w.WriteUInt32(0); + w.WriteUInt16(1); // frame_count + Span compressor = stackalloc byte[32]; + compressor.Clear(); + compressor[0] = 6; + "Titano"u8.CopyTo(compressor[1..]); + w.WriteBytes(compressor); + w.WriteUInt16(0x0018); + w.WriteInt16(-1); + + w.BeginBox(configuration.BoxType); + w.WriteBytes(configuration.Payload); + w.EndBox(); + + w.BeginBox("colr"); + w.WriteFourCc("nclx"); + w.WriteUInt16(1); // BT.709 primaries + w.WriteUInt16(1); // BT.709 transfer + w.WriteUInt16(1); // BT.709 matrix + w.WriteByte(_config.FullRange ? 0x80 : 0); + w.EndBox(); + + w.BeginBox("pasp"); + w.WriteUInt32(1); + w.WriteUInt32(1); + w.EndBox(); + + w.BeginBox("btrt"); + uint maxSize = 0; + foreach (var s in _sizes) maxSize = Math.Max(maxSize, s); + var seconds = _totalTicks / (double)TimeScale; + w.WriteUInt32(maxSize); + w.WriteUInt32((uint)Math.Min(uint.MaxValue, PeakBitrate())); + w.WriteUInt32((uint)Math.Min(uint.MaxValue, _totalBytes * 8 / Math.Max(seconds, 1e-9))); + w.EndBox(); + + w.EndBox(); // sample entry + w.EndBox(); // stsd + } + + /// Largest number of bits in any one-second window of decode time. + private long PeakBitrate() + { + long peak = 0, window = 0, startTime = 0, time = 0; + var first = 0; + for (var i = 0; i < _sizes.Count; i++) + { + window += _sizes[i]; + while (time - startTime >= TimeScale) + { + window -= _sizes[first]; + startTime += _durations[first]; + first++; + } + peak = Math.Max(peak, window * 8); + time += _durations[i]; + } + return _totalTicks < TimeScale ? (long)(peak * (TimeScale / (double)_totalTicks)) : peak; + } + + private void WriteTimeToSample(BoxWriter w) + { + var runs = new List<(uint Count, uint Delta)>(); + foreach (var d in _durations) + { + if (runs.Count > 0 && runs[^1].Delta == d) runs[^1] = (runs[^1].Count + 1, d); + else runs.Add((1, d)); + } + w.BeginFullBox("stts", 0, 0); + w.WriteUInt32((uint)runs.Count); + foreach (var (count, delta) in runs) + { + w.WriteUInt32(count); + w.WriteUInt32(delta); + } + w.EndBox(); + } + + private void WriteCompositionOffsets(BoxWriter w, long shift) + { + var runs = new List<(uint Count, uint Offset)>(); + foreach (var o in _compositionOffsets) + { + var v = (uint)(o + shift); + if (runs.Count > 0 && runs[^1].Offset == v) runs[^1] = (runs[^1].Count + 1, v); + else runs.Add((1, v)); + } + w.BeginFullBox("ctts", 0, 0); + w.WriteUInt32((uint)runs.Count); + foreach (var (count, offset) in runs) + { + w.WriteUInt32(count); + w.WriteUInt32(offset); + } + w.EndBox(); + } + + private void WriteSampleToChunk(BoxWriter w) + { + var entries = new List<(uint FirstChunk, uint Samples)>(); + for (var i = 0; i < _chunkSampleCounts.Count; i++) + { + if (entries.Count == 0 || entries[^1].Samples != _chunkSampleCounts[i]) entries.Add(((uint)i + 1, _chunkSampleCounts[i])); + } + w.BeginFullBox("stsc", 0, 0); + w.WriteUInt32((uint)entries.Count); + foreach (var (firstChunk, samples) in entries) + { + w.WriteUInt32(firstChunk); + w.WriteUInt32(samples); + w.WriteUInt32(1); + } + w.EndBox(); + } + + private void WriteSampleSizes(BoxWriter w) + { + var constant = true; + foreach (var s in _sizes) constant &= s == _sizes[0]; + w.BeginFullBox("stsz", 0, 0); + w.WriteUInt32(constant ? _sizes[0] : 0); + w.WriteUInt32((uint)_sizes.Count); + if (!constant) + { + foreach (var s in _sizes) w.WriteUInt32(s); + } + w.EndBox(); + } + + private void WriteChunkOffsets(BoxWriter w, long delta) + { + var wide = _chunkOffsets.Count > 0 && _chunkOffsets[^1] + delta > uint.MaxValue; + w.BeginFullBox(wide ? "co64" : "stco", 0, 0); + w.WriteUInt32((uint)_chunkOffsets.Count); + foreach (var offset in _chunkOffsets) + { + if (wide) w.WriteUInt64((ulong)(offset + delta)); + else w.WriteUInt32((uint)(offset + delta)); + } + w.EndBox(); + } +} diff --git a/src/Titano.Core/Video/Mp4Inspector.cs b/src/Titano.Core/Video/Mp4Inspector.cs new file mode 100644 index 00000000..1edfa8c7 --- /dev/null +++ b/src/Titano.Core/Video/Mp4Inspector.cs @@ -0,0 +1,683 @@ +using System.Buffers.Binary; +using System.Globalization; +using System.Text; +using Titano.Core.Video.Bitstream; +using Titano.Core.Video.Mp4; + +namespace Titano.Core.Video; + +/// One ISO-BMFF box of an inspected file. +public sealed record Mp4Box(string Type, long Offset, long Size, int HeaderSize, IReadOnlyList Children) +{ + /// Payload bytes (only kept for boxes inside moov). + internal ReadOnlyMemory Payload { get; init; } + + public Mp4Box? Child(string type) + { + foreach (var c in Children) + { + if (c.Type == type) return c; + } + return null; + } +} + +/// One sample of the inspected video track, in decode order. +public readonly record struct Mp4SampleInfo(long Offset, int Size, long DecodeTime, uint Duration, long CompositionOffset, bool IsSync); + +/// Result of : structure, stream properties and problems found. +public sealed class Mp4Report +{ + public List Errors { get; } = []; + public List Warnings { get; } = []; + public bool IsValid => Errors.Count == 0; + public IReadOnlyList Boxes { get; internal set; } = []; + + /// Sample entry four-character code ("avc1", "hvc1", "av01"…). + public string? SampleEntry { get; internal set; } + + public VideoCodec? Codec { get; internal set; } + + /// RFC 6381 codec string, e.g. "avc1.640029". + public string? CodecString { get; internal set; } + + /// Coded picture size from the sample entry. + public int Width { get; internal set; } + + public int Height { get; internal set; } + public int BitDepth { get; internal set; } + public int FrameCount { get; internal set; } + public int SyncSampleCount { get; internal set; } + public uint TimeScale { get; internal set; } + + /// Presentation duration in ticks (edit list applied). + public long DurationTicks { get; internal set; } + + public TimeSpan Duration => TimeScale == 0 ? TimeSpan.Zero : TimeSpan.FromSeconds(DurationTicks / (double)TimeScale); + public long MediaBytes { get; internal set; } + + /// Average bit rate in bits per second. + public double AverageBitrate { get; internal set; } + + /// moov precedes mdat (progressive download). + public bool FastStart { get; internal set; } + + public bool UsesCo64 { get; internal set; } + public bool UsesLargeMdat { get; internal set; } + public bool HasCompositionOffsets { get; internal set; } + public bool HasEditList { get; internal set; } + public IReadOnlyList Samples { get; internal set; } = []; + + public override string ToString() + { + var sb = new StringBuilder(); + var inv = CultureInfo.InvariantCulture; + sb.AppendLine(inv, $"{CodecString ?? SampleEntry ?? "unknown"} {Width}x{Height} {BitDepth}-bit, {FrameCount} frames ({SyncSampleCount} sync), {Duration.TotalSeconds:0.###} s, {AverageBitrate / 1000:0} kbit/s"); + sb.AppendLine(inv, $"fast start: {FastStart}, co64: {UsesCo64}, 64-bit mdat: {UsesLargeMdat}, composition offsets: {HasCompositionOffsets}, edit list: {HasEditList}"); + foreach (var e in Errors) sb.AppendLine("error: " + e); + foreach (var w in Warnings) sb.AppendLine("warning: " + w); + return sb.ToString(); + } +} + +/// +/// Parses an MP4 into a box tree and validates the first video track: required boxes, sample-table consistency, +/// chunk offsets inside mdat, sync start, durations, decoder configuration versus sample entry. Never throws on +/// malformed input. +/// +public static class Mp4Inspector +{ + private const int MaxDepth = 16; + private const int MaxBoxes = 100_000; + private const long MaxMoovBytes = 256L << 20; + private const int CheckedSamples = 4; + + public static Mp4Report Inspect(string path) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16, FileOptions.RandomAccess); + return Inspect(stream); + } + + public static Mp4Report Inspect(Stream stream) + { + var report = new Mp4Report(); + if (!stream.CanSeek || !stream.CanRead) + { + report.Errors.Add("stream is not readable and seekable"); + return report; + } + try + { + Run(stream, report); + } + catch (IOException ex) + { + report.Errors.Add("read failed: " + ex.Message); + } + return report; + } + + private static void Run(Stream stream, Mp4Report report) + { + var length = stream.Length; + var top = new List(); + var mdats = new List<(long Start, long End)>(); + Mp4Box? moov = null; + long pos = 0; + var budget = MaxBoxes; + Span header = stackalloc byte[16]; + while (pos + 8 <= length && budget-- > 0) + { + stream.Position = pos; + var want = (int)Math.Min(16, length - pos); + var read = stream.ReadAtLeast(header[..want], want, throwOnEndOfStream: false); + long size = BinaryPrimitives.ReadUInt32BigEndian(header); + var type = FourCc(header[4..8]); + var headerSize = 8; + if (size == 1) + { + if (read < 16) + { + report.Errors.Add($"box '{type}' at {pos} is truncated"); + break; + } + size = (long)Math.Min(BinaryPrimitives.ReadUInt64BigEndian(header[8..]), long.MaxValue); + headerSize = 16; + if (type == "mdat") report.UsesLargeMdat = true; + } + else if (size == 0) + { + size = length - pos; + } + if (size < headerSize) + { + report.Errors.Add($"box '{type}' at {pos} has invalid size {size}"); + break; + } + if (size > length - pos) + { + report.Errors.Add($"box '{type}' at {pos} extends past the end of the file ({size} > {length - pos} bytes)"); + size = length - pos; + } + + var children = (IReadOnlyList)[]; + if (type == "moov") + { + if (moov is not null) + { + report.Errors.Add("more than one moov box"); + } + else if (size - headerSize > MaxMoovBytes) + { + report.Errors.Add($"moov is implausibly large ({size} bytes)"); + } + else + { + var data = new byte[size - headerSize]; + stream.Position = pos + headerSize; + stream.ReadExactly(data); + children = ParseBoxes(data, 0, data.Length, pos + headerSize, 1, report, ref budget); + } + } + var box = new Mp4Box(type, pos, size, headerSize, children); + if (type == "moov" && moov is null && children.Count > 0) moov = box; + if (type == "mdat") mdats.Add((pos + headerSize, pos + size)); + if (type == "moof") report.Warnings.Add("fragmented MP4 (moof) is not inspected"); + top.Add(box); + pos += size; + } + if (pos < length && budget > 0) report.Warnings.Add($"{length - pos} trailing bytes after the last box"); + report.Boxes = top; + + if (top.Count == 0 || top[0].Type != "ftyp") + { + if (top.Exists(b => b.Type == "ftyp")) report.Warnings.Add("ftyp is not the first box"); + else report.Errors.Add("ftyp box missing"); + } + if (mdats.Count == 0) report.Errors.Add("mdat box missing"); + if (moov is null) + { + report.Errors.Add("moov box missing or empty"); + return; + } + report.FastStart = mdats.Count > 0 && moov.Offset < mdats[0].Start; + InspectMovie(stream, moov, mdats, report); + } + + private static List ParseBoxes(byte[] data, int start, int end, long baseOffset, int depth, Mp4Report report, ref int budget) + { + var list = new List(); + var pos = start; + while (end - pos >= 8 && budget-- > 0) + { + long size = BinaryPrimitives.ReadUInt32BigEndian(data.AsSpan(pos)); + var type = FourCc(data.AsSpan(pos + 4, 4)); + var headerSize = 8; + if (size == 1) + { + if (end - pos < 16) break; + size = (long)Math.Min(BinaryPrimitives.ReadUInt64BigEndian(data.AsSpan(pos + 8)), long.MaxValue); + headerSize = 16; + } + else if (size == 0) + { + size = end - pos; + } + if (size < headerSize || size > end - pos) + { + report.Errors.Add($"box '{type}' at {baseOffset + pos} overruns its parent"); + return list; + } + var payloadStart = pos + headerSize; + var boxEnd = pos + (int)size; + var childStart = ChildrenStart(type, payloadStart); + var children = childStart >= 0 && childStart <= boxEnd && depth < MaxDepth + ? ParseBoxes(data, childStart, boxEnd, baseOffset, depth + 1, report, ref budget) + : []; + list.Add(new Mp4Box(type, baseOffset + pos, size, headerSize, children) + { + Payload = data.AsMemory(payloadStart, boxEnd - payloadStart), + }); + pos = boxEnd; + } + return list; + } + + private static int ChildrenStart(string type, int payload) => type switch + { + "moov" or "trak" or "edts" or "mdia" or "minf" or "dinf" or "stbl" or "mvex" => payload, + "stsd" or "dref" => payload + 8, + "avc1" or "avc3" or "hvc1" or "hev1" or "av01" => payload + 78, + _ => -1, + }; + + private static void InspectMovie(Stream stream, Mp4Box moov, List<(long Start, long End)> mdats, Mp4Report report) + { + var mvhd = moov.Child("mvhd"); + if (mvhd is null) report.Errors.Add("mvhd missing"); + var movieScale = mvhd is null ? 0u : ReadTimescale(mvhd.Payload.Span, out _); + + Mp4Box? trak = null; + foreach (var t in moov.Children) + { + if (t.Type == "trak" && Handler(t) == "vide") + { + trak = t; + break; + } + } + if (trak is null) + { + report.Errors.Add("no video track (trak with hdlr 'vide')"); + return; + } + if (trak.Child("tkhd") is null) report.Errors.Add("tkhd missing"); + var mdia = trak.Child("mdia"); + var mdhd = mdia?.Child("mdhd"); + var minf = mdia?.Child("minf"); + var stbl = minf?.Child("stbl"); + if (mdhd is null) report.Errors.Add("mdhd missing"); + if (minf?.Child("vmhd") is null) report.Errors.Add("vmhd missing"); + if (minf?.Child("dinf")?.Child("dref") is null) report.Errors.Add("dinf/dref missing"); + if (stbl is null) + { + report.Errors.Add("stbl missing"); + return; + } + long mdhdDuration = 0; + var mediaScale = mdhd is null ? 0u : ReadTimescale(mdhd.Payload.Span, out mdhdDuration); + if (mdhd is not null && mediaScale == 0) report.Errors.Add("mdhd timescale is zero"); + report.TimeScale = mediaScale; + + var config = InspectSampleDescription(stbl, report); + var durations = ReadTimeToSample(stbl, report); + var sizes = ReadSampleSizes(stbl, report); + var chunkOffsets = ReadChunkOffsets(stbl, report); + var samplesPerChunk = ReadSampleToChunk(stbl, chunkOffsets?.Length ?? 0, report); + if (durations is null || sizes is null || chunkOffsets is null || samplesPerChunk is null) return; + + long stscTotal = 0; + foreach (var n in samplesPerChunk) stscTotal += n; + if (durations.Length != sizes.Length) report.Errors.Add($"stts covers {durations.Length} samples, stsz has {sizes.Length}"); + if (stscTotal != sizes.Length) report.Errors.Add($"stsc/stco map {stscTotal} samples, stsz has {sizes.Length}"); + var count = sizes.Length; + report.FrameCount = count; + if (count == 0) + { + report.Errors.Add("track has no samples"); + return; + } + + var offsets = ReadCompositionOffsets(stbl, count, report); + var sync = ReadSyncSamples(stbl, count, report); + report.SyncSampleCount = sync?.Count ?? count; + if (sync is not null && (sync.Count == 0 || sync[0] != 1)) report.Errors.Add("first sample is not a sync sample"); + + var samples = new Mp4SampleInfo[count]; + var index = 0; + long decode = 0, bytes = 0; + var syncIndex = 0; + for (var c = 0; c < chunkOffsets.Length && index < count; c++) + { + var offset = chunkOffsets[c]; + var chunkStart = offset; + for (var s = 0; s < samplesPerChunk[c] && index < count; s++, index++) + { + var isSync = sync is null || (syncIndex < sync.Count && sync[syncIndex] == index + 1); + if (isSync && sync is not null) syncIndex++; + var duration = index < durations.Length ? durations[index] : 0u; + samples[index] = new Mp4SampleInfo(offset, (int)Math.Min(sizes[index], int.MaxValue), decode, duration, offsets?[index] ?? 0, isSync); + offset += sizes[index]; + decode += duration; + bytes += sizes[index]; + } + if (!InsideMdat(chunkStart, offset, mdats)) report.Errors.Add($"chunk {c + 1} [{chunkStart}, {offset}) lies outside mdat"); + } + report.Samples = samples; + report.MediaBytes = bytes; + + for (var i = 0; i < durations.Length; i++) + { + if (durations[i] != 0) continue; + if (i == durations.Length - 1) report.Warnings.Add("last sample has zero duration"); + else report.Errors.Add($"sample {i + 1} has zero duration"); + } + if (mdhd is not null && mdhdDuration != decode) report.Warnings.Add($"mdhd duration {mdhdDuration} differs from the sample durations sum {decode}"); + + report.DurationTicks = decode; + var elst = trak.Child("edts")?.Child("elst"); + if (elst is not null && ReadEditList(elst.Payload.Span, out var segment, out var mediaTime, report)) + { + report.HasEditList = true; + if (mediaTime < 0 || (mediaTime >= decode && decode > 0)) report.Errors.Add($"edit list media_time {mediaTime} outside the media"); + if (movieScale > 0) report.DurationTicks = (long)Math.Round(segment * (double)mediaScale / movieScale); + } + var seconds = mediaScale == 0 ? 0 : report.DurationTicks / (double)mediaScale; + report.AverageBitrate = seconds > 0 ? bytes * 8 / seconds : 0; + + if (config is not null) CheckSamplePayloads(stream, samples, config.Codec, report); + } + + private static DecoderConfiguration? InspectSampleDescription(Mp4Box stbl, Mp4Report report) + { + var stsd = stbl.Child("stsd"); + if (stsd is null || stsd.Children.Count == 0) + { + report.Errors.Add("stsd missing or empty"); + return null; + } + var entry = stsd.Children[0]; + report.SampleEntry = entry.Type; + var payload = entry.Payload.Span; + if (payload.Length < 78) + { + report.Errors.Add($"sample entry '{entry.Type}' truncated"); + return null; + } + report.Width = BinaryPrimitives.ReadUInt16BigEndian(payload[24..]); + report.Height = BinaryPrimitives.ReadUInt16BigEndian(payload[26..]); + var configBox = entry.Type switch + { + "avc1" or "avc3" => "avcC", + "hvc1" or "hev1" => "hvcC", + "av01" => "av1C", + _ => null, + }; + if (configBox is null) + { + report.Warnings.Add($"sample entry '{entry.Type}' is not validated"); + return null; + } + var box = entry.Child(configBox); + if (box is null) + { + report.Errors.Add($"{configBox} missing in '{entry.Type}'"); + return null; + } + if (!DecoderConfiguration.TryParse(configBox, box.Payload.Span, out var config, out var error) || config is null) + { + report.Errors.Add($"{configBox} unparseable: {error}"); + return null; + } + if (error is not null) report.Errors.Add(error); + report.Codec = config.Codec; + report.CodecString = config.CodecString; + report.BitDepth = config.BitDepth; + if (config.Width != report.Width || config.Height != report.Height) + report.Errors.Add($"{configBox} declares {config.Width}x{config.Height}, sample entry {report.Width}x{report.Height}"); + return config; + } + + /// Walks the NAL length prefixes / OBU sizes of the first samples and checks the first picture is a random access point. + private static void CheckSamplePayloads(Stream stream, Mp4SampleInfo[] samples, VideoCodec codec, Mp4Report report) + { + var length = stream.Length; + for (var i = 0; i < Math.Min(CheckedSamples, samples.Length); i++) + { + var s = samples[i]; + if (s.Size <= 0 || s.Offset + s.Size > length || s.Size > 64 << 20) continue; + var data = new byte[s.Size]; + stream.Position = s.Offset; + stream.ReadExactly(data); + if (codec == VideoCodec.Av1) + { + var obus = Av1ObuParser.Split(data); + if (obus.Count == 0 || obus[^1].Offset + obus[^1].TotalLength != data.Length) + report.Errors.Add($"sample {i + 1}: OBU sizes do not cover the sample"); + continue; + } + if (!AnnexBParser.TrySplitLengthPrefixed(data, 4, out var units)) + { + report.Errors.Add($"sample {i + 1}: NAL length prefixes do not cover the sample"); + continue; + } + if (i != 0) continue; + var randomAccess = false; + foreach (var u in units) + { + var h = data[u.Offset]; + randomAccess |= codec == VideoCodec.H264 + ? AnnexBParser.H264Type(h) == H264NalUnitType.IdrSlice + : AnnexBParser.HevcType(h) is >= HevcNalUnitType.BlaWLp and <= HevcNalUnitType.Cra; + } + if (!randomAccess) report.Warnings.Add("first sample contains no IDR/IRAP picture"); + } + } + + private static uint[]? ReadTimeToSample(Mp4Box stbl, Mp4Report report) + { + var box = stbl.Child("stts"); + if (box is null) + { + report.Errors.Add("stts missing"); + return null; + } + var p = box.Payload.Span; + if (!Entries(p, 8, report, "stts", out var entries)) return null; + var list = new List(); + for (var i = 0; i < entries; i++) + { + var n = BinaryPrimitives.ReadUInt32BigEndian(p[(8 + i * 8)..]); + var delta = BinaryPrimitives.ReadUInt32BigEndian(p[(12 + i * 8)..]); + if (list.Count + (long)n > 50_000_000) + { + report.Errors.Add("stts sample count implausible"); + return null; + } + for (var k = 0; k < n; k++) list.Add(delta); + } + return [.. list]; + } + + private static long[]? ReadCompositionOffsets(Mp4Box stbl, int count, Mp4Report report) + { + var box = stbl.Child("ctts"); + if (box is null) return null; + var p = box.Payload.Span; + if (!Entries(p, 8, report, "ctts", out var entries)) return null; + var signed = p[0] == 1; + var result = new long[count]; + long covered = 0; + for (var i = 0; i < entries; i++) + { + var n = BinaryPrimitives.ReadUInt32BigEndian(p[(8 + i * 8)..]); + var raw = BinaryPrimitives.ReadUInt32BigEndian(p[(12 + i * 8)..]); + long offset = signed ? (int)raw : raw; + for (var k = covered; k < Math.Min(count, covered + n); k++) result[k] = offset; + covered += n; + } + report.HasCompositionOffsets = true; + if (covered != count) report.Errors.Add($"ctts covers {covered} of {count} samples"); + return result; + } + + private static List? ReadSyncSamples(Mp4Box stbl, int count, Mp4Report report) + { + var box = stbl.Child("stss"); + if (box is null) return null; + var p = box.Payload.Span; + if (!Entries(p, 4, report, "stss", out var entries)) return []; + var list = new List(entries); + for (var i = 0; i < entries; i++) + { + var s = BinaryPrimitives.ReadUInt32BigEndian(p[(8 + i * 4)..]); + if (s == 0 || s > count || (list.Count > 0 && s <= list[^1])) + { + report.Errors.Add($"stss entry {s} out of order or range"); + break; + } + list.Add(s); + } + return list; + } + + private static uint[]? ReadSampleSizes(Mp4Box stbl, Mp4Report report) + { + var box = stbl.Child("stsz") ?? stbl.Child("stz2"); + if (box is null) + { + report.Errors.Add("stsz missing"); + return null; + } + var p = box.Payload.Span; + if (p.Length < 12) + { + report.Errors.Add($"{box.Type} truncated"); + return null; + } + var count = BinaryPrimitives.ReadUInt32BigEndian(p[8..]); + if (count > 50_000_000) + { + report.Errors.Add($"{box.Type} sample count implausible"); + return null; + } + var sizes = new uint[count]; + if (box.Type == "stsz") + { + var constant = BinaryPrimitives.ReadUInt32BigEndian(p[4..]); + if (constant != 0) + { + Array.Fill(sizes, constant); + return sizes; + } + if (p.Length < 12 + count * 4L) + { + report.Errors.Add("stsz truncated"); + return null; + } + for (var i = 0; i < count; i++) sizes[i] = BinaryPrimitives.ReadUInt32BigEndian(p[(12 + i * 4)..]); + return sizes; + } + var field = p[7]; + if (field is not (4 or 8 or 16) || p.Length < 12 + (count * field + 7) / 8) + { + report.Errors.Add("stz2 invalid"); + return null; + } + for (var i = 0; i < count; i++) + { + sizes[i] = field switch + { + 16 => BinaryPrimitives.ReadUInt16BigEndian(p[(12 + i * 2)..]), + 8 => p[12 + i], + _ => (uint)((p[12 + i / 2] >> ((i & 1) == 0 ? 4 : 0)) & 0x0F), + }; + } + return sizes; + } + + private static long[]? ReadChunkOffsets(Mp4Box stbl, Mp4Report report) + { + var box = stbl.Child("stco") ?? stbl.Child("co64"); + if (box is null) + { + report.Errors.Add("stco/co64 missing"); + return null; + } + var wide = box.Type == "co64"; + report.UsesCo64 = wide; + var p = box.Payload.Span; + if (!Entries(p, wide ? 8 : 4, report, box.Type, out var entries)) return null; + var offsets = new long[entries]; + for (var i = 0; i < entries; i++) + { + offsets[i] = wide + ? (long)Math.Min(BinaryPrimitives.ReadUInt64BigEndian(p[(8 + i * 8)..]), long.MaxValue) + : BinaryPrimitives.ReadUInt32BigEndian(p[(8 + i * 4)..]); + } + return offsets; + } + + private static uint[]? ReadSampleToChunk(Mp4Box stbl, int chunks, Mp4Report report) + { + var box = stbl.Child("stsc"); + if (box is null) + { + report.Errors.Add("stsc missing"); + return null; + } + var p = box.Payload.Span; + if (!Entries(p, 12, report, "stsc", out var entries)) return null; + var perChunk = new uint[chunks]; + for (var i = 0; i < entries; i++) + { + var first = BinaryPrimitives.ReadUInt32BigEndian(p[(8 + i * 12)..]); + var samples = BinaryPrimitives.ReadUInt32BigEndian(p[(12 + i * 12)..]); + var next = i + 1 < entries ? BinaryPrimitives.ReadUInt32BigEndian(p[(8 + (i + 1) * 12)..]) : (uint)chunks + 1; + if (first == 0 || next <= first || (i == 0 && first != 1)) + { + report.Errors.Add("stsc first_chunk values invalid"); + return null; + } + if (BinaryPrimitives.ReadUInt32BigEndian(p[(16 + i * 12)..]) != 1) report.Errors.Add("stsc references a sample description other than 1"); + for (var c = first; c < next && c <= chunks; c++) perChunk[c - 1] = samples; + } + return perChunk; + } + + private static bool ReadEditList(ReadOnlySpan p, out long segment, out long mediaTime, Mp4Report report) + { + segment = mediaTime = 0; + var wide = p.Length > 0 && p[0] == 1; + if (!Entries(p, wide ? 20 : 12, report, "elst", out var entries) || entries == 0) return false; + if (entries > 1) report.Warnings.Add($"edit list has {entries} entries; only the first is evaluated"); + segment = wide ? (long)Math.Min(BinaryPrimitives.ReadUInt64BigEndian(p[8..]), long.MaxValue) : BinaryPrimitives.ReadUInt32BigEndian(p[8..]); + mediaTime = wide ? BinaryPrimitives.ReadInt64BigEndian(p[16..]) : BinaryPrimitives.ReadInt32BigEndian(p[12..]); + return true; + } + + /// Reads entry_count of a full box and checks the table fits its payload. + private static bool Entries(ReadOnlySpan p, int entrySize, Mp4Report report, string type, out int entries) + { + entries = 0; + if (p.Length < 8) + { + report.Errors.Add($"{type} truncated"); + return false; + } + var n = BinaryPrimitives.ReadUInt32BigEndian(p[4..]); + if (n > (p.Length - 8) / entrySize) + { + report.Errors.Add($"{type} declares {n} entries but holds {(p.Length - 8) / entrySize}"); + return false; + } + entries = (int)n; + return true; + } + + private static uint ReadTimescale(ReadOnlySpan p, out long duration) + { + duration = 0; + if (p.Length < 20) return 0; + if (p[0] == 1) + { + if (p.Length < 32) return 0; + duration = (long)Math.Min(BinaryPrimitives.ReadUInt64BigEndian(p[24..]), long.MaxValue); + return BinaryPrimitives.ReadUInt32BigEndian(p[20..]); + } + duration = BinaryPrimitives.ReadUInt32BigEndian(p[16..]); + return BinaryPrimitives.ReadUInt32BigEndian(p[12..]); + } + + private static string? Handler(Mp4Box trak) + { + var hdlr = trak.Child("mdia")?.Child("hdlr"); + return hdlr is null || hdlr.Payload.Length < 12 ? null : FourCc(hdlr.Payload.Span[8..12]); + } + + private static bool InsideMdat(long start, long end, List<(long Start, long End)> mdats) + { + foreach (var (s, e) in mdats) + { + if (start >= s && end <= e) return true; + } + return false; + } + + private static string FourCc(ReadOnlySpan bytes) + { + Span chars = stackalloc char[4]; + for (var i = 0; i < 4; i++) chars[i] = bytes[i] is >= 0x20 and < 0x7F ? (char)bytes[i] : '?'; + return new string(chars); + } +} diff --git a/tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs b/tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs new file mode 100644 index 00000000..d505885b --- /dev/null +++ b/tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs @@ -0,0 +1,240 @@ +using System.Buffers.Binary; +using Titano.Core.Video; +using Titano.Core.Video.Bitstream; +using Titano.Core.Video.Mp4; +using Titano.Testing; + +namespace Titano.Core.Tests.Video; + +public sealed class Mp4MuxerTests(TestContext context) +{ + private static long Duration(int i) => 3000 + i % 3 * 1500; + + [Test] + public void H264WithVariableDurationsInspectsClean() + { + using var ms = new MemoryStream(); + var muxer = new Mp4Muxer(ms, new VideoTrackConfig { Codec = VideoCodec.H264, Width = 1920, Height = 1080 }); + var expected = new List(); + long total = 0; + for (var i = 0; i < 30; i++) + { + var idr = i % 10 == 0; + var slice = SyntheticStreams.H264Slice(idr, 200 + i * 7, i); + List nals = [SyntheticStreams.H264AccessUnitDelimiter()]; + if (idr) nals.AddRange([SyntheticStreams.H264Sps(), SyntheticStreams.H264Pps()]); + nals.Add(slice); + var annexB = AnnexBParser.ToAnnexB(nals); + // Alternate input formats: the muxer accepts both. + muxer.WriteSample(i % 2 == 0 ? annexB : AnnexBParser.ToLengthPrefixed(annexB), Duration(i), idr); + expected.Add(AnnexBParser.ToLengthPrefixed(AnnexBParser.ToAnnexB([slice]))); + total += Duration(i); + } + muxer.Complete(); + + var report = Mp4Inspector.Inspect(ms); + AssertClean(report); + Assert.Equal("avc1", report.SampleEntry); + Assert.Equal("avc1.640029", report.CodecString); + Assert.Equal(1920, report.Width); + Assert.Equal(1080, report.Height); + Assert.Equal(30, report.FrameCount); + Assert.Equal(3, report.SyncSampleCount); + Assert.Equal(90000u, report.TimeScale); + Assert.Equal(total, report.DurationTicks); + Assert.False(report.FastStart); + Assert.False(report.HasEditList); + for (var i = 0; i < 30; i++) + { + Assert.Equal(Duration(i), (long)report.Samples[i].Duration); + Assert.BytesEqual(expected[i], ReadSample(ms, report.Samples[i])); + } + } + + [Test] + public void HevcMain10InspectsClean() + { + using var ms = new MemoryStream(); + var muxer = new Mp4Muxer(ms, new VideoTrackConfig { Codec = VideoCodec.Hevc, Width = 3840, Height = 2160 }); + for (var i = 0; i < 12; i++) + { + var idr = i % 6 == 0; + List nals = idr ? [SyntheticStreams.HevcVps(), SyntheticStreams.HevcSps(3840, 2160), SyntheticStreams.HevcPps()] : []; + nals.Add(SyntheticStreams.HevcSlice(idr, 500, i)); + muxer.WriteSample(AnnexBParser.ToAnnexB(nals), 1500, idr); + } + muxer.Complete(); + var report = Mp4Inspector.Inspect(ms); + AssertClean(report); + Assert.Equal("hvc1", report.SampleEntry); + Assert.Equal(VideoCodec.Hevc, report.Codec); + Assert.Equal("hvc1.2.4.L153.90", report.CodecString); + Assert.Equal(3840, report.Width); + Assert.Equal(2160, report.Height); + Assert.Equal(10, report.BitDepth); + Assert.Equal(12, report.FrameCount); + Assert.Equal(18000L, report.DurationTicks); + Assert.Approximately(0.2, report.Duration.TotalSeconds, 1e-9); + } + + [Test] + public void Av1InspectsCleanAndGetsSizeFields() + { + using var ms = new MemoryStream(); + var muxer = new Mp4Muxer(ms, new VideoTrackConfig { Codec = VideoCodec.Av1, Width = 1280, Height = 720 }); + for (var i = 0; i < 10; i++) + { + var key = i == 0; + var frame = SyntheticStreams.Random(300, i); + // Last OBU of the temporal unit without obu_size, as encoders may emit it. + byte[] lastWithoutSize = [(byte)Av1ObuType.Frame << 3, .. frame]; + var tu = key + ? SyntheticStreams.Concat(SyntheticStreams.Av1TemporalDelimiter(), SyntheticStreams.Av1SequenceHeaderObu(1280, 720, 5), lastWithoutSize) + : SyntheticStreams.Concat(SyntheticStreams.Av1TemporalDelimiter(), lastWithoutSize); + muxer.WriteSample(tu, 3003, key); + } + muxer.Complete(); + var report = Mp4Inspector.Inspect(ms); + AssertClean(report); + Assert.Equal("av01", report.SampleEntry); + Assert.Equal("av01.0.05M.08", report.CodecString); + Assert.Equal(1280, report.Width); + Assert.Equal(720, report.Height); + Assert.Equal(10, report.FrameCount); + Assert.Equal(1, report.SyncSampleCount); + var second = ReadSample(ms, report.Samples[1]); + Assert.True(Av1ObuParser.TryReadObu(second, 0, out var obu)); + Assert.True(obu.HasSizeField); + Assert.Equal(Av1ObuType.Frame, obu.Type); + Assert.Equal(second.Length, obu.TotalLength); + } + + [Test] + public void FastStartMovesMoovAndKeepsOffsetsValid() + { + var path = Path.Combine(context.CreateTempDirectory(), "fast.mp4"); + var expected = new List(); + using (var fs = new FileStream(path, FileMode.CreateNew, FileAccess.ReadWrite)) + { + var muxer = new Mp4Muxer(fs, new VideoTrackConfig { Codec = VideoCodec.H264, Width = 1280, Height = 720, FastStart = true }); + for (var i = 0; i < 90; i++) + { + var idr = i % 30 == 0; + var slice = SyntheticStreams.H264Slice(idr, 4000 + i, i); + List nals = idr ? [SyntheticStreams.H264Sps(1280, 720, 77, 31), SyntheticStreams.H264Pps(), slice] : [slice]; + muxer.WriteSample(AnnexBParser.ToAnnexB(nals), 3000, idr); + expected.Add(AnnexBParser.ToLengthPrefixed(AnnexBParser.ToAnnexB([slice]))); + } + muxer.Complete(); + } + var report = Mp4Inspector.Inspect(path); + AssertClean(report); + Assert.True(report.FastStart); + Assert.SequenceEqual(["ftyp", "moov", "free", "mdat"], report.Boxes.Select(b => b.Type)); + Assert.Equal(new FileInfo(path).Length, report.Boxes.Sum(b => b.Size)); + using var read = File.OpenRead(path); + for (var i = 0; i < expected.Count; i++) Assert.BytesEqual(expected[i], ReadSample(read, report.Samples[i])); + } + + [Test] + public void CompositionOffsetsProduceCttsAndEditList() + { + // Decode order I0 P3 B1 B2 P6 B4 B5 with unit durations of 3000 ticks; encoder gives PTS − DTS. + int[] pts = [0, 3, 1, 2, 6, 4, 5]; + using var ms = new MemoryStream(); + var muxer = new Mp4Muxer(ms, new VideoTrackConfig { Codec = VideoCodec.H264, Width = 1920, Height = 1080 }); + for (var i = 0; i < pts.Length; i++) + { + List nals = i == 0 ? [SyntheticStreams.H264Sps(), SyntheticStreams.H264Pps()] : []; + nals.Add(SyntheticStreams.H264Slice(i == 0, 100, i)); + muxer.WriteSample(AnnexBParser.ToAnnexB(nals), 3000, i == 0, (pts[i] - i) * 3000L); + } + muxer.Complete(); + var report = Mp4Inspector.Inspect(ms); + AssertClean(report); + Assert.True(report.HasCompositionOffsets); + Assert.True(report.HasEditList); + Assert.Equal(7 * 3000L, report.DurationTicks); + // After the shift every composition time is non-negative and I0 presents first. + var cts = report.Samples.Select(s => s.DecodeTime + s.CompositionOffset).ToArray(); + Assert.Equal(cts.Min(), cts[0]); + Assert.True(report.Samples.All(s => s.CompositionOffset >= 0)); + } + + [Test] + public void LargeFilesUseCo64AndLargeMdat() + { + const int sampleSize = 1 << 20; + const int samples = 4200; // ≈ 4.4 GB of payload, dropped by the sparse stream + var slice = new byte[sampleSize]; + BinaryPrimitives.WriteInt32BigEndian(slice, sampleSize - 4); + slice[4] = 0x41; // non-IDR slice header + var first = SyntheticStreams.Concat( + AnnexBParser.ToLengthPrefixed(AnnexBParser.ToAnnexB([SyntheticStreams.H264Sps(), SyntheticStreams.H264Pps()])), slice); + first[first.Length - sampleSize + 4] = 0x65; + + using var sparse = new SparseStream(); + var muxer = new Mp4Muxer(sparse, new VideoTrackConfig { Codec = VideoCodec.H264, Width = 1920, Height = 1080 }); + muxer.WriteSample(first, 3000, true); + for (var i = 1; i < samples; i++) muxer.WriteSample(slice, 3000, i % 60 == 0); + muxer.Complete(); + + var report = Mp4Inspector.Inspect(sparse); + AssertClean(report); + Assert.True(report.UsesCo64); + Assert.True(report.UsesLargeMdat); + Assert.Equal(samples, report.FrameCount); + Assert.Greater(report.Samples[^1].Offset, uint.MaxValue); + Assert.Equal((long)sampleSize * samples, report.MediaBytes); + } + + [Test] + public void CompleteWithoutParameterSetsThrows() + { + using var ms = new MemoryStream(); + var muxer = new Mp4Muxer(ms, new VideoTrackConfig { Codec = VideoCodec.H264, Width = 64, Height = 64 }); + muxer.WriteSample(AnnexBParser.ToAnnexB([SyntheticStreams.H264Slice(true, 10, 1)]), 3000, true); + Assert.Throws(muxer.Complete); + Assert.Throws(() => muxer.WriteSample([0, 0, 1, 0x65, 1], 0, true)); + } + + [Test] + public void InspectorSurvivesTruncationAndCorruption() + { + using var ms = new MemoryStream(); + var muxer = new Mp4Muxer(ms, new VideoTrackConfig { Codec = VideoCodec.H264, Width = 1920, Height = 1080 }); + for (var i = 0; i < 5; i++) + { + List nals = i == 0 ? [SyntheticStreams.H264Sps(), SyntheticStreams.H264Pps()] : []; + nals.Add(SyntheticStreams.H264Slice(i == 0, 50, i)); + muxer.WriteSample(AnnexBParser.ToAnnexB(nals), 3000, i == 0); + } + muxer.Complete(); + var valid = ms.ToArray(); + for (var cut = 0; cut < valid.Length; cut += 7) + { + var report = Mp4Inspector.Inspect(new MemoryStream(valid, 0, cut)); + Assert.False(report.IsValid, $"truncated at {cut} should not validate"); + } + var rng = new Random(11); + for (var trial = 0; trial < 500; trial++) + { + var corrupt = (byte[])valid.Clone(); + for (var k = 0; k < 3; k++) corrupt[rng.Next(corrupt.Length)] = (byte)rng.Next(256); + Mp4Inspector.Inspect(new MemoryStream(corrupt)).ToString(); + } + } + + private static void AssertClean(Mp4Report report) + { + if (!report.IsValid) Assert.Fail("Inspector errors: " + string.Join("; ", report.Errors)); + } + + private static byte[] ReadSample(Stream stream, Mp4SampleInfo sample) + { + var data = new byte[sample.Size]; + stream.Position = sample.Offset; + stream.ReadExactly(data); + return data; + } +} diff --git a/tests/Titano.Core.Tests/Video/SparseStream.cs b/tests/Titano.Core.Tests/Video/SparseStream.cs new file mode 100644 index 00000000..d470d960 --- /dev/null +++ b/tests/Titano.Core.Tests/Video/SparseStream.cs @@ -0,0 +1,76 @@ +namespace Titano.Core.Tests.Video; + +/// +/// A seekable stream that keeps small writes (headers, length prefixes, moov) and drops large payload writes, +/// which read back as zeros. Lets tests produce multi-gigabyte container layouts without the bytes. +/// +internal sealed class SparseStream : Stream +{ + private const int PageSize = 1024; + private const int LargeWrite = 64 * 1024; + private readonly Dictionary _pages = []; + private long _length; + private long _position; + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => true; + public override long Length => _length; + + public override long Position + { + get => _position; + set => _position = value; + } + + public override void Flush() + { + } + + public override long Seek(long offset, SeekOrigin origin) => _position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _position + offset, + _ => _length + offset, + }; + + public override void SetLength(long value) => _length = value; + + public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + var n = (int)Math.Clamp(_length - _position, 0, buffer.Length); + for (var i = 0; i < n;) + { + var at = _position + i; + var inPage = (int)(at % PageSize); + var chunk = Math.Min(n - i, PageSize - inPage); + if (_pages.TryGetValue(at / PageSize, out var page)) page.AsSpan(inPage, chunk).CopyTo(buffer.Slice(i, chunk)); + else buffer.Slice(i, chunk).Clear(); + i += chunk; + } + _position += n; + return n; + } + + public override void Write(byte[] buffer, int offset, int count) => Write(buffer.AsSpan(offset, count)); + + public override void Write(ReadOnlySpan buffer) + { + if (buffer.Length < LargeWrite) + { + for (var i = 0; i < buffer.Length;) + { + var at = _position + i; + var inPage = (int)(at % PageSize); + var chunk = Math.Min(buffer.Length - i, PageSize - inPage); + if (!_pages.TryGetValue(at / PageSize, out var page)) _pages[at / PageSize] = page = new byte[PageSize]; + buffer.Slice(i, chunk).CopyTo(page.AsSpan(inPage, chunk)); + i += chunk; + } + } + _position += buffer.Length; + _length = Math.Max(_length, _position); + } +} From bc543f99b6c50d3320cf9760e31039b1c7d9728f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 09:35:00 +0000 Subject: [PATCH 3/9] feat(video): add H.264/HEVC/AV1 level tables with level selection and size capping Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Video/CodecLimits.cs | 214 ++++++++++++++++++ .../Video/CodecLimitsTests.cs | 70 ++++++ 2 files changed, 284 insertions(+) create mode 100644 src/Titano.Core/Video/CodecLimits.cs create mode 100644 tests/Titano.Core.Tests/Video/CodecLimitsTests.cs diff --git a/src/Titano.Core/Video/CodecLimits.cs b/src/Titano.Core/Video/CodecLimits.cs new file mode 100644 index 00000000..4c95e39c --- /dev/null +++ b/src/Titano.Core/Video/CodecLimits.cs @@ -0,0 +1,214 @@ +using System.Globalization; + +namespace Titano.Core.Video; + +/// A codec level such as H.264 4.1, HEVC 5.1 or AV1 5.0. +public readonly record struct CodecLevel(VideoCodec Codec, int Major, int Minor) +{ + public string Name => string.Create(CultureInfo.InvariantCulture, $"{Major}.{Minor}"); + + /// H.264 level_idc (10 × level), also the eAVEncH264VLevel value. + public int H264LevelIdc => Major * 10 + Minor; + + /// HEVC general_level_idc (30 × level), also the eAVEncH265VLevel value. + public int HevcLevelIdc => Major * 30 + Minor * 3; + + /// AV1 seq_level_idx. + public int Av1SeqLevelIdx => (Major - 2) * 4 + Minor; + + public override string ToString() => Name; +} + +/// Level constraints a configuration can break. +[Flags] +public enum LevelLimit +{ + None = 0, + PictureSize = 1, + Dimension = 2, + SampleRate = 4, + BitRate = 8, +} + +/// Outcome of . +/// Lowest level that fits, or null when even the highest level is exceeded. +/// Highest level defined for the codec. +/// Limits the highest level violates ( when a level fits). +/// Largest even width with the same aspect ratio that fits the highest level at this frame rate. +/// Height matching . +/// Bit-rate ceiling of the highest level for the profile, in bits per second. +public sealed record LevelCheck(CodecLevel? Level, CodecLevel HighestLevel, LevelLimit Exceeded, int SuggestedWidth, int SuggestedHeight, long MaxBitRate) +{ + public bool Fits => Level is not null; +} + +/// +/// Level tables for H.264 (Table A-1), HEVC (Tables A.8/A.9, Main/Main10, main tier) and AV1 (Annex A.3, main tier): +/// picks the lowest level for a size, frame rate and bit rate, and proposes a capped size when none fits. +/// +public static class CodecLimits +{ + // Major, Minor, max luma picture samples, max luma samples per second, max bit rate (kbit/s, before the profile factor), + // max width, max height (0 = derived from the picture size as sqrt(8 × size)). + private readonly record struct Limits(int Major, int Minor, long PictureSamples, long SampleRate, long BitRateKbps, int MaxWidth, int MaxHeight); + + // H.264 limits are in macroblocks: MaxFS × 256 samples, MaxMBPS × 256 samples/s. + private static readonly Limits[] H264 = + [ + new(1, 0, 99 * 256L, 1485 * 256L, 64, 0, 0), + new(1, 1, 396 * 256L, 3000 * 256L, 192, 0, 0), + new(1, 2, 396 * 256L, 6000 * 256L, 384, 0, 0), + new(1, 3, 396 * 256L, 11880 * 256L, 768, 0, 0), + new(2, 0, 396 * 256L, 11880 * 256L, 2000, 0, 0), + new(2, 1, 792 * 256L, 19800 * 256L, 4000, 0, 0), + new(2, 2, 1620 * 256L, 20250 * 256L, 4000, 0, 0), + new(3, 0, 1620 * 256L, 40500 * 256L, 10000, 0, 0), + new(3, 1, 3600 * 256L, 108000 * 256L, 14000, 0, 0), + new(3, 2, 5120 * 256L, 216000 * 256L, 20000, 0, 0), + new(4, 0, 8192 * 256L, 245760 * 256L, 20000, 0, 0), + new(4, 1, 8192 * 256L, 245760 * 256L, 50000, 0, 0), + new(4, 2, 8704 * 256L, 522240 * 256L, 50000, 0, 0), + new(5, 0, 22080 * 256L, 589824 * 256L, 135000, 0, 0), + new(5, 1, 36864 * 256L, 983040 * 256L, 240000, 0, 0), + new(5, 2, 36864 * 256L, 2073600 * 256L, 240000, 0, 0), + new(6, 0, 139264 * 256L, 4177920 * 256L, 240000, 0, 0), + new(6, 1, 139264 * 256L, 8355840 * 256L, 480000, 0, 0), + new(6, 2, 139264 * 256L, 16711680 * 256L, 800000, 0, 0), + ]; + + private static readonly Limits[] Hevc = + [ + new(1, 0, 36864, 552960, 128, 0, 0), + new(2, 0, 122880, 3686400, 1500, 0, 0), + new(2, 1, 245760, 7372800, 3000, 0, 0), + new(3, 0, 552960, 16588800, 6000, 0, 0), + new(3, 1, 983040, 33177600, 10000, 0, 0), + new(4, 0, 2228224, 66846720, 12000, 0, 0), + new(4, 1, 2228224, 133693440, 20000, 0, 0), + new(5, 0, 8912896, 267386880, 25000, 0, 0), + new(5, 1, 8912896, 534773760, 40000, 0, 0), + new(5, 2, 8912896, 1069547520, 60000, 0, 0), + new(6, 0, 35651584, 1069547520, 60000, 0, 0), + new(6, 1, 35651584, 2139095040, 120000, 0, 0), + new(6, 2, 35651584, 4278190080, 240000, 0, 0), + ]; + + // AV1: MaxPicSize, MaxDisplayRate, main-tier MaxBitrate, MaxHSize, MaxVSize. + private static readonly Limits[] Av1 = + [ + new(2, 0, 147456, 4423680, 1500, 2048, 1152), + new(2, 1, 278784, 8363520, 3000, 2816, 1584), + new(3, 0, 665856, 19975680, 6000, 4352, 2448), + new(3, 1, 1065024, 31950720, 10000, 5504, 3096), + new(4, 0, 2359296, 70778880, 12000, 6144, 3456), + new(4, 1, 2359296, 141557760, 20000, 6144, 3456), + new(5, 0, 8912896, 267386880, 30000, 8192, 4352), + new(5, 1, 8912896, 534773760, 40000, 8192, 4352), + new(5, 2, 8912896, 1069547520, 60000, 8192, 4352), + new(5, 3, 8912896, 1069547520, 60000, 8192, 4352), + new(6, 0, 35651584, 1069547520, 60000, 16384, 8704), + new(6, 1, 35651584, 2139095040, 100000, 16384, 8704), + new(6, 2, 35651584, 4278190080, 160000, 16384, 8704), + new(6, 3, 35651584, 4278190080, 160000, 16384, 8704), + ]; + + /// Levels defined for , lowest first. + public static IReadOnlyList Levels(VideoCodec codec) => [.. Table(codec).Select(l => new CodecLevel(codec, l.Major, l.Minor))]; + + /// Finds the lowest level for the configuration; ≤ 0 skips the bit-rate check. + public static LevelCheck Check(VideoCodec codec, int width, int height, double fps, long bitRate, CodecProfile profile = CodecProfile.Auto) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(width); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(height); + if (!(fps > 0)) throw new ArgumentOutOfRangeException(nameof(fps)); + var table = Table(codec); + var factor = BitRateFactor(codec, profile); + foreach (var limits in table) + { + if (Violations(codec, limits, width, height, fps, bitRate, factor) == LevelLimit.None) + return new LevelCheck(new CodecLevel(codec, limits.Major, limits.Minor), Highest(codec), LevelLimit.None, width, height, MaxBitRate(table[^1], factor)); + } + var top = table[^1]; + var exceeded = Violations(codec, top, width, height, fps, bitRate, factor); + var (w, h) = Cap(codec, top, width, height, fps); + return new LevelCheck(null, Highest(codec), exceeded, w, h, MaxBitRate(top, factor)); + } + + /// Largest even size with the aspect ratio of × that fits the highest level. + public static (int Width, int Height) CapResolution(VideoCodec codec, int width, int height, double fps) + { + var check = Check(codec, width, height, fps, 0); + return (check.SuggestedWidth, check.SuggestedHeight); + } + + private static (int Width, int Height) Cap(VideoCodec codec, Limits top, int width, int height, double fps) + { + // Fitting is monotonic in the scale, so bisect on half the width (keeps sizes even). + int lo = 0, hi = width / 2; + while (lo < hi) + { + var mid = (lo + hi + 1) / 2; + var (w, h) = Scaled(width, height, mid * 2); + if ((Violations(codec, top, w, h, fps, 0, 1) & ~LevelLimit.BitRate) == LevelLimit.None) lo = mid; + else hi = mid - 1; + } + return lo == 0 ? (0, 0) : Scaled(width, height, lo * 2); + } + + private static (int Width, int Height) Scaled(int width, int height, int newWidth) + { + var h = (int)Math.Max(2, Math.Round(newWidth * (double)height / width / 2) * 2); + return (newWidth, h); + } + + private static LevelLimit Violations(VideoCodec codec, Limits l, int width, int height, double fps, long bitRate, double factor) + { + long w = width, h = height; + if (codec == VideoCodec.H264) + { + w = (width + 15) / 16 * 16; + h = (height + 15) / 16 * 16; + } + else if (codec == VideoCodec.Hevc) + { + w = (width + 7) / 8 * 8; // MinCbSizeY of 8 + h = (height + 7) / 8 * 8; + } + var size = w * h; + var result = LevelLimit.None; + if (size > l.PictureSamples) result |= LevelLimit.PictureSize; + if (l.MaxWidth > 0) + { + if (w > l.MaxWidth || h > l.MaxHeight) result |= LevelLimit.Dimension; + } + else + { + // H.264 A.3.1 / HEVC A.4.1: each side at most sqrt(8 × maximum picture size). + var side = Math.Sqrt(8.0 * l.PictureSamples); + if (w > side || h > side) result |= LevelLimit.Dimension; + } + if (size * fps > l.SampleRate * (1 + 1e-9)) result |= LevelLimit.SampleRate; + if (bitRate > 0 && bitRate > MaxBitRate(l, factor)) result |= LevelLimit.BitRate; + return result; + } + + private static long MaxBitRate(Limits l, double factor) => (long)(l.BitRateKbps * 1000 * factor); + + /// cpbBrVclFactor relative to 1000: H.264 High is 1250 (A.3.3.2); Baseline/Main, HEVC Main/Main10 and AV1 main are 1000. + private static double BitRateFactor(VideoCodec codec, CodecProfile profile) => + codec == VideoCodec.H264 && profile is CodecProfile.H264High or CodecProfile.Auto ? 1.25 : 1.0; + + private static CodecLevel Highest(VideoCodec codec) + { + var top = Table(codec)[^1]; + return new CodecLevel(codec, top.Major, top.Minor); + } + + private static Limits[] Table(VideoCodec codec) => codec switch + { + VideoCodec.H264 => H264, + VideoCodec.Hevc => Hevc, + VideoCodec.Av1 => Av1, + _ => throw new ArgumentException($"No level table for {codec}", nameof(codec)), + }; +} diff --git a/tests/Titano.Core.Tests/Video/CodecLimitsTests.cs b/tests/Titano.Core.Tests/Video/CodecLimitsTests.cs new file mode 100644 index 00000000..27cc9521 --- /dev/null +++ b/tests/Titano.Core.Tests/Video/CodecLimitsTests.cs @@ -0,0 +1,70 @@ +using Titano.Core.Video; +using Titano.Testing; + +namespace Titano.Core.Tests.Video; + +public sealed class CodecLimitsTests +{ + [Theory] + [Case(VideoCodec.H264, 1920, 1080, 30.0, 6_220_800L, "4.0", Name = "1080p30 H.264 (0.10 bpp)")] + [Case(VideoCodec.H264, 1920, 1080, 60.0, 12_441_600L, "4.2")] + [Case(VideoCodec.H264, 1280, 720, 30.0, 0L, "3.1")] + [Case(VideoCodec.H264, 3840, 2160, 30.0, 24_883_200L, "5.1")] + [Case(VideoCodec.H264, 1920, 1080, 30.0, 40_000_000L, "4.1", Name = "1080p30 H.264 at 40 Mbit/s")] + [Case(VideoCodec.Hevc, 3840, 2160, 60.0, 34_836_480L, "5.1", Name = "4K60 HEVC (0.07 bpp)")] + [Case(VideoCodec.Hevc, 7680, 4320, 30.0, 69_672_960L, "6.1", Name = "8K30 HEVC (0.07 bpp)")] + [Case(VideoCodec.Hevc, 7680, 4320, 30.0, 50_000_000L, "6.0")] + [Case(VideoCodec.Hevc, 1920, 1080, 30.0, 0L, "4.0")] + [Case(VideoCodec.Av1, 1920, 1080, 30.0, 0L, "4.0")] + [Case(VideoCodec.Av1, 3840, 2160, 60.0, 29_859_840L, "5.1")] + [Case(VideoCodec.Av1, 640, 360, 24.0, 0L, "2.1")] + public void ChoosesLowestFittingLevel(VideoCodec codec, int width, int height, double fps, long bitRate, string level) + { + var check = CodecLimits.Check(codec, width, height, fps, bitRate); + Assert.True(check.Fits); + Assert.Equal(level, check.Level!.Value.Name); + Assert.Equal(LevelLimit.None, check.Exceeded); + Assert.Equal(width, check.SuggestedWidth); + } + + [Test] + public void LevelIdentifiersMatchTheBitstreamCodes() + { + Assert.Equal(41, new CodecLevel(VideoCodec.H264, 4, 1).H264LevelIdc); + Assert.Equal(153, new CodecLevel(VideoCodec.Hevc, 5, 1).HevcLevelIdc); + Assert.Equal(186, new CodecLevel(VideoCodec.Hevc, 6, 2).HevcLevelIdc); + Assert.Equal(13, new CodecLevel(VideoCodec.Av1, 5, 1).Av1SeqLevelIdx); + Assert.Equal(19, CodecLimits.Levels(VideoCodec.H264).Count); + Assert.Equal("6.3", CodecLimits.Levels(VideoCodec.Av1)[^1].Name); + } + + [Theory] + [Case(VideoCodec.H264, 16384, 8640, 30.0)] + [Case(VideoCodec.Hevc, 15360, 8640, 60.0)] + [Case(VideoCodec.Av1, 20000, 10000, 30.0)] + public void OversizedConfigurationsGetACappedSuggestion(VideoCodec codec, int width, int height, double fps) + { + var check = CodecLimits.Check(codec, width, height, fps, 0); + Assert.False(check.Fits); + Assert.True((check.Exceeded & (LevelLimit.PictureSize | LevelLimit.SampleRate | LevelLimit.Dimension)) != 0); + Assert.True(check.SuggestedWidth % 2 == 0 && check.SuggestedHeight % 2 == 0); + Assert.Approximately(width / (double)height, check.SuggestedWidth / (double)check.SuggestedHeight, 0.01); + Assert.True(CodecLimits.Check(codec, check.SuggestedWidth, check.SuggestedHeight, fps, 0).Fits); + // The suggestion is tight: a slightly larger size no longer fits. + var w = check.SuggestedWidth + 64; + var h = (int)Math.Round(w * (double)height / width); + Assert.False(CodecLimits.Check(codec, w, h, fps, 0).Fits); + } + + [Test] + public void ExcessiveBitRateIsReportedWithoutResizing() + { + var check = CodecLimits.Check(VideoCodec.H264, 1920, 1080, 30, 2_000_000_000, CodecProfile.H264High); + Assert.False(check.Fits); + Assert.Equal(LevelLimit.BitRate, check.Exceeded); + Assert.Equal(1920, check.SuggestedWidth); + Assert.Equal(1080, check.SuggestedHeight); + Assert.Equal(1_000_000_000L, check.MaxBitRate); + Assert.Equal(800_000_000L, CodecLimits.Check(VideoCodec.H264, 64, 64, 30, 0, CodecProfile.H264Main).MaxBitRate); + } +} From f9b742b3b7384197e45ea7314db251324c3f5b83 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 09:38:57 +0000 Subject: [PATCH 4/9] feat(video): add NV12/P010 BT.709 limited-range converter with SIMD luma and parallel rows Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Video/Yuv/Nv12Converter.cs | 299 ++++++++++++++++++ .../Video/Nv12ConverterTests.cs | 181 +++++++++++ 2 files changed, 480 insertions(+) create mode 100644 src/Titano.Core/Video/Yuv/Nv12Converter.cs create mode 100644 tests/Titano.Core.Tests/Video/Nv12ConverterTests.cs diff --git a/src/Titano.Core/Video/Yuv/Nv12Converter.cs b/src/Titano.Core/Video/Yuv/Nv12Converter.cs new file mode 100644 index 00000000..44b7f477 --- /dev/null +++ b/src/Titano.Core/Video/Yuv/Nv12Converter.cs @@ -0,0 +1,299 @@ +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using Titano.Core.Imaging; + +namespace Titano.Core.Video.Yuv; + +/// Position of 4:2:0 chroma samples relative to luma. +public enum ChromaSiting +{ + /// Co-sited with the left luma sample, vertically between rows (MPEG-2/H.264/HEVC default, chroma_sample_loc_type 0). + Left, + + /// Centred between four luma samples (JPEG/MPEG-1, chroma_sample_loc_type 1). + Center, +} + +/// +/// Linear-light RGB ⇄ BT.709 limited-range Y'CbCr 4:2:0: NV12 (8-bit) and P010 +/// (10-bit in the top bits of 16-bit words). The sRGB transfer is applied before the matrix, as encoders expect +/// display-referred input. Width and height must be even; luma is vectorised, rows run in parallel. +/// +public static class Nv12Converter +{ + private const float Kr = 0.2126f, Kg = 0.7152f, Kb = 0.0722f; + private const float CbScale = 1 / (2 * (1 - Kb)); // 1 / 1.8556 + private const float CrScale = 1 / (2 * (1 - Kr)); // 1 / 1.5748 + private const int ParallelThreshold = 1 << 16; + private static readonly float[] EncodeLut = BuildEncodeLut(); + + /// Bytes of an NV12 frame (elements of a P010 frame) for a row stride in bytes (elements). + public static int FrameLength(int stride, int height) => checked(stride * height * 3 / 2); + + /// Writes NV12: Y rows of bytes, then interleaved CbCr rows of the same stride. + public static unsafe void ToNv12(ImageBuffer source, Span destination, ChromaSiting siting = ChromaSiting.Left, int stride = 0) + { + stride = Validate(source.Width, source.Height, stride, destination.Length); + fixed (byte* d = destination) Convert(source, (nint)d, stride, siting, tenBit: false); + } + + /// Writes P010: as NV12 with 16-bit elements holding 10-bit codes in the top bits; counts elements. + public static unsafe void ToP010(ImageBuffer source, Span destination, ChromaSiting siting = ChromaSiting.Left, int stride = 0) + { + stride = Validate(source.Width, source.Height, stride, destination.Length); + fixed (ushort* d = destination) Convert(source, (nint)d, stride, siting, tenBit: true); + } + + /// NV12 back to linear RGB (chroma upsampled for the given siting); used for verification. + public static void FromNv12(ReadOnlySpan source, ImageBuffer destination, ChromaSiting siting = ChromaSiting.Left, int stride = 0) + { + stride = Validate(destination.Width, destination.Height, stride, source.Length); + var (luma, cb, cr) = Planes(destination.Width, destination.Height); + var w = destination.Width; + for (var y = 0; y < destination.Height; y++) + { + for (var x = 0; x < w; x++) luma[y * w + x] = (source[y * stride + x] - 16) / 219f; + } + var uv = source[(stride * destination.Height)..]; + for (var j = 0; j < destination.Height / 2; j++) + { + for (var i = 0; i < w / 2; i++) + { + cb[j * (w / 2) + i] = (uv[j * stride + 2 * i] - 128) / 224f; + cr[j * (w / 2) + i] = (uv[j * stride + 2 * i + 1] - 128) / 224f; + } + } + Reverse(destination, siting, luma, cb, cr); + } + + /// P010 back to linear RGB; used for verification. + public static void FromP010(ReadOnlySpan source, ImageBuffer destination, ChromaSiting siting = ChromaSiting.Left, int stride = 0) + { + stride = Validate(destination.Width, destination.Height, stride, source.Length); + var (luma, cb, cr) = Planes(destination.Width, destination.Height); + var w = destination.Width; + for (var y = 0; y < destination.Height; y++) + { + for (var x = 0; x < w; x++) luma[y * w + x] = ((source[y * stride + x] >> 6) - 64) / 876f; + } + var uv = source[(stride * destination.Height)..]; + for (var j = 0; j < destination.Height / 2; j++) + { + for (var i = 0; i < w / 2; i++) + { + cb[j * (w / 2) + i] = ((uv[j * stride + 2 * i] >> 6) - 512) / 896f; + cr[j * (w / 2) + i] = ((uv[j * stride + 2 * i + 1] >> 6) - 512) / 896f; + } + } + Reverse(destination, siting, luma, cb, cr); + } + + private static (float[] Luma, float[] Cb, float[] Cr) Planes(int width, int height) => + (new float[width * height], new float[width * height / 4], new float[width * height / 4]); + + private static int Validate(int width, int height, int stride, int length) + { + if ((width & 1) != 0 || (height & 1) != 0) throw new ArgumentException("4:2:0 needs even width and height"); + stride = stride == 0 ? width : stride; + if (stride < width) throw new ArgumentOutOfRangeException(nameof(stride)); + if (length < FrameLength(stride, height)) throw new ArgumentException("Buffer too small for the frame"); + return stride; + } + + private static void Convert(ImageBuffer source, nint destination, int stride, ChromaSiting siting, bool tenBit) + { + var pairs = source.Height / 2; + if (source.Width * source.Height < ParallelThreshold) + { + var scratch = new Scratch(source.Width); + for (var j = 0; j < pairs; j++) ConvertRowPair(source, destination, stride, j, siting, tenBit, scratch); + return; + } + Parallel.For(0, pairs, () => new Scratch(source.Width), (j, _, scratch) => + { + ConvertRowPair(source, destination, stride, j, siting, tenBit, scratch); + return scratch; + }, static _ => { }); + } + + private sealed class Scratch(int width) + { + public readonly float[] R0 = new float[width], G0 = new float[width], B0 = new float[width]; + public readonly float[] R1 = new float[width], G1 = new float[width], B1 = new float[width]; + } + + private static unsafe void ConvertRowPair(ImageBuffer source, nint destination, int stride, int pair, ChromaSiting siting, bool tenBit, Scratch s) + { + var width = source.Width; + var y0 = pair * 2; + EncodeRow(source, y0, s.R0, s.G0, s.B0); + EncodeRow(source, y0 + 1, s.R1, s.G1, s.B1); + var chromaRow = source.Height + pair; + if (tenBit) + { + var d = (ushort*)destination; + LumaRow10(s.R0, s.G0, s.B0, new Span(d + (long)y0 * stride, width)); + LumaRow10(s.R1, s.G1, s.B1, new Span(d + (long)(y0 + 1) * stride, width)); + var uv = new Span(d + (long)chromaRow * stride, width); + for (var i = 0; i < width / 2; i++) + { + var (cb, cr) = Chroma(s, i, siting); + uv[2 * i] = (ushort)(Quantize(cb, 896, 512, 1023) << 6); + uv[2 * i + 1] = (ushort)(Quantize(cr, 896, 512, 1023) << 6); + } + } + else + { + var d = (byte*)destination; + LumaRow8(s.R0, s.G0, s.B0, new Span(d + (long)y0 * stride, width)); + LumaRow8(s.R1, s.G1, s.B1, new Span(d + (long)(y0 + 1) * stride, width)); + var uv = new Span(d + (long)chromaRow * stride, width); + for (var i = 0; i < width / 2; i++) + { + var (cb, cr) = Chroma(s, i, siting); + uv[2 * i] = (byte)Quantize(cb, 224, 128, 255); + uv[2 * i + 1] = (byte)Quantize(cr, 224, 128, 255); + } + } + } + + /// sRGB-encodes one row into R'G'B' scratch rows (grey sources fill all three). + private static void EncodeRow(ImageBuffer source, int y, float[] r, float[] g, float[] b) + { + Encode(source.Row(0, y), r); + if (source.Channels == 1) + { + r.CopyTo(g, 0); + r.CopyTo(b, 0); + return; + } + Encode(source.Row(1, y), g); + Encode(source.Row(2, y), b); + } + + /// + /// Direct 16-bit lookup of the sRGB transfer: the steepest slope (12.92) keeps the error below 1e-4, + /// a tenth of a 10-bit code, at a fraction of the cost of the interpolated encoder. + /// + private static void Encode(ReadOnlySpan linear, float[] encoded) + { + var lut = EncodeLut; + for (var x = 0; x < linear.Length; x++) + { + var v = linear[x]; + v = v > 0f ? (v < 1f ? v : 1f) : 0f; // NaN → 0 + encoded[x] = lut[(int)(v * 65535f + 0.5f)]; + } + } + + private static float[] BuildEncodeLut() + { + var lut = new float[65536]; + for (var i = 0; i < lut.Length; i++) lut[i] = (float)ColorSpace.LinearToSrgb(i / 65535.0); + return lut; + } + + /// Filters R'G'B' of the 2×2 (centre) or 3×2 [1 2 1] (left) footprint, then applies the matrix. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static (float Cb, float Cr) Chroma(Scratch s, int i, ChromaSiting siting) + { + var x = 2 * i; + float r, g, b; + if (siting == ChromaSiting.Center) + { + r = (s.R0[x] + s.R0[x + 1] + s.R1[x] + s.R1[x + 1]) * 0.25f; + g = (s.G0[x] + s.G0[x + 1] + s.G1[x] + s.G1[x + 1]) * 0.25f; + b = (s.B0[x] + s.B0[x + 1] + s.B1[x] + s.B1[x + 1]) * 0.25f; + } + else + { + var l = x == 0 ? 0 : x - 1; + r = (s.R0[l] + 2 * s.R0[x] + s.R0[x + 1] + s.R1[l] + 2 * s.R1[x] + s.R1[x + 1]) * 0.125f; + g = (s.G0[l] + 2 * s.G0[x] + s.G0[x + 1] + s.G1[l] + 2 * s.G1[x] + s.G1[x + 1]) * 0.125f; + b = (s.B0[l] + 2 * s.B0[x] + s.B0[x + 1] + s.B1[l] + 2 * s.B1[x] + s.B1[x + 1]) * 0.125f; + } + var y = Kr * r + Kg * g + Kb * b; + return ((b - y) * CbScale, (r - y) * CrScale); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Quantize(float value, float scale, float offset, int max) => Math.Clamp((int)(value * scale + offset + 0.5f), 0, max); + + private static void LumaRow8(float[] r, float[] g, float[] b, Span y) + { + var x = 0; + if (Vector256.IsHardwareAccelerated) + { + var max = Vector256.Create(255); + for (; x <= r.Length - 32; x += 32) + { + var a = Luma(r, g, b, x, 219f, 16.5f, max); + var c = Luma(r, g, b, x + 8, 219f, 16.5f, max); + var e = Luma(r, g, b, x + 16, 219f, 16.5f, max); + var f = Luma(r, g, b, x + 24, 219f, 16.5f, max); + var low = Vector256.Narrow(a.AsUInt32(), c.AsUInt32()); + var high = Vector256.Narrow(e.AsUInt32(), f.AsUInt32()); + Vector256.Narrow(low, high).CopyTo(y[x..]); + } + } + for (; x < r.Length; x++) y[x] = (byte)Math.Clamp((int)((Kr * r[x] + Kg * g[x] + Kb * b[x]) * 219f + 16.5f), 0, 255); + } + + private static void LumaRow10(float[] r, float[] g, float[] b, Span y) + { + var x = 0; + if (Vector256.IsHardwareAccelerated) + { + var max = Vector256.Create(1023); + for (; x <= r.Length - 16; x += 16) + { + var a = Vector256.ShiftLeft(Luma(r, g, b, x, 876f, 64.5f, max), 6); + var c = Vector256.ShiftLeft(Luma(r, g, b, x + 8, 876f, 64.5f, max), 6); + Vector256.Narrow(a.AsUInt32(), c.AsUInt32()).CopyTo(y[x..]); + } + } + for (; x < r.Length; x++) y[x] = (ushort)(Math.Clamp((int)((Kr * r[x] + Kg * g[x] + Kb * b[x]) * 876f + 64.5f), 0, 1023) << 6); + } + + /// Eight luma codes: round(Y' × scale + offset) clamped to [0, max]. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 Luma(float[] r, float[] g, float[] b, int x, float scale, float offset, Vector256 max) + { + var y = Vector256.Create(r, x) * Kr + Vector256.Create(g, x) * Kg + Vector256.Create(b, x) * Kb; + var code = Vector256.ConvertToInt32(y * scale + Vector256.Create(offset)); + return Vector256.Min(Vector256.Max(code, Vector256.Zero), max); + } + + private static void Reverse(ImageBuffer destination, ChromaSiting siting, float[] luma, float[] cbPlane, float[] crPlane) + { + int width = destination.Width, height = destination.Height, half = width / 2; + for (var y = 0; y < height; y++) + { + var row = y / 2 * half; + for (var x = 0; x < width; x++) + { + var i = x / 2; + float cb = cbPlane[row + i], cr = crPlane[row + i]; + if (siting == ChromaSiting.Left && (x & 1) == 1) + { + // Odd luma columns sit halfway between two co-sited chroma samples. + var n = Math.Min(i + 1, half - 1); + cb = (cb + cbPlane[row + n]) * 0.5f; + cr = (cr + crPlane[row + n]) * 0.5f; + } + var yv = luma[y * width + x]; + if (destination.Channels == 1) + { + destination[0, x, y] = (float)ColorSpace.SrgbToLinear(Math.Clamp(yv, 0, 1)); + continue; + } + var r = yv + cr / CrScale; + var b = yv + cb / CbScale; + var g = (yv - Kr * r - Kb * b) / Kg; + destination[0, x, y] = (float)ColorSpace.SrgbToLinear(Math.Clamp(r, 0, 1)); + destination[1, x, y] = (float)ColorSpace.SrgbToLinear(Math.Clamp(g, 0, 1)); + destination[2, x, y] = (float)ColorSpace.SrgbToLinear(Math.Clamp(b, 0, 1)); + } + } + } +} diff --git a/tests/Titano.Core.Tests/Video/Nv12ConverterTests.cs b/tests/Titano.Core.Tests/Video/Nv12ConverterTests.cs new file mode 100644 index 00000000..5231b8a9 --- /dev/null +++ b/tests/Titano.Core.Tests/Video/Nv12ConverterTests.cs @@ -0,0 +1,181 @@ +using Titano.Core.Imaging; +using Titano.Core.Video.Yuv; +using Titano.Testing; + +namespace Titano.Core.Tests.Video; + +public sealed class Nv12ConverterTests +{ + [Theory] + [Case(1f, 1f, 1f, 235, 128, 128)] + [Case(0f, 0f, 0f, 16, 128, 128)] + [Case(1f, 0f, 0f, 63, 102, 240)] + [Case(0f, 1f, 0f, 173, 42, 26)] + [Case(0f, 0f, 1f, 32, 240, 118)] + public void FlatColoursHitBt709LimitedRangeCodes(float r, float g, float b, int y, int u, int v) + { + using var image = Flat(8, 4, r, g, b); + var nv12 = new byte[Nv12Converter.FrameLength(8, 4)]; + Nv12Converter.ToNv12(image, nv12); + Assert.True(nv12.AsSpan(0, 32).ToArray().All(c => c == y), $"luma {nv12[0]} expected {y}"); + Assert.Equal((byte)u, nv12[32]); + Assert.Equal((byte)v, nv12[33]); + + var p010 = new ushort[Nv12Converter.FrameLength(8, 4)]; + Nv12Converter.ToP010(image, p010); + Assert.InRange(p010[0] >> 6, y * 4 - 2, y * 4 + 2); + Assert.Equal(0, p010[0] & 0x3F); + Assert.InRange(p010[32] >> 6, u * 4 - 2, u * 4 + 2); + } + + [Test] + public void TenBitWhiteAndBlackAreExact() + { + using var white = Flat(4, 2, 1, 1, 1); + var p010 = new ushort[Nv12Converter.FrameLength(4, 2)]; + Nv12Converter.ToP010(white, p010); + Assert.Equal((ushort)(940 << 6), p010[0]); + Assert.Equal((ushort)(512 << 6), p010[8]); + using var black = Flat(4, 2, 0, 0, 0); + Nv12Converter.ToP010(black, p010); + Assert.Equal((ushort)(64 << 6), p010[3]); + } + + [Theory] + [Case(70, 34)] + [Case(512, 256)] + public void LumaMatchesScalarReferenceIncludingRemainders(int width, int height) + { + using var image = Gradient(width, height); + var nv12 = new byte[Nv12Converter.FrameLength(width, height)]; + var p010 = new ushort[Nv12Converter.FrameLength(width, height)]; + Nv12Converter.ToNv12(image, nv12); + Nv12Converter.ToP010(image, p010); + for (var y = 0; y < height; y++) + { + for (var x = 0; x < width; x++) + { + var luma = 0.2126f * ColorSpace.EncodeFast(image[0, x, y]) + 0.7152f * ColorSpace.EncodeFast(image[1, x, y]) + 0.0722f * ColorSpace.EncodeFast(image[2, x, y]); + Assert.InRange(nv12[y * width + x], luma * 219 + 16 - 1, luma * 219 + 17); + Assert.InRange(p010[y * width + x] >> 6, luma * 876 + 64 - 1, luma * 876 + 65); + } + } + } + + [Theory] + [Case(ChromaSiting.Left)] + [Case(ChromaSiting.Center)] + public void RoundTripStaysWithinQuantisationError(ChromaSiting siting) + { + const int w = 256, h = 128; + using var image = Gradient(w, h); + using var back = new ImageBuffer(w, h); + var nv12 = new byte[Nv12Converter.FrameLength(w, h)]; + Nv12Converter.ToNv12(image, nv12, siting); + Nv12Converter.FromNv12(nv12, back, siting); + var maxNv12 = MaxEncodedError(image, back); + + var p010 = new ushort[Nv12Converter.FrameLength(w, h)]; + Nv12Converter.ToP010(image, p010, siting); + Nv12Converter.FromP010(p010, back, siting); + var maxP010 = MaxEncodedError(image, back); + + Assert.LessOrEqual(maxNv12, 3.5 / 255); + Assert.LessOrEqual(maxP010, 2.0 / 255); + Assert.Less(maxP010, maxNv12); + } + + [Test] + public void GreyRampRoundTripOnlyCarriesLumaQuantisation() + { + const int w = 256, h = 4; + using var image = new ImageBuffer(w, h, 1); + for (var y = 0; y < h; y++) + { + for (var x = 0; x < w; x++) image[0, x, y] = ColorSpace.Linear8((byte)x); + } + using var back = new ImageBuffer(w, h, 1); + var nv12 = new byte[Nv12Converter.FrameLength(w, h)]; + Nv12Converter.ToNv12(image, nv12); + Assert.True(nv12.AsSpan(w * h).ToArray().All(c => c == 128), "grey has neutral chroma"); + Nv12Converter.FromNv12(nv12, back); + double max = 0; + for (var i = 0; i < w * h; i++) max = Math.Max(max, Math.Abs(ColorSpace.LinearToSrgb(image.Plane(0)[i]) - ColorSpace.LinearToSrgb(back.Plane(0)[i]))); + // One limited-range step is 255/219 full-range levels; rounding keeps half of it. + Assert.LessOrEqual(max, 0.6 / 219); + } + + [Test] + public void SitingChangesTheChromaFilterFootprint() + { + // Columns: red, red | blue, blue. Centre siting keeps the second chroma pure blue, + // left siting (co-sited with column 2, taps 1:2:1) mixes in a quarter of red. + using var image = new ImageBuffer(4, 2); + for (var y = 0; y < 2; y++) + { + image[0, 0, y] = image[0, 1, y] = 1; + image[2, 2, y] = image[2, 3, y] = 1; + } + var centre = new byte[Nv12Converter.FrameLength(4, 2)]; + var left = new byte[Nv12Converter.FrameLength(4, 2)]; + Nv12Converter.ToNv12(image, centre, ChromaSiting.Center); + Nv12Converter.ToNv12(image, left, ChromaSiting.Left); + Assert.Equal((byte)240, centre[8 + 2]); // Cb of pure blue + Assert.Equal((byte)118, centre[8 + 3]); // Cr of pure blue + Assert.Equal((byte)240, left[8 + 1]); // Cr of pure red at column 0 + // R'G'B' = (0.25, 0, 0.75): Cb = (0.75 − 0.1073) / 1.8556 → 206, Cr = (0.25 − 0.1073) / 1.5748 → 148. + Assert.Equal((byte)206, left[8 + 2]); + Assert.Equal((byte)148, left[8 + 3]); + } + + [Test] + public void RespectsStrideAndRejectsOddSizes() + { + using var image = Flat(6, 2, 1, 1, 1); + var nv12 = new byte[Nv12Converter.FrameLength(16, 2)]; + nv12.AsSpan().Fill(7); + Nv12Converter.ToNv12(image, nv12, stride: 16); + Assert.Equal((byte)235, nv12[5]); + Assert.Equal((byte)7, nv12[6]); // padding untouched + Assert.Equal((byte)128, nv12[32]); + using var odd = new ImageBuffer(5, 2); + Assert.Throws(() => Nv12Converter.ToNv12(odd, new byte[64])); + } + + private static double MaxEncodedError(ImageBuffer a, ImageBuffer b) + { + double max = 0; + for (var c = 0; c < 3; c++) + { + var pa = a.Plane(c); + var pb = b.Plane(c); + for (var i = 0; i < pa.Length; i++) max = Math.Max(max, Math.Abs(ColorSpace.LinearToSrgb(pa[i]) - ColorSpace.LinearToSrgb(pb[i]))); + } + return max; + } + + private static ImageBuffer Flat(int w, int h, float r, float g, float b) + { + var image = new ImageBuffer(w, h); + image.Plane(0).Fill(r); + image.Plane(1).Fill(g); + image.Plane(2).Fill(b); + return image; + } + + /// Smooth colour ramps in encoded space (linearised), so 4:2:0 loss stays small. + private static ImageBuffer Gradient(int w, int h) + { + var image = new ImageBuffer(w, h); + for (var y = 0; y < h; y++) + { + for (var x = 0; x < w; x++) + { + image[0, x, y] = (float)ColorSpace.SrgbToLinear(0.1 + 0.8 * x / w); + image[1, x, y] = (float)ColorSpace.SrgbToLinear(0.2 + 0.6 * y / h); + image[2, x, y] = (float)ColorSpace.SrgbToLinear(0.5 + 0.2 * Math.Sin((x + y) * 0.02)); + } + } + return image; + } +} From e15bbb997410e0d7b75fd56e0eb881bfd14cce8f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 09:41:34 +0000 Subject: [PATCH 5/9] feat(video): add AVI MJPEG writer with idx1 and OpenDML indexes Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Video/Avi/AviMjpegWriter.cs | 312 ++++++++++++++++++ .../Video/AviMjpegWriterTests.cs | 205 ++++++++++++ 2 files changed, 517 insertions(+) create mode 100644 src/Titano.Core/Video/Avi/AviMjpegWriter.cs create mode 100644 tests/Titano.Core.Tests/Video/AviMjpegWriterTests.cs diff --git a/src/Titano.Core/Video/Avi/AviMjpegWriter.cs b/src/Titano.Core/Video/Avi/AviMjpegWriter.cs new file mode 100644 index 00000000..1e42fb5b --- /dev/null +++ b/src/Titano.Core/Video/Avi/AviMjpegWriter.cs @@ -0,0 +1,312 @@ +using System.Buffers.Binary; + +namespace Titano.Core.Video.Avi; + +/// +/// Streaming RIFF AVI writer for already-encoded JPEG frames ('MJPG', chunks '00dc'). Writes the legacy idx1 index +/// for the first RIFF and OpenDML indexes ('indx' super index, one 'ix00' per RIFF) so files can continue in +/// 'AVIX' RIFF lists past the per-RIFF size limit. AVI is constant frame rate: repeat a frame to hold it longer. +/// The output stream must be seekable and is not disposed. +/// +public sealed class AviMjpegWriter +{ + /// Default size of each RIFF list; 1 GiB keeps the first RIFF readable by AVI 1.0 software. + public const long DefaultRiffSizeLimit = 1L << 30; + + private const int SuperIndexCapacity = 256; + private const uint KeyFrameFlag = 0x10; // AVIIF_KEYFRAME + private static readonly uint ChunkId = FourCc("00dc"); + + private readonly Stream _stream; + private readonly int _width; + private readonly int _height; + private readonly uint _rate; + private readonly uint _scale; + private readonly long _riffLimit; + private readonly long _start; + private readonly List<(long Offset, uint Size, uint Frames)> _superIndex = []; + private readonly List<(long DataOffset, uint Size)> _segmentFrames = []; + private long _avih, _strh, _indx, _dmlh; + private long _riffStart, _moviStart, _position; + private int _framesInFirstRiff; + private int _totalFrames; + private uint _maxFrame; + private bool _completed; + + /// Bytes per RIFF list before an 'AVIX' list starts (tests use small values). + public AviMjpegWriter(Stream output, int width, int height, Rational frameRate, long riffSizeLimit = DefaultRiffSizeLimit) + { + ArgumentNullException.ThrowIfNull(output); + if (!output.CanSeek || !output.CanWrite) throw new ArgumentException("The output stream must be writable and seekable", nameof(output)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(width); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(height); + if (frameRate.Numerator <= 0 || frameRate.Denominator <= 0) throw new ArgumentOutOfRangeException(nameof(frameRate)); + ArgumentOutOfRangeException.ThrowIfLessThan(riffSizeLimit, 16 * 1024); + ArgumentOutOfRangeException.ThrowIfGreaterThan(riffSizeLimit, uint.MaxValue); + _stream = output; + _width = width; + _height = height; + var g = Gcd(frameRate.Numerator, frameRate.Denominator); + _rate = (uint)(frameRate.Numerator / g); + _scale = (uint)(frameRate.Denominator / g); + _riffLimit = riffSizeLimit; + _start = output.Position; + WriteHeaders(); + } + + public int FrameCount => _totalFrames; + + /// Appends one JPEG (SOI-prefixed) frame. + public void WriteFrame(ReadOnlySpan jpeg) + { + if (_completed) throw new InvalidOperationException("The writer is complete"); + if (jpeg.Length < 4 || jpeg[0] != 0xFF || jpeg[1] != 0xD8) throw new ArgumentException("Not a JPEG frame", nameof(jpeg)); + var chunk = 8L + jpeg.Length + (jpeg.Length & 1); + if (_segmentFrames.Count > 0 && _position - _riffStart + chunk + IndexOverhead(_segmentFrames.Count + 1) > _riffLimit) + { + CloseSegment(); + OpenSegment("AVIX"); + } + + Span header = stackalloc byte[8]; + BinaryPrimitives.WriteUInt32LittleEndian(header, ChunkId); + BinaryPrimitives.WriteUInt32LittleEndian(header[4..], (uint)jpeg.Length); + _stream.Write(header); + _stream.Write(jpeg); + if ((jpeg.Length & 1) != 0) _stream.WriteByte(0); + _segmentFrames.Add((_position + 8, (uint)jpeg.Length)); + _position += chunk; + _maxFrame = Math.Max(_maxFrame, (uint)jpeg.Length); + _totalFrames++; + } + + /// Writes the indexes, patches sizes and frame counts, and flushes. + public void Complete() + { + if (_completed) throw new InvalidOperationException("The writer is already complete"); + CloseSegment(); + var buffer = _maxFrame + 8; + var bytesPerSecond = (uint)Math.Min(uint.MaxValue, (double)_maxFrame * _rate / _scale); + Patch(_avih + 4, bytesPerSecond); + Patch(_avih + 16, (uint)_framesInFirstRiff); + Patch(_avih + 28, buffer); + Patch(_strh + 32, (uint)_totalFrames); + Patch(_strh + 36, buffer); + Patch(_dmlh, (uint)_totalFrames); + Patch(_indx + 4, (uint)_superIndex.Count); + Span entry = stackalloc byte[16]; + for (var i = 0; i < _superIndex.Count; i++) + { + BinaryPrimitives.WriteInt64LittleEndian(entry, _superIndex[i].Offset); + BinaryPrimitives.WriteUInt32LittleEndian(entry[8..], _superIndex[i].Size); + BinaryPrimitives.WriteUInt32LittleEndian(entry[12..], _superIndex[i].Frames); + _stream.Position = _indx + 24 + i * 16; + _stream.Write(entry); + } + _stream.Position = _position; + _stream.Flush(); + _completed = true; + } + + /// Bytes the segment's ix00 (and idx1 in the first RIFF) will need for frames. + private long IndexOverhead(int frames) => 32 + 8L * frames + (_superIndex.Count == 0 ? 8 + 16L * frames : 0); + + private void WriteHeaders() + { + var h = new List(4800); + var riff = Begin(h, "RIFF", "AVI "); + var hdrl = Begin(h, "LIST", "hdrl"); + + _avih = ChunkStart(h, "avih", 56); + Add32(h, (uint)Math.Round(1_000_000.0 * _scale / _rate)); // dwMicroSecPerFrame + Add32(h, 0); // dwMaxBytesPerSec (patched) + Add32(h, 0); // dwPaddingGranularity + Add32(h, 0x10 | 0x100); // AVIF_HASINDEX | AVIF_ISINTERLEAVED + Add32(h, 0); // dwTotalFrames of the first RIFF (patched) + Add32(h, 0); // dwInitialFrames + Add32(h, 1); // dwStreams + Add32(h, 0); // dwSuggestedBufferSize (patched) + Add32(h, (uint)_width); + Add32(h, (uint)_height); + AddZeros(h, 16); + + var strl = Begin(h, "LIST", "strl"); + _strh = ChunkStart(h, "strh", 56); + Add32(h, FourCc("vids")); + Add32(h, FourCc("MJPG")); + Add32(h, 0); // dwFlags + Add32(h, 0); // wPriority, wLanguage + Add32(h, 0); // dwInitialFrames + Add32(h, _scale); + Add32(h, _rate); + Add32(h, 0); // dwStart + Add32(h, 0); // dwLength (patched) + Add32(h, 0); // dwSuggestedBufferSize (patched) + Add32(h, uint.MaxValue); // dwQuality: default + Add32(h, 0); // dwSampleSize: variable + Add16(h, 0); + Add16(h, 0); + Add16(h, (ushort)Math.Min(_width, short.MaxValue)); + Add16(h, (ushort)Math.Min(_height, short.MaxValue)); + + ChunkStart(h, "strf", 40); + Add32(h, 40); // BITMAPINFOHEADER.biSize + Add32(h, (uint)_width); + Add32(h, (uint)_height); + Add16(h, 1); // biPlanes + Add16(h, 24); // biBitCount + Add32(h, FourCc("MJPG")); + Add32(h, (uint)Math.Min(uint.MaxValue, (long)_width * _height * 3)); + AddZeros(h, 16); + + _indx = ChunkStart(h, "indx", 24 + 16 * SuperIndexCapacity); + Add16(h, 4); // wLongsPerEntry + h.Add(0); // bIndexSubType + h.Add(0); // bIndexType = AVI_INDEX_OF_INDEXES + Add32(h, 0); // nEntriesInUse (patched) + Add32(h, ChunkId); + AddZeros(h, 12 + 16 * SuperIndexCapacity); + End(h, strl); + + var odml = Begin(h, "LIST", "odml"); + _dmlh = ChunkStart(h, "dmlh", 248); + AddZeros(h, 248); + End(h, odml); + End(h, hdrl); + + // The RIFF size is patched when the first segment closes; the movi list follows directly. + _riffStart = _start + riff; + _stream.Write(h.ToArray()); + _avih += _start; + _strh += _start; + _indx += _start; + _dmlh += _start; + _position = _start + h.Count; + OpenMovi(); + } + + private void OpenSegment(string type) + { + _riffStart = _position; + Span header = stackalloc byte[12]; + BinaryPrimitives.WriteUInt32LittleEndian(header, FourCc("RIFF")); + BinaryPrimitives.WriteUInt32LittleEndian(header[8..], FourCc(type)); + _stream.Write(header); + _position += 12; + OpenMovi(); + } + + private void OpenMovi() + { + Span header = stackalloc byte[12]; + BinaryPrimitives.WriteUInt32LittleEndian(header, FourCc("LIST")); + BinaryPrimitives.WriteUInt32LittleEndian(header[8..], FourCc("movi")); + _stream.Write(header); + _moviStart = _position + 8; // offsets in idx1 and ix00 are relative to the 'movi' four-cc + _position += 12; + } + + private void CloseSegment() + { + if (_superIndex.Count == SuperIndexCapacity) throw new InvalidOperationException("AVI super index is full"); + var frames = _segmentFrames.Count; + + // Standard index (OpenDML ix00) closes the movi list. + var ixOffset = _position; + var ix = new byte[32 + 8 * frames]; + BinaryPrimitives.WriteUInt32LittleEndian(ix, FourCc("ix00")); + BinaryPrimitives.WriteUInt32LittleEndian(ix.AsSpan(4), (uint)(ix.Length - 8)); + BinaryPrimitives.WriteUInt16LittleEndian(ix.AsSpan(8), 2); // wLongsPerEntry + ix[11] = 1; // AVI_INDEX_OF_CHUNKS + BinaryPrimitives.WriteUInt32LittleEndian(ix.AsSpan(12), (uint)frames); + BinaryPrimitives.WriteUInt32LittleEndian(ix.AsSpan(16), ChunkId); + BinaryPrimitives.WriteInt64LittleEndian(ix.AsSpan(20), _moviStart); + for (var i = 0; i < frames; i++) + { + BinaryPrimitives.WriteUInt32LittleEndian(ix.AsSpan(32 + i * 8), (uint)(_segmentFrames[i].DataOffset - _moviStart)); + BinaryPrimitives.WriteUInt32LittleEndian(ix.AsSpan(36 + i * 8), _segmentFrames[i].Size); // bit 31 clear: key frame + } + _stream.Position = ixOffset; + _stream.Write(ix); + _position += ix.Length; + Patch(_moviStart - 4, (uint)(_position - _moviStart)); + + if (_superIndex.Count == 0) + { + var idx1 = new byte[8 + 16 * frames]; + BinaryPrimitives.WriteUInt32LittleEndian(idx1, FourCc("idx1")); + BinaryPrimitives.WriteUInt32LittleEndian(idx1.AsSpan(4), (uint)(idx1.Length - 8)); + for (var i = 0; i < frames; i++) + { + var e = idx1.AsSpan(8 + i * 16); + BinaryPrimitives.WriteUInt32LittleEndian(e, ChunkId); + BinaryPrimitives.WriteUInt32LittleEndian(e[4..], KeyFrameFlag); + BinaryPrimitives.WriteUInt32LittleEndian(e[8..], (uint)(_segmentFrames[i].DataOffset - 8 - _moviStart)); + BinaryPrimitives.WriteUInt32LittleEndian(e[12..], _segmentFrames[i].Size); + } + _stream.Position = _position; + _stream.Write(idx1); + _position += idx1.Length; + _framesInFirstRiff = frames; + } + Patch(_riffStart + 4, (uint)(_position - _riffStart - 8)); + _superIndex.Add((ixOffset, (uint)ix.Length, (uint)frames)); + _segmentFrames.Clear(); + _stream.Position = _position; + } + + private void Patch(long offset, uint value) + { + Span bytes = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + _stream.Position = offset; + _stream.Write(bytes); + } + + private static int Begin(List h, string id, string type) + { + var start = h.Count; + Add32(h, FourCc(id)); + Add32(h, 0); + Add32(h, FourCc(type)); + return start; + } + + private static void End(List h, int start) + { + var size = (uint)(h.Count - start - 8); + for (var i = 0; i < 4; i++) h[start + 4 + i] = (byte)(size >> (8 * i)); + } + + /// Writes a chunk header and returns the offset of its data. + private static int ChunkStart(List h, string id, int size) + { + Add32(h, FourCc(id)); + Add32(h, (uint)size); + return h.Count; + } + + private static void Add16(List h, ushort v) + { + h.Add((byte)v); + h.Add((byte)(v >> 8)); + } + + private static void Add32(List h, uint v) + { + for (var i = 0; i < 4; i++) h.Add((byte)(v >> (8 * i))); + } + + private static void AddZeros(List h, int count) + { + for (var i = 0; i < count; i++) h.Add(0); + } + + private static uint FourCc(string s) => (uint)(s[0] | s[1] << 8 | s[2] << 16 | s[3] << 24); + + private static int Gcd(int a, int b) + { + while (b != 0) (a, b) = (b, a % b); + return a; + } +} diff --git a/tests/Titano.Core.Tests/Video/AviMjpegWriterTests.cs b/tests/Titano.Core.Tests/Video/AviMjpegWriterTests.cs new file mode 100644 index 00000000..47ee61f6 --- /dev/null +++ b/tests/Titano.Core.Tests/Video/AviMjpegWriterTests.cs @@ -0,0 +1,205 @@ +using System.Buffers.Binary; +using System.Text; +using Titano.Core.Video; +using Titano.Core.Video.Avi; +using Titano.Testing; + +namespace Titano.Core.Tests.Video; + +public sealed class AviMjpegWriterTests +{ + [Test] + public void SingleRiffHasHeadersIdx1AndStandardIndex() + { + var frames = Frames(10, 1); + var bytes = Write(frames, Rational.Fps2997, AviMjpegWriter.DefaultRiffSizeLimit); + var file = Riff.Parse(bytes); + Assert.Equal(bytes.Length, file.Sum(r => r.Size + 8)); + Assert.Count(1, file); + var riff = file[0]; + Assert.Equal("RIFF", riff.Id); + Assert.Equal("AVI ", riff.Type); + + var hdrl = riff.List("hdrl"); + var avih = hdrl.Chunk("avih").Data; + Assert.Equal(33367u, U32(avih, 0)); + Assert.Equal(10u, U32(avih, 16)); + Assert.Equal(1u, U32(avih, 24)); + Assert.Equal(64u, U32(avih, 32)); + Assert.Equal(48u, U32(avih, 36)); + var strl = hdrl.List("strl"); + var strh = strl.Chunk("strh").Data; + Assert.Equal("vids", Encoding.ASCII.GetString(strh, 0, 4)); + Assert.Equal("MJPG", Encoding.ASCII.GetString(strh, 4, 4)); + Assert.Equal(1001u, U32(strh, 20)); + Assert.Equal(30000u, U32(strh, 24)); + Assert.Equal(10u, U32(strh, 32)); + Assert.Equal("MJPG", Encoding.ASCII.GetString(strl.Chunk("strf").Data, 16, 4)); + Assert.Equal(10u, U32(hdrl.List("odml").Chunk("dmlh").Data, 0)); + + var movi = riff.List("movi"); + var chunks = movi.Children.Where(c => c.Id == "00dc").ToList(); + Assert.Equal(10, chunks.Count); + for (var i = 0; i < 10; i++) + { + Assert.BytesEqual(frames[i], chunks[i].Data); + Assert.Equal(0L, chunks[i].Offset % 2, "chunks stay word aligned after odd frames"); + } + + var idx1 = riff.Chunk("idx1").Data; + Assert.Equal(160, idx1.Length); + for (var i = 0; i < 10; i++) + { + Assert.Equal("00dc", Encoding.ASCII.GetString(idx1, i * 16, 4)); + Assert.Equal(0x10u, U32(idx1, i * 16 + 4)); + Assert.Equal(chunks[i].Offset, movi.TypeOffset + U32(idx1, i * 16 + 8)); + Assert.Equal((uint)frames[i].Length, U32(idx1, i * 16 + 12)); + } + AssertOpenDmlIndex(bytes, file, frames); + } + + [Test] + public void LargeOutputsContinueInAvixLists() + { + var frames = Frames(40, 2); + const long limit = 16 * 1024; + var bytes = Write(frames, new Rational(25, 1), limit); + var file = Riff.Parse(bytes); + Assert.Equal(bytes.Length, file.Sum(r => r.Size + 8)); + Assert.Greater(file.Count, 3); + Assert.Equal("AVI ", file[0].Type); + Assert.True(file.Skip(1).All(r => r.Type == "AVIX" && r.Children.All(c => c.Id != "idx1"))); + Assert.True(file.All(r => r.Size + 8 <= limit), "every RIFF respects the limit"); + + var first = file[0].List("movi").Children.Count(c => c.Id == "00dc"); + Assert.Equal((uint)first, U32(file[0].List("hdrl").Chunk("avih").Data, 16)); + Assert.Equal(first * 16, file[0].Chunk("idx1").Data.Length); + Assert.Equal(40u, U32(file[0].List("hdrl").List("strl").Chunk("strh").Data, 32)); + Assert.Equal(40u, U32(file[0].List("hdrl").List("odml").Chunk("dmlh").Data, 0)); + AssertOpenDmlIndex(bytes, file, frames); + } + + [Theory] + [Case(24, 1, 41667u)] + [Case(25, 1, 40000u)] + [Case(60000, 1001, 16683u)] + [Case(48, 2, 41667u)] + public void FrameRateFieldsAreExact(int numerator, int denominator, uint microseconds) + { + var bytes = Write(Frames(2, 3), new Rational(numerator, denominator), AviMjpegWriter.DefaultRiffSizeLimit); + var hdrl = Riff.Parse(bytes)[0].List("hdrl"); + Assert.Equal(microseconds, U32(hdrl.Chunk("avih").Data, 0)); + var strh = hdrl.List("strl").Chunk("strh").Data; + var g = (uint)System.Numerics.BigInteger.GreatestCommonDivisor(numerator, denominator); + Assert.Equal((uint)denominator / g, U32(strh, 20)); + Assert.Equal((uint)numerator / g, U32(strh, 24)); + } + + [Test] + public void RejectsNonJpegFrames() + { + using var ms = new MemoryStream(); + var writer = new AviMjpegWriter(ms, 64, 48, new Rational(30, 1)); + Assert.Throws(() => writer.WriteFrame([0x89, 0x50, 0x4E, 0x47])); + } + + private static void AssertOpenDmlIndex(byte[] bytes, List file, List frames) + { + var indx = file[0].List("hdrl").List("strl").Chunk("indx").Data; + Assert.Equal(4, BinaryPrimitives.ReadUInt16LittleEndian(indx)); + Assert.Equal((byte)0, indx[3]); // AVI_INDEX_OF_INDEXES + var entries = U32(indx, 4); + Assert.Equal((uint)file.Count, entries); + var frame = 0; + for (var e = 0; e < entries; e++) + { + var offset = BinaryPrimitives.ReadInt64LittleEndian(indx.AsSpan(24 + e * 16)); + var size = U32(indx, 32 + e * 16); + var duration = U32(indx, 36 + e * 16); + Assert.Equal("ix00", Encoding.ASCII.GetString(bytes, (int)offset, 4)); + Assert.Equal(size, U32(bytes, (int)offset + 4) + 8); + var ix = bytes.AsSpan((int)offset + 8); + Assert.Equal((byte)1, ix[3]); // AVI_INDEX_OF_CHUNKS + Assert.Equal(duration, U32(bytes, (int)offset + 12)); + var baseOffset = BinaryPrimitives.ReadInt64LittleEndian(ix[12..]); + for (var i = 0; i < duration; i++, frame++) + { + var dataOffset = baseOffset + U32(bytes, (int)offset + 32 + i * 8); + var length = U32(bytes, (int)offset + 36 + i * 8); + Assert.Equal(0u, length & 0x8000_0000u, "every MJPEG frame is a key frame"); + Assert.BytesEqual(frames[frame], bytes.AsSpan((int)dataOffset, (int)length)); + } + } + Assert.Equal(frames.Count, frame); + } + + private static byte[] Write(List frames, Rational rate, long limit) + { + using var ms = new MemoryStream(); + var writer = new AviMjpegWriter(ms, 64, 48, rate, limit); + foreach (var f in frames) writer.WriteFrame(f); + writer.Complete(); + Assert.Equal(frames.Count, writer.FrameCount); + return ms.ToArray(); + } + + /// JPEG-shaped payloads (SOI … EOI) of odd and even lengths. + private static List Frames(int count, int seed) + { + var rng = new Random(seed); + var list = new List(); + for (var i = 0; i < count; i++) + { + var f = new byte[1500 + rng.Next(2000)]; + rng.NextBytes(f); + f[0] = 0xFF; + f[1] = 0xD8; + f[^2] = 0xFF; + f[^1] = 0xD9; + list.Add(f); + } + return list; + } + + private static uint U32(byte[] data, int offset) => BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(offset)); + + /// Minimal RIFF tree reader for verification. + private static class Riff + { + public sealed record Node(string Id, string? Type, long Offset, uint Size, byte[] Data, List Children) + { + /// File offset of the list type four-cc (the base of idx1 offsets). + public long TypeOffset => Offset + 8; + + public Node List(string type) => Children.Single(c => c.Type == type); + + public Node Chunk(string id) => Children.Single(c => c.Id == id); + } + + public static List Parse(byte[] bytes) => ParseRange(bytes, 0, bytes.Length); + + private static List ParseRange(byte[] bytes, long start, long end) + { + var nodes = new List(); + var pos = start; + while (pos + 8 <= end) + { + var id = Encoding.ASCII.GetString(bytes, (int)pos, 4); + var size = U32(bytes, (int)pos + 4); + Assert.True(pos + 8 + size <= end, $"chunk {id} at {pos} overruns its parent"); + if (id is "RIFF" or "LIST") + { + var type = Encoding.ASCII.GetString(bytes, (int)pos + 8, 4); + nodes.Add(new Node(id, type, pos, size, [], ParseRange(bytes, pos + 12, pos + 8 + size))); + } + else + { + nodes.Add(new Node(id, null, pos, size, bytes.AsSpan((int)pos + 8, (int)size).ToArray(), [])); + } + pos += 8 + size + (size & 1); + } + Assert.Equal(end, pos); + return nodes; + } + } +} From 71c0113f5fdce5c7045bb37ddb8195b8dea62463 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 09:49:30 +0000 Subject: [PATCH 6/9] feat(gif): add animated GIF encoder with variance-split palettes, dithering and frame differencing Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Codecs/Gif/GifEncoder.cs | 394 ++++++++++++++++++ src/Titano.Core/Codecs/Gif/GifLzw.cs | 104 +++++ src/Titano.Core/Codecs/Gif/GifPalette.cs | 216 ++++++++++ .../Video/GifEncoderTests.cs | 255 ++++++++++++ .../Titano.Core.Tests/Video/GifTestDecoder.cs | 197 +++++++++ 5 files changed, 1166 insertions(+) create mode 100644 src/Titano.Core/Codecs/Gif/GifEncoder.cs create mode 100644 src/Titano.Core/Codecs/Gif/GifLzw.cs create mode 100644 src/Titano.Core/Codecs/Gif/GifPalette.cs create mode 100644 tests/Titano.Core.Tests/Video/GifEncoderTests.cs create mode 100644 tests/Titano.Core.Tests/Video/GifTestDecoder.cs diff --git a/src/Titano.Core/Codecs/Gif/GifEncoder.cs b/src/Titano.Core/Codecs/Gif/GifEncoder.cs new file mode 100644 index 00000000..fa754467 --- /dev/null +++ b/src/Titano.Core/Codecs/Gif/GifEncoder.cs @@ -0,0 +1,394 @@ +using Titano.Core.Diagnostics; +using Titano.Core.Imaging; + +namespace Titano.Core.Codecs.Gif; + +/// Where GIF frames take their colours from. +public enum GifPaletteMode +{ + /// One palette for the whole animation, built from the first frames (or supplied). + Global, + + /// A palette per frame, built from that frame. + Local, +} + +/// Options of . +public sealed record GifEncoderOptions +{ + /// NETSCAPE2.0 loop count: 0 loops forever; null plays once (no loop extension). + public int? LoopCount { get; init; } = 0; + + /// Serpentine Floyd–Steinberg error diffusion. + public bool Dither { get; init; } = true; + + /// Unchanged pixels become transparent and frames are cropped to the changed area. + public bool FrameDifferencing { get; init; } = true; + + /// Largest per-channel difference (8-bit) between a pixel and its value when last drawn that still counts as unchanged. + public int DifferenceThreshold { get; init; } = 2; + + public GifPaletteMode PaletteMode { get; init; } = GifPaletteMode.Global; + + /// Frames buffered to build the global palette when is not supplied. + public int PaletteSampleFrames { get; init; } = 8; + + /// Optional global palette as RGB triples (see ). + public byte[]? Palette { get; init; } + + /// Colours per palette, 2–256 (one is reserved for transparency when differencing). + public int MaxColors { get; init; } = 256; +} + +/// +/// Animated GIF89a encoder. Frames arrive one by one as linear-light s (converted to +/// 8-bit sRGB) or 8-bit sRGB RGB bytes, with durations in 90 kHz ticks. Delays are rounded on the cumulative +/// timeline so the total duration is exact to the centisecond; identical frames merge into one longer frame. +/// The output stream is not disposed. +/// +public sealed class GifEncoder +{ + private const string LogCategory = "GifEncoder"; + private const int PaletteSamples = 1 << 18; + + private readonly Stream _output; + private readonly int _width; + private readonly int _height; + private readonly GifEncoderOptions _options; + private readonly List<(byte[] Rgb, long Duration)> _sampling = []; + private byte[]? _globalPalette; + private GifPalette.Map? _globalMap; + private byte[]? _canvas; + private byte[]? _drawn; + private PendingFrame? _pending; + private long _time; + private bool _headerWritten; + private bool _warnedShortDelay; + private bool _completed; + + public GifEncoder(Stream output, int width, int height, GifEncoderOptions? options = null) + { + ArgumentNullException.ThrowIfNull(output); + if (width is <= 0 or > ushort.MaxValue || height is <= 0 or > ushort.MaxValue) throw new ArgumentOutOfRangeException(nameof(width), "GIF sizes are 1–65535"); + _options = options ?? new GifEncoderOptions(); + if (_options.MaxColors is < 2 or > 256) throw new ArgumentOutOfRangeException(nameof(options), "MaxColors must be 2–256"); + if (_options.Palette is { } p && (p.Length % 3 != 0 || p.Length / 3 > PaletteBudget || p.Length == 0)) + throw new ArgumentException("Palette must hold 1 to MaxColors (minus the transparent slot) RGB triples", nameof(options)); + _output = output; + _width = width; + _height = height; + if (_options.PaletteMode == GifPaletteMode.Global && _options.Palette is { } palette) SetGlobalPalette(palette); + } + + /// Frames written to the file so far (identical consecutive frames count once). + public int FrameCount { get; private set; } + + /// Builds a palette (RGB triples) from sample frames, e.g. spread across a sequence. + public static byte[] BuildPalette(IEnumerable frames, int maxColors = 255) + { + using var ms = new MemoryStream(); + foreach (var frame in frames) ms.Write(Subsample(ToRgb(frame), frame.Width * frame.Height, PaletteSamples / 8)); + return GifPalette.Build(ms.ToArray(), maxColors); + } + + /// Adds a linear-light frame (alpha ignored) shown for ticks. + public void AddFrame(ImageBuffer frame, long duration90k) + { + if (frame.Width != _width || frame.Height != _height) throw new ArgumentException("Frame size differs from the animation size", nameof(frame)); + AddRgb(ToRgb(frame), duration90k); + } + + /// Adds an interleaved 8-bit sRGB frame. + public void AddFrame(ReadOnlySpan rgb24, long duration90k) + { + if (rgb24.Length != _width * _height * 3) throw new ArgumentException("Expected width × height × 3 bytes", nameof(rgb24)); + AddRgb(rgb24.ToArray(), duration90k); + } + + /// Writes the remaining frames and the trailer. + public void Complete() + { + if (_completed) throw new InvalidOperationException("The encoder is already complete"); + if (_globalMap is null && _options.PaletteMode == GifPaletteMode.Global) FlushSampling(); + if (!_headerWritten) throw new InvalidOperationException("No frames were added"); + Emit(_time); + _output.WriteByte(0x3B); + _output.Flush(); + _completed = true; + } + + private int PaletteBudget => _options.FrameDifferencing ? _options.MaxColors - 1 : _options.MaxColors; + + private void AddRgb(byte[] rgb, long duration) + { + if (_completed) throw new InvalidOperationException("The encoder is complete"); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(duration); + if (_options.PaletteMode == GifPaletteMode.Global && _globalMap is null) + { + _sampling.Add((rgb, duration)); + if (_sampling.Count >= Math.Max(1, _options.PaletteSampleFrames)) FlushSampling(); + return; + } + EncodeFrame(rgb, duration); + } + + private void FlushSampling() + { + if (_sampling.Count == 0) return; + using var ms = new MemoryStream(); + foreach (var (rgb, _) in _sampling) ms.Write(Subsample(rgb, _width * _height, PaletteSamples / _sampling.Count)); + SetGlobalPalette(GifPalette.Build(ms.ToArray(), PaletteBudget)); + foreach (var (rgb, duration) in _sampling) EncodeFrame(rgb, duration); + _sampling.Clear(); + } + + private void SetGlobalPalette(byte[] palette) + { + _globalPalette = palette; + _globalMap = new GifPalette.Map(palette); + } + + private sealed record PendingFrame(long Start, int Left, int Top, int Width, int Height, byte[]? LocalPalette, int Transparent, byte[] Indices, int TableBits); + + private void EncodeFrame(byte[] rgb, long duration) + { + byte[] palette; + GifPalette.Map map; + if (_options.PaletteMode == GifPaletteMode.Local) + { + palette = GifPalette.Build(Subsample(rgb, _width * _height, PaletteSamples), PaletteBudget); + map = new GifPalette.Map(palette); + } + else + { + palette = _globalPalette!; + map = _globalMap!; + } + var colors = palette.Length / 3; + var differencing = _options.FrameDifferencing && _canvas is not null; + var transparent = _options.FrameDifferencing ? colors : -1; + + var indices = new byte[_width * _height]; + var changed = Quantize(rgb, palette, map, indices, differencing); + var (left, top, right, bottom) = differencing ? Bounds(changed) : (0, 0, _width - 1, _height - 1); + if (right < left) + { + // Nothing visible changed: the previous frame simply stays longer. + _time += duration; + return; + } + + _canvas ??= new byte[_width * _height * 3]; + var w = right - left + 1; + var h = bottom - top + 1; + var cropped = new byte[w * h]; + for (var y = 0; y < h; y++) + { + for (var x = 0; x < w; x++) + { + var i = (top + y) * _width + left + x; + if (differencing && !changed[i]) + { + cropped[y * w + x] = (byte)transparent; + continue; + } + var index = indices[i]; + cropped[y * w + x] = index; + _canvas[i * 3] = palette[index * 3]; + _canvas[i * 3 + 1] = palette[index * 3 + 1]; + _canvas[i * 3 + 2] = palette[index * 3 + 2]; + } + } + + if (!_headerWritten) WriteHeader(); + Emit(_time); + var tableBits = TableBits(colors + (transparent >= 0 ? 1 : 0)); + _pending = new PendingFrame(_time, left, top, w, h, _options.PaletteMode == GifPaletteMode.Local ? palette : null, + differencing ? transparent : -1, cropped, tableBits); + _time += duration; + } + + /// + /// Maps the frame to palette indices with optional serpentine Floyd–Steinberg diffusion. With a previous canvas, + /// pixels whose source stays within the threshold of the source they were last drawn from keep the displayed + /// colour and pass no error on, so static areas keep their dither pattern while other regions change. + /// + private bool[] Quantize(byte[] rgb, byte[] palette, GifPalette.Map map, byte[] indices, bool differencing) + { + var changed = new bool[_width * _height]; + _drawn ??= new byte[rgb.Length]; + var dither = _options.Dither; + var threshold = _options.DifferenceThreshold; + var current = new float[(_width + 2) * 3]; + var next = new float[(_width + 2) * 3]; + for (var y = 0; y < _height; y++) + { + var reverse = dither && (y & 1) == 1; + var step = reverse ? -1 : 1; + for (var n = 0; n < _width; n++) + { + var x = reverse ? _width - 1 - n : n; + var i = y * _width + x; + var p = i * 3; + if (differencing + && Math.Abs(rgb[p] - _drawn[p]) <= threshold && Math.Abs(rgb[p + 1] - _drawn[p + 1]) <= threshold && Math.Abs(rgb[p + 2] - _drawn[p + 2]) <= threshold) + { + continue; + } + var e = (x + 1) * 3; + var r = Math.Clamp(rgb[p] + current[e], 0f, 255f); + var g = Math.Clamp(rgb[p + 1] + current[e + 1], 0f, 255f); + var b = Math.Clamp(rgb[p + 2] + current[e + 2], 0f, 255f); + var index = map.Nearest((int)(r + 0.5f), (int)(g + 0.5f), (int)(b + 0.5f)); + int outR = palette[index * 3], outG = palette[index * 3 + 1], outB = palette[index * 3 + 2]; + indices[i] = (byte)index; + changed[i] = !differencing || outR != _canvas![p] || outG != _canvas[p + 1] || outB != _canvas[p + 2]; + _drawn[p] = rgb[p]; + _drawn[p + 1] = rgb[p + 1]; + _drawn[p + 2] = rgb[p + 2]; + if (!dither) continue; + var er = r - outR; + var eg = g - outG; + var eb = b - outB; + var ahead = e + step * 3; + var behind = e - step * 3; + current[ahead] += er * (7 / 16f); + current[ahead + 1] += eg * (7 / 16f); + current[ahead + 2] += eb * (7 / 16f); + next[behind] += er * (3 / 16f); + next[behind + 1] += eg * (3 / 16f); + next[behind + 2] += eb * (3 / 16f); + next[e] += er * (5 / 16f); + next[e + 1] += eg * (5 / 16f); + next[e + 2] += eb * (5 / 16f); + next[ahead] += er * (1 / 16f); + next[ahead + 1] += eg * (1 / 16f); + next[ahead + 2] += eb * (1 / 16f); + } + (current, next) = (next, current); + Array.Clear(next); + } + return changed; + } + + private (int Left, int Top, int Right, int Bottom) Bounds(bool[] changed) + { + int left = _width, top = _height, right = -1, bottom = -1; + for (var y = 0; y < _height; y++) + { + for (var x = 0; x < _width; x++) + { + if (!changed[y * _width + x]) continue; + left = Math.Min(left, x); + right = Math.Max(right, x); + top = Math.Min(top, y); + bottom = y; + } + } + return (left, top, right, bottom); + } + + private void WriteHeader() + { + _output.Write("GIF89a"u8); + Span screen = stackalloc byte[7]; + screen[0] = (byte)_width; + screen[1] = (byte)(_width >> 8); + screen[2] = (byte)_height; + screen[3] = (byte)(_height >> 8); + var globalBits = 0; + if (_globalPalette is not null) + { + globalBits = TableBits(_globalPalette.Length / 3 + (_options.FrameDifferencing ? 1 : 0)); + screen[4] = (byte)(0x80 | 0x70 | (globalBits - 1)); + } + _output.Write(screen); + if (_globalPalette is not null) WriteColorTable(_globalPalette, globalBits); + if (_options.LoopCount is { } loops) + { + _output.Write([0x21, 0xFF, 0x0B]); + _output.Write("NETSCAPE2.0"u8); + _output.Write([0x03, 0x01, (byte)loops, (byte)(loops >> 8), 0x00]); + } + _headerWritten = true; + } + + /// Writes the pending frame with the delay up to (cumulative rounding keeps the total exact). + private void Emit(long end) + { + if (_pending is not { } frame) return; + var delay = (int)(Centiseconds(end) - Centiseconds(frame.Start)); + if (delay > ushort.MaxValue) + { + Log.Warning(LogCategory, "Frame delay above 655 s clamped"); + delay = ushort.MaxValue; + } + if (delay < 2 && !_warnedShortDelay) + { + Log.Warning(LogCategory, "Frame delays below 2 cs are slowed down by most viewers"); + _warnedShortDelay = true; + } + + Span gce = [0x21, 0xF9, 0x04, (byte)(1 << 2 | (frame.Transparent >= 0 ? 1 : 0)), (byte)delay, (byte)(delay >> 8), (byte)Math.Max(frame.Transparent, 0), 0x00]; + _output.Write(gce); + Span descriptor = + [ + 0x2C, (byte)frame.Left, (byte)(frame.Left >> 8), (byte)frame.Top, (byte)(frame.Top >> 8), + (byte)frame.Width, (byte)(frame.Width >> 8), (byte)frame.Height, (byte)(frame.Height >> 8), + frame.LocalPalette is null ? (byte)0 : (byte)(0x80 | (frame.TableBits - 1)), + ]; + _output.Write(descriptor); + if (frame.LocalPalette is not null) WriteColorTable(frame.LocalPalette, frame.TableBits); + GifLzw.Encode(frame.Indices, Math.Max(2, frame.TableBits), _output); + FrameCount++; + _pending = null; + } + + private void WriteColorTable(byte[] palette, int bits) + { + var table = new byte[3 << bits]; + palette.CopyTo(table, 0); + _output.Write(table); + } + + private static long Centiseconds(long ticks90k) => (ticks90k + 450) / 900; + + /// Colour table size exponent (1–8) holding entries. + private static int TableBits(int entries) + { + var bits = 1; + while (1 << bits < entries) bits++; + return bits; + } + + private static byte[] ToRgb(ImageBuffer frame) + { + var n = frame.Width * frame.Height; + var rgb = new byte[n * 3]; + var r = frame.Plane(0); + var g = frame.Channels >= 3 ? frame.Plane(1) : r; + var b = frame.Channels >= 3 ? frame.Plane(2) : r; + for (var i = 0; i < n; i++) + { + rgb[i * 3] = ColorSpace.Encode8(r[i]); + rgb[i * 3 + 1] = ColorSpace.Encode8(g[i]); + rgb[i * 3 + 2] = ColorSpace.Encode8(b[i]); + } + return rgb; + } + + /// Every n-th pixel so at most about pixels feed the palette (deterministic). + private static byte[] Subsample(byte[] rgb, int pixels, int budget) + { + var stride = Math.Max(1, pixels / Math.Max(budget, 4096)); + if (stride == 1) return rgb; + var result = new byte[(pixels + stride - 1) / stride * 3]; + for (int i = 0, o = 0; i < pixels; i += stride, o += 3) + { + result[o] = rgb[i * 3]; + result[o + 1] = rgb[i * 3 + 1]; + result[o + 2] = rgb[i * 3 + 2]; + } + return result; + } +} diff --git a/src/Titano.Core/Codecs/Gif/GifLzw.cs b/src/Titano.Core/Codecs/Gif/GifLzw.cs new file mode 100644 index 00000000..38fba552 --- /dev/null +++ b/src/Titano.Core/Codecs/Gif/GifLzw.cs @@ -0,0 +1,104 @@ +namespace Titano.Core.Codecs.Gif; + +/// GIF variable-length LZW encoder (GIF89a appendix F): clear/end codes, 12-bit cap, 255-byte sub-blocks. +internal static class GifLzw +{ + private const int MaxCode = 4096; + private const int TableSize = 8192; // power of two > MaxCode for open addressing + + /// Writes the LZW minimum code size, the data sub-blocks and the block terminator. + public static void Encode(ReadOnlySpan indices, int minCodeSize, Stream output) + { + output.WriteByte((byte)minCodeSize); + var writer = new BlockWriter(output); + var clear = 1 << minCodeSize; + var end = clear + 1; + var keys = new int[TableSize]; + var codes = new short[TableSize]; + Array.Fill(keys, -1); + var next = end + 1; + var codeSize = minCodeSize + 1; + writer.Write(clear, codeSize); + if (indices.IsEmpty) + { + writer.Write(end, codeSize); + writer.Finish(); + return; + } + + int prefix = indices[0]; + for (var i = 1; i < indices.Length; i++) + { + int k = indices[i]; + var key = prefix << 8 | k; + var slot = (int)(((uint)key * 0x9E3779B1u) >> 19); // 13 bits: 0..TableSize-1 + while (keys[slot] != -1 && keys[slot] != key) slot = (slot + 1) & (TableSize - 1); + if (keys[slot] == key) + { + prefix = codes[slot]; + continue; + } + writer.Write(prefix, codeSize); + if (next == MaxCode) + { + // Table full: restart the dictionary (the clear code is still written at 12 bits). + writer.Write(clear, codeSize); + Array.Fill(keys, -1); + next = end + 1; + codeSize = minCodeSize + 1; + } + else + { + if (next >= 1 << codeSize) codeSize++; + keys[slot] = key; + codes[slot] = (short)next++; + } + prefix = k; + } + writer.Write(prefix, codeSize); + writer.Write(end, codeSize); + writer.Finish(); + } + + /// Packs codes LSB-first into sub-blocks of at most 255 bytes. + private sealed class BlockWriter(Stream output) + { + private readonly byte[] _block = new byte[256]; + private int _length; + private ulong _bits; + private int _count; + + public void Write(int code, int size) + { + _bits |= (ulong)code << _count; + _count += size; + while (_count >= 8) + { + Put((byte)_bits); + _bits >>= 8; + _count -= 8; + } + } + + public void Finish() + { + if (_count > 0) Put((byte)_bits); + Flush(); + output.WriteByte(0); + } + + private void Put(byte b) + { + _block[1 + _length++] = b; + if (_length == 255) Flush(); + } + + private void Flush() + { + if (_length == 0) return; + _block[0] = (byte)_length; + output.Write(_block, 0, _length + 1); + _length = 0; + } + } +} diff --git a/src/Titano.Core/Codecs/Gif/GifPalette.cs b/src/Titano.Core/Codecs/Gif/GifPalette.cs new file mode 100644 index 00000000..d6f29823 --- /dev/null +++ b/src/Titano.Core/Codecs/Gif/GifPalette.cs @@ -0,0 +1,216 @@ +namespace Titano.Core.Codecs.Gif; + +/// Palette construction (exact or median cut) and nearest-colour mapping for GIF frames. +internal static class GifPalette +{ + private const int WeightR = 2, WeightG = 4, WeightB = 3; + + /// + /// Builds at most colours (RGB triples) from interleaved RGB samples: the exact set when + /// it is small enough, otherwise median cut over a 5-bit-per-channel histogram where, as in Wu's quantiser, the box + /// with the largest squared error is split at the position that minimises the summed error of both halves. + /// + public static byte[] Build(ReadOnlySpan rgb, int maxColors) + { + var distinct = new HashSet(); + for (var i = 0; i + 2 < rgb.Length && distinct.Count <= maxColors; i += 3) distinct.Add(rgb[i] << 16 | rgb[i + 1] << 8 | rgb[i + 2]); + if (distinct.Count <= maxColors) + { + var exact = distinct.Order().ToArray(); + var palette = new byte[exact.Length * 3]; + for (var i = 0; i < exact.Length; i++) + { + palette[i * 3] = (byte)(exact[i] >> 16); + palette[i * 3 + 1] = (byte)(exact[i] >> 8); + palette[i * 3 + 2] = (byte)exact[i]; + } + return palette; + } + return MedianCut(rgb, maxColors); + } + + /// Population moments of a set of histogram bins. + private struct Moments + { + public long N, R, G, B, Squares; + + public void Add(in Moments m) + { + N += m.N; + R += m.R; + G += m.G; + B += m.B; + Squares += m.Squares; + } + + public readonly Moments Minus(in Moments m) => new() { N = N - m.N, R = R - m.R, G = G - m.G, B = B - m.B, Squares = Squares - m.Squares }; + + /// Sum of squared distances to the mean colour. + public readonly double Error => N == 0 ? 0 : Squares - ((double)R * R + (double)G * G + (double)B * B) / N; + } + + private readonly record struct Box(int Start, int End, Moments Moments); + + private static byte[] MedianCut(ReadOnlySpan rgb, int maxColors) + { + var bins = new Moments[1 << 15]; + for (var i = 0; i + 2 < rgb.Length; i += 3) + { + int r = rgb[i], g = rgb[i + 1], b = rgb[i + 2]; + ref var m = ref bins[(r >> 3) << 10 | (g >> 3) << 5 | (b >> 3)]; + m.N++; + m.R += r; + m.G += g; + m.B += b; + m.Squares += r * r + g * g + b * b; + } + var used = new List(); + var total = default(Moments); + for (var b = 0; b < bins.Length; b++) + { + if (bins[b].N == 0) continue; + used.Add(b); + total.Add(bins[b]); + } + var order = used.ToArray(); + var keys = new int[order.Length]; + var boxes = new List { new(0, order.Length, total) }; + + while (boxes.Count < maxColors) + { + var pick = -1; + double worst = 0; + for (var i = 0; i < boxes.Count; i++) + { + if (boxes[i].End - boxes[i].Start > 1 && boxes[i].Moments.Error > worst) + { + worst = boxes[i].Moments.Error; + pick = i; + } + } + if (pick < 0) break; + var box = boxes[pick]; + var (axis, split, left) = BestSplit(order, keys, bins, box); + var shift = 10 - axis * 5; + for (var i = box.Start; i < box.End; i++) keys[i] = (order[i] >> shift) & 31; + Array.Sort(keys, order, box.Start, box.End - box.Start); + boxes[pick] = new Box(box.Start, split, left); + boxes.Add(new Box(split, box.End, box.Moments.Minus(left))); + } + + var palette = new byte[boxes.Count * 3]; + for (var p = 0; p < boxes.Count; p++) + { + var m = boxes[p].Moments; + palette[p * 3] = (byte)((m.R + m.N / 2) / m.N); + palette[p * 3 + 1] = (byte)((m.G + m.N / 2) / m.N); + palette[p * 3 + 2] = (byte)((m.B + m.N / 2) / m.N); + } + return palette; + } + + /// Tries each axis; returns the cut (first index of the right half) with the least summed error. + private static (int Axis, int Split, Moments Left) BestSplit(int[] order, int[] keys, Moments[] bins, Box box) + { + (int Axis, int Split, Moments Left) best = (0, box.Start + 1, bins[order[box.Start]]); + var bestError = double.MaxValue; + for (var axis = 0; axis < 3; axis++) + { + var shift = 10 - axis * 5; + for (var i = box.Start; i < box.End; i++) keys[i] = (order[i] >> shift) & 31; + Array.Sort(keys, order, box.Start, box.End - box.Start); + var left = default(Moments); + for (var i = box.Start; i < box.End - 1; i++) + { + left.Add(bins[order[i]]); + if (keys[i] == keys[i + 1]) continue; // cut only between distinct coordinates + var error = left.Error + box.Moments.Minus(left).Error; + if (error < bestError) + { + bestError = error; + best = (axis, i + 1, left); + } + } + } + return best; + } + + /// Nearest palette entry (weighted RGB distance) with a green-sorted pruned search and a colour cache. + public sealed class Map + { + private readonly byte[] _palette; + private readonly int[] _order; + private readonly int[] _green; + private readonly int[] _cacheKey = new int[1 << 16]; + private readonly byte[] _cacheValue = new byte[1 << 16]; + + public Map(byte[] palette) + { + _palette = palette; + var n = palette.Length / 3; + _order = new int[n]; + _green = new int[n]; + for (var i = 0; i < n; i++) + { + _order[i] = i; + _green[i] = palette[i * 3 + 1]; + } + Array.Sort(_green, _order); + } + + public int Nearest(int r, int g, int b) + { + var color = r << 16 | g << 8 | b; + var slot = (int)(((uint)color * 0x9E3779B1u) >> 16); + if (_cacheKey[slot] == color + 1) return _cacheValue[slot]; + + var best = int.MaxValue; + var bestIndex = 0; + var up = LowerBound(g); + var down = up - 1; + while (up < _green.Length || down >= 0) + { + if (up < _green.Length) + { + var dg = _green[up] - g; + if (WeightG * dg * dg >= best) up = _green.Length; + else Consider(_order[up++], r, g, b, ref best, ref bestIndex); + } + if (down >= 0) + { + var dg = g - _green[down]; + if (WeightG * dg * dg >= best) down = -1; + else Consider(_order[down--], r, g, b, ref best, ref bestIndex); + } + } + _cacheKey[slot] = color + 1; + _cacheValue[slot] = (byte)bestIndex; + return bestIndex; + } + + private void Consider(int index, int r, int g, int b, ref int best, ref int bestIndex) + { + var dr = _palette[index * 3] - r; + var dg = _palette[index * 3 + 1] - g; + var db = _palette[index * 3 + 2] - b; + var d = WeightR * dr * dr + WeightG * dg * dg + WeightB * db * db; + if (d < best) + { + best = d; + bestIndex = index; + } + } + + private int LowerBound(int g) + { + int lo = 0, hi = _green.Length; + while (lo < hi) + { + var mid = (lo + hi) >> 1; + if (_green[mid] < g) lo = mid + 1; + else hi = mid; + } + return lo; + } + } +} diff --git a/tests/Titano.Core.Tests/Video/GifEncoderTests.cs b/tests/Titano.Core.Tests/Video/GifEncoderTests.cs new file mode 100644 index 00000000..4ac317cc --- /dev/null +++ b/tests/Titano.Core.Tests/Video/GifEncoderTests.cs @@ -0,0 +1,255 @@ +using Titano.Core.Codecs.Gif; +using Titano.Core.Imaging; +using Titano.Testing; + +namespace Titano.Core.Tests.Video; + +public sealed class GifEncoderTests(TestContext context) +{ + private const int W = 64, H = 48; + private const long Frame30 = 3000; // 1/30 s in 90 kHz ticks + + [Test] + public void SmoothFramesRoundTripWithinPaletteError() + { + var frames = Enumerable.Range(0, 4).Select(i => Smooth(i * 3)).ToList(); + var plain = Encode(frames, new GifEncoderOptions { Dither = false }); + var dithered = Encode(frames, new GifEncoderOptions()); + Assert.Equal(4, plain.Frames.Count); + Assert.True(plain.GlobalTable); + for (var f = 0; f < frames.Count; f++) + { + var (mean, max) = Error(frames[f], plain.Frames[f].Canvas); + Assert.Less(mean, 6.0, $"frame {f} mean error"); + Assert.Less(max, 40, $"frame {f} max error"); + context.Log($"frame {f}: plain {mean:0.00}/{max}, dithered tiles {TileError(frames[f], dithered.Frames[f].Canvas):0.00}"); + // Dithering trades per-pixel error for accurate local averages. + Assert.Less(TileError(frames[f], dithered.Frames[f].Canvas), 2.0, $"frame {f} 4×4 average error"); + } + } + + /// Mean absolute difference of 4×4 tile averages. + private static double TileError(byte[] rgb, byte[] rgba) + { + double sum = 0; + var tiles = 0; + for (var ty = 0; ty < H / 4; ty++) + { + for (var tx = 0; tx < W / 4; tx++) + { + for (var c = 0; c < 3; c++) + { + double a = 0, b = 0; + for (var y = 0; y < 4; y++) + { + for (var x = 0; x < 4; x++) + { + var i = (ty * 4 + y) * W + tx * 4 + x; + a += rgb[i * 3 + c]; + b += rgba[i * 4 + c]; + } + } + sum += Math.Abs(a - b) / 16; + tiles++; + } + } + } + return sum / tiles; + } + + [Test] + public void ExactColoursReproduceExactly() + { + var palette = RandomColours(120, 5); + var frames = Enumerable.Range(0, 3).Select(i => Blocks(palette, i)).ToList(); + var gif = Encode(frames, new GifEncoderOptions { Dither = false, DifferenceThreshold = 0 }); + for (var f = 0; f < frames.Count; f++) Assert.Equal((0.0, 0), Error(frames[f], gif.Frames[f].Canvas)); + } + + [Test] + public void DelaysFollowTheCumulativeTimelineAndLoopIsWritten() + { + var frames = Enumerable.Range(0, 10).Select(i => Blocks(RandomColours(8, i), i)).ToList(); + var gif = Encode(frames, new GifEncoderOptions()); + Assert.Equal(0, gif.LoopCount); + Assert.SequenceEqual([3, 4, 3, 3, 4, 3, 3, 4, 3, 3], gif.Frames.Select(f => f.DelayCs)); + Assert.Equal(33, gif.Frames.Sum(f => f.DelayCs)); + + Assert.Equal(5, Encode(frames, new GifEncoderOptions { LoopCount = 5 }).LoopCount); + Assert.Null(Encode(frames, new GifEncoderOptions { LoopCount = null }).LoopCount); + } + + [Test] + public void DifferencingCropsToTheChangedAreaAndKeepsTheRest() + { + var palette = RandomColours(64, 9); + var a = Blocks(palette, 0); + var b = (byte[])a.Clone(); + for (var y = 7; y < 11; y++) + { + for (var x = 10; x < 15; x++) + { + var i = (y * W + x) * 3; + b[i] = (byte)(255 - b[i]); + b[i + 1] = (byte)(255 - b[i + 1]); + } + } + var gif = Encode([a, b], new GifEncoderOptions { Dither = false, DifferenceThreshold = 0 }); + Assert.Equal(2, gif.Frames.Count); + var second = gif.Frames[1]; + Assert.Equal((10, 7, 5, 4), (second.Left, second.Top, second.Width, second.Height)); + Assert.True(second.Transparent); + Assert.Equal(1, second.Disposal); + Assert.Equal(1, gif.Frames[0].Disposal); + Assert.False(gif.Frames[0].Transparent); + Assert.Equal((0.0, 0), Error(b, second.Canvas)); + } + + [Test] + public void IdenticalFramesMergeIntoOneLongerFrame() + { + var palette = RandomColours(16, 2); + var a = Blocks(palette, 0); + var b = Blocks(palette, 1); + var gif = Encode([a, a, a, b], new GifEncoderOptions { Dither = false }); + Assert.Equal(2, gif.Frames.Count); + Assert.Equal(10, gif.Frames[0].DelayCs); + Assert.Equal(3, gif.Frames[1].DelayCs); + } + + [Test] + public void LocalPalettesFollowEachFrame() + { + var reds = Enumerable.Range(0, 200).SelectMany(i => new[] { (byte)(55 + i), (byte)(i / 4), (byte)0 }).ToArray(); + var blues = Enumerable.Range(0, 200).SelectMany(i => new[] { (byte)0, (byte)(i / 4), (byte)(55 + i) }).ToArray(); + var frames = new List { Blocks(reds, 0), Blocks(blues, 0) }; + var gif = Encode(frames, new GifEncoderOptions { PaletteMode = GifPaletteMode.Local, Dither = false, DifferenceThreshold = 0 }); + Assert.False(gif.GlobalTable); + Assert.True(gif.Frames.All(f => f.LocalTable)); + for (var f = 0; f < 2; f++) Assert.Equal((0.0, 0), Error(frames[f], gif.Frames[f].Canvas)); + } + + [Test] + public void LzwSurvivesDictionaryResets() + { + // 200×200 pixels of random indices fill the 4096-entry dictionary many times over. + const int size = 200; + var palette = RandomColours(200, 4); + var rng = new Random(8); + var rgb = new byte[size * size * 3]; + for (var i = 0; i < size * size; i++) + { + var c = rng.Next(200); + rgb[i * 3] = palette[c * 3]; + rgb[i * 3 + 1] = palette[c * 3 + 1]; + rgb[i * 3 + 2] = palette[c * 3 + 2]; + } + using var ms = new MemoryStream(); + var encoder = new GifEncoder(ms, size, size, new GifEncoderOptions { Dither = false, FrameDifferencing = false }); + encoder.AddFrame(rgb, Frame30); + encoder.Complete(); + var gif = GifTestDecoder.Decode(ms.ToArray()); + Assert.Equal((0.0, 0), Error(rgb, gif.Frames[0].Canvas)); + } + + [Test] + public void LinearImageBuffersAreEncodedAsSrgb() + { + var palette = RandomColours(30, 6); + var rgb = Blocks(palette, 3); + using var image = new ImageBuffer(W, H); + for (var i = 0; i < W * H; i++) + { + for (var c = 0; c < 3; c++) image.Plane(c)[i] = ColorSpace.Linear8(rgb[i * 3 + c]); + } + using var ms = new MemoryStream(); + var encoder = new GifEncoder(ms, W, H, new GifEncoderOptions { Dither = false }); + encoder.AddFrame(image, Frame30); + encoder.Complete(); + Assert.Equal(1, encoder.FrameCount); + Assert.Equal((0.0, 0), Error(rgb, GifTestDecoder.Decode(ms.ToArray()).Frames[0].Canvas)); + } + + [Test] + public void SuppliedPaletteIsUsedAsGlobalTable() + { + using var image = new ImageBuffer(W, H); + image.Plane(0).Fill(0.5f); + var palette = GifEncoder.BuildPalette([image], 4); + Assert.Equal(3, palette.Length); + using var ms = new MemoryStream(); + var encoder = new GifEncoder(ms, W, H, new GifEncoderOptions { Palette = palette }); + encoder.AddFrame(image, Frame30); + encoder.Complete(); + var gif = GifTestDecoder.Decode(ms.ToArray()); + Assert.True(gif.GlobalTable); + Assert.Equal(ColorSpace.Encode8(0.5f), gif.Frames[0].Canvas[0]); + } + + private static GifTestDecoder Encode(List frames, GifEncoderOptions options) + { + using var ms = new MemoryStream(); + var encoder = new GifEncoder(ms, W, H, options); + foreach (var f in frames) encoder.AddFrame(f, Frame30); + encoder.Complete(); + return GifTestDecoder.Decode(ms.ToArray()); + } + + private static (double Mean, int Max) Error(byte[] rgb, byte[] rgba) + { + long sum = 0; + var max = 0; + for (var i = 0; i < rgb.Length / 3; i++) + { + for (var c = 0; c < 3; c++) + { + var d = Math.Abs(rgb[i * 3 + c] - rgba[i * 4 + c]); + sum += d; + max = Math.Max(max, d); + } + } + return (sum / (double)rgb.Length, max); + } + + private static byte[] Smooth(int shift) + { + var rgb = new byte[W * H * 3]; + for (var y = 0; y < H; y++) + { + for (var x = 0; x < W; x++) + { + var i = (y * W + x) * 3; + rgb[i] = (byte)((x + shift) * 255 / (W + 12)); + rgb[i + 1] = (byte)(y * 255 / H); + rgb[i + 2] = (byte)(128 + 100 * Math.Sin((x + y + shift) * 0.1)); + } + } + return rgb; + } + + private static byte[] RandomColours(int count, int seed) + { + var bytes = new byte[count * 3]; + new Random(seed).NextBytes(bytes); + return bytes; + } + + /// 4×4 blocks coloured from , pattern varying with . + private static byte[] Blocks(byte[] palette, int variant) + { + var colours = palette.Length / 3; + var rgb = new byte[W * H * 3]; + for (var y = 0; y < H; y++) + { + for (var x = 0; x < W; x++) + { + var c = ((x / 4) * 7 + (y / 4) * 13 + variant * 5) % colours; + var i = (y * W + x) * 3; + rgb[i] = palette[c * 3]; + rgb[i + 1] = palette[c * 3 + 1]; + rgb[i + 2] = palette[c * 3 + 2]; + } + } + return rgb; + } +} diff --git a/tests/Titano.Core.Tests/Video/GifTestDecoder.cs b/tests/Titano.Core.Tests/Video/GifTestDecoder.cs new file mode 100644 index 00000000..1283fae6 --- /dev/null +++ b/tests/Titano.Core.Tests/Video/GifTestDecoder.cs @@ -0,0 +1,197 @@ +using System.Text; +using Titano.Testing; + +namespace Titano.Core.Tests.Video; + +/// Minimal GIF89a decoder used to verify GifEncoder output (composition with disposal 0–3). +internal sealed class GifTestDecoder +{ + public sealed record Frame(byte[] Canvas, int DelayCs, int Left, int Top, int Width, int Height, int Disposal, bool Transparent, bool LocalTable); + + public int Width { get; private set; } + public int Height { get; private set; } + public bool GlobalTable { get; private set; } + public int? LoopCount { get; private set; } + public List Frames { get; } = []; + + public static GifTestDecoder Decode(byte[] data) + { + var gif = new GifTestDecoder(); + Assert.Equal("GIF89a", Encoding.ASCII.GetString(data, 0, 6)); + gif.Width = data[6] | data[7] << 8; + gif.Height = data[8] | data[9] << 8; + var pos = 13; + byte[]? global = null; + if ((data[10] & 0x80) != 0) + { + gif.GlobalTable = true; + var size = 3 << ((data[10] & 7) + 1); + global = data.AsSpan(pos, size).ToArray(); + pos += size; + } + var canvas = new byte[gif.Width * gif.Height * 4]; + int delay = 0, disposal = 0, transparentIndex = -1; + while (true) + { + var block = data[pos++]; + if (block == 0x3B) break; + if (block == 0x21) + { + var label = data[pos++]; + if (label == 0xF9) + { + Assert.Equal((byte)4, data[pos]); + disposal = (data[pos + 1] >> 2) & 7; + transparentIndex = (data[pos + 1] & 1) != 0 ? data[pos + 4] : -1; + delay = data[pos + 2] | data[pos + 3] << 8; + pos += 6; + } + else if (label == 0xFF && Encoding.ASCII.GetString(data, pos + 1, 11) == "NETSCAPE2.0") + { + gif.LoopCount = data[pos + 14] | data[pos + 15] << 8; + pos += 12; + pos = SkipSubBlocks(data, pos); + } + else + { + pos = SkipSubBlocks(data, pos); + } + continue; + } + Assert.Equal((byte)0x2C, block); + var left = data[pos] | data[pos + 1] << 8; + var top = data[pos + 2] | data[pos + 3] << 8; + var w = data[pos + 4] | data[pos + 5] << 8; + var h = data[pos + 6] | data[pos + 7] << 8; + var packed = data[pos + 8]; + pos += 9; + var table = global; + var local = (packed & 0x80) != 0; + if (local) + { + var size = 3 << ((packed & 7) + 1); + table = data.AsSpan(pos, size).ToArray(); + pos += size; + } + Assert.False((packed & 0x40) != 0, "interlaced frames are not produced"); + Assert.True(left + w <= gif.Width && top + h <= gif.Height, "frame inside the logical screen"); + var minCode = data[pos++]; + using var lzw = new MemoryStream(); + while (data[pos] != 0) + { + lzw.Write(data, pos + 1, data[pos]); + pos += data[pos] + 1; + } + pos++; + var indices = DecodeLzw(lzw.ToArray(), minCode, w * h); + var saved = disposal == 3 ? (byte[])canvas.Clone() : null; + for (var y = 0; y < h; y++) + { + for (var x = 0; x < w; x++) + { + var index = indices[y * w + x]; + if (index == transparentIndex) continue; + var o = ((top + y) * gif.Width + left + x) * 4; + canvas[o] = table![index * 3]; + canvas[o + 1] = table[index * 3 + 1]; + canvas[o + 2] = table[index * 3 + 2]; + canvas[o + 3] = 255; + } + } + gif.Frames.Add(new Frame((byte[])canvas.Clone(), delay, left, top, w, h, disposal, transparentIndex >= 0, local)); + if (disposal == 2) + { + for (var y = 0; y < h; y++) canvas.AsSpan(((top + y) * gif.Width + left) * 4, w * 4).Clear(); + } + else if (disposal == 3) + { + canvas = saved!; + } + delay = 0; + disposal = 0; + transparentIndex = -1; + } + return gif; + } + + private static int SkipSubBlocks(byte[] data, int pos) + { + while (data[pos] != 0) pos += data[pos] + 1; + return pos + 1; + } + + private static byte[] DecodeLzw(byte[] data, int minCode, int pixels) + { + var output = new List(pixels); + var prefix = new short[4096]; + var suffix = new byte[4096]; + var stack = new byte[4097]; + var clear = 1 << minCode; + var end = clear + 1; + var codeSize = minCode + 1; + var next = end + 1; + var old = -1; + long bits = 0; + var count = 0; + var pos = 0; + while (true) + { + while (count < codeSize && pos < data.Length) + { + bits |= (long)data[pos++] << count; + count += 8; + } + Assert.True(count >= codeSize, "LZW data ended before the end code"); + var code = (int)(bits & ((1 << codeSize) - 1)); + bits >>= codeSize; + count -= codeSize; + if (code == clear) + { + codeSize = minCode + 1; + next = end + 1; + old = -1; + continue; + } + if (code == end) break; + if (old == -1) + { + Assert.True(code < clear, "first code after clear is a literal"); + output.Add((byte)code); + old = code; + continue; + } + Assert.True(code <= next, $"LZW code {code} beyond the table ({next})"); + var depth = 0; + var c = code; + if (code == next) + { + stack[depth++] = First(old, prefix, suffix, clear); + c = old; + } + while (c >= clear) + { + stack[depth++] = suffix[c]; + c = prefix[c]; + } + stack[depth++] = (byte)c; + var firstByte = (byte)c; + for (var i = depth - 1; i >= 0; i--) output.Add(stack[i]); + if (next < 4096) + { + prefix[next] = (short)old; + suffix[next] = firstByte; + next++; + if (next == 1 << codeSize && codeSize < 12) codeSize++; + } + old = code; + } + Assert.Equal(pixels, output.Count); + return [.. output]; + } + + private static byte First(int code, short[] prefix, byte[] suffix, int clear) + { + while (code >= clear) code = prefix[code]; + return (byte)code; + } +} From 04f31e7f436ebd19bb18ea3e6bc93663ba44ce3a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 09:51:37 +0000 Subject: [PATCH 7/9] feat(apng): add APNG encoder with frame differencing and exact delays Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Codecs/Apng/ApngEncoder.cs | 246 ++++++++++++++++++ .../Video/ApngEncoderTests.cs | 221 ++++++++++++++++ 2 files changed, 467 insertions(+) create mode 100644 src/Titano.Core/Codecs/Apng/ApngEncoder.cs create mode 100644 tests/Titano.Core.Tests/Video/ApngEncoderTests.cs diff --git a/src/Titano.Core/Codecs/Apng/ApngEncoder.cs b/src/Titano.Core/Codecs/Apng/ApngEncoder.cs new file mode 100644 index 00000000..5047d90a --- /dev/null +++ b/src/Titano.Core/Codecs/Apng/ApngEncoder.cs @@ -0,0 +1,246 @@ +using System.Buffers.Binary; +using System.IO.Compression; +using Titano.Core.Codecs.Png; +using Titano.Core.Imaging; + +namespace Titano.Core.Codecs.Apng; + +/// Options of . +public sealed record ApngEncoderOptions +{ + /// Write RGBA (alpha taken from 4-channel buffers); otherwise RGB. + public bool Alpha { get; init; } + + /// acTL num_plays: 0 loops forever. + public int LoopCount { get; init; } + + /// Crop frames to the changed area (blend OVER with transparent unchanged pixels when possible). + public bool FrameDifferencing { get; init; } = true; + + public CompressionLevel CompressionLevel { get; init; } = CompressionLevel.Optimal; +} + +/// +/// Animated PNG encoder (8-bit RGB/RGBA): acTL/fcTL/IDAT/fdAT with one shared sequence counter; the first frame is +/// the default image. Delays are exact fractions of the 90 kHz durations when they fit 16 bits, otherwise rounded on +/// the cumulative timeline. Identical consecutive frames merge. The output must be seekable (acTL is patched at the +/// end) and is not disposed. +/// +public sealed class ApngEncoder +{ + private const int TimeScale = 90_000; + private const byte DisposeNone = 0; + private const byte BlendSource = 0, BlendOver = 1; + + private readonly Stream _output; + private readonly int _width; + private readonly int _height; + private readonly int _bpp; + private readonly ApngEncoderOptions _options; + private readonly long _actlOffset; + private byte[]? _canvas; + private Pending? _pending; + private long _time; + private uint _sequence; + private bool _completed; + + public ApngEncoder(Stream output, int width, int height, ApngEncoderOptions? options = null) + { + ArgumentNullException.ThrowIfNull(output); + if (!output.CanSeek || !output.CanWrite) throw new ArgumentException("The output stream must be writable and seekable", nameof(output)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(width); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(height); + _options = options ?? new ApngEncoderOptions(); + ArgumentOutOfRangeException.ThrowIfNegative(_options.LoopCount, nameof(options)); + _output = output; + _width = width; + _height = height; + _bpp = _options.Alpha ? 4 : 3; + + output.Write([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]); + Span ihdr = stackalloc byte[13]; + BinaryPrimitives.WriteInt32BigEndian(ihdr, width); + BinaryPrimitives.WriteInt32BigEndian(ihdr[4..], height); + ihdr[8] = 8; + ihdr[9] = (byte)(_options.Alpha ? PngColorType.Rgba : PngColorType.Rgb); + PngEncoder.WriteChunk(output, "IHDR", ihdr); + _actlOffset = output.Position; + WriteAnimationControl(0); + } + + /// Frames written so far (merged identical frames count once). + public int FrameCount { get; private set; } + + /// Adds a linear-light frame shown for ticks. + public void AddFrame(ImageBuffer frame, long duration90k) + { + if (frame.Width != _width || frame.Height != _height) throw new ArgumentException("Frame size differs from the animation size", nameof(frame)); + var n = _width * _height; + var pixels = new byte[n * _bpp]; + var r = frame.Plane(0); + var g = frame.Channels >= 3 ? frame.Plane(1) : r; + var b = frame.Channels >= 3 ? frame.Plane(2) : r; + var a = frame.Channels == 4 ? frame.Plane(3) : default; + for (var i = 0; i < n; i++) + { + var o = i * _bpp; + pixels[o] = ColorSpace.Encode8(r[i]); + pixels[o + 1] = ColorSpace.Encode8(g[i]); + pixels[o + 2] = ColorSpace.Encode8(b[i]); + if (_bpp == 4) pixels[o + 3] = a.IsEmpty ? (byte)255 : (byte)(Math.Clamp(a[i], 0f, 1f) * 255f + 0.5f); + } + AddPixels(pixels, duration90k); + } + + /// Adds interleaved 8-bit sRGB pixels (RGB, or RGBA when ). + public void AddFrame(ReadOnlySpan pixels, long duration90k) + { + if (pixels.Length != _width * _height * _bpp) throw new ArgumentException($"Expected width × height × {_bpp} bytes", nameof(pixels)); + AddPixels(pixels.ToArray(), duration90k); + } + + /// Writes the last frame and IEND, then patches the frame count. + public void Complete() + { + if (_completed) throw new InvalidOperationException("The encoder is already complete"); + if (_pending is null) throw new InvalidOperationException("No frames were added"); + Emit(_time); + PngEncoder.WriteChunk(_output, "IEND", []); + var end = _output.Position; + _output.Position = _actlOffset; + WriteAnimationControl(FrameCount); + _output.Position = end; + _output.Flush(); + _completed = true; + } + + private sealed record Pending(long Start, int X, int Y, int Width, int Height, byte Blend, byte[] Data); + + private void AddPixels(byte[] pixels, long duration) + { + if (_completed) throw new InvalidOperationException("The encoder is complete"); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(duration); + int x = 0, y = 0, w = _width, h = _height; + var blend = BlendSource; + var region = pixels; + if (_canvas is not null && _options.FrameDifferencing) + { + (x, y, w, h) = ChangedArea(pixels, _canvas); + if (w == 0) + { + _time += duration; // identical frame: the previous one stays longer + return; + } + region = Crop(pixels, x, y, w, h, out var opaque); + if (_bpp == 4 && opaque) + { + // Unchanged pixels become fully transparent and OVER keeps what is displayed beneath them. + for (var row = 0; row < h; row++) + { + for (var col = 0; col < w; col++) + { + var c = ((y + row) * _width + x + col) * 4; + if (pixels.AsSpan(c, 4).SequenceEqual(_canvas.AsSpan(c, 4))) region.AsSpan((row * w + col) * 4, 4).Clear(); + } + } + blend = BlendOver; + } + } + _canvas = pixels; + var data = PngEncoder.Compress(region, h, w * _bpp, _bpp, _options.CompressionLevel); + Emit(_time); + _pending = new Pending(_time, x, y, w, h, blend, data); + _time += duration; + } + + private (int X, int Y, int Width, int Height) ChangedArea(byte[] pixels, byte[] canvas) + { + int left = _width, top = _height, right = -1, bottom = -1; + for (var y = 0; y < _height; y++) + { + var row = y * _width * _bpp; + for (var x = 0; x < _width; x++) + { + var o = row + x * _bpp; + if (pixels.AsSpan(o, _bpp).SequenceEqual(canvas.AsSpan(o, _bpp))) continue; + left = Math.Min(left, x); + right = Math.Max(right, x); + top = Math.Min(top, y); + bottom = y; + } + } + return right < 0 ? (0, 0, 0, 0) : (left, top, right - left + 1, bottom - top + 1); + } + + private byte[] Crop(byte[] pixels, int x, int y, int w, int h, out bool opaque) + { + var region = new byte[w * h * _bpp]; + opaque = true; + for (var row = 0; row < h; row++) pixels.AsSpan(((y + row) * _width + x) * _bpp, w * _bpp).CopyTo(region.AsSpan(row * w * _bpp)); + if (_bpp == 4) + { + for (var i = 3; i < region.Length && opaque; i += 4) opaque = region[i] == 255; + } + return region; + } + + private void Emit(long end) + { + if (_pending is not { } frame) return; + var (num, den) = Delay(frame.Start, end); + Span fctl = stackalloc byte[26]; + BinaryPrimitives.WriteUInt32BigEndian(fctl, _sequence++); + BinaryPrimitives.WriteInt32BigEndian(fctl[4..], frame.Width); + BinaryPrimitives.WriteInt32BigEndian(fctl[8..], frame.Height); + BinaryPrimitives.WriteInt32BigEndian(fctl[12..], frame.X); + BinaryPrimitives.WriteInt32BigEndian(fctl[16..], frame.Y); + BinaryPrimitives.WriteUInt16BigEndian(fctl[20..], num); + BinaryPrimitives.WriteUInt16BigEndian(fctl[22..], den); + fctl[24] = DisposeNone; + fctl[25] = frame.Blend; + PngEncoder.WriteChunk(_output, "fcTL", fctl); + if (FrameCount == 0) + { + PngEncoder.WriteChunk(_output, "IDAT", frame.Data); + } + else + { + var fdat = new byte[4 + frame.Data.Length]; + BinaryPrimitives.WriteUInt32BigEndian(fdat, _sequence++); + frame.Data.CopyTo(fdat, 4); + PngEncoder.WriteChunk(_output, "fdAT", fdat); + } + FrameCount++; + _pending = null; + } + + /// Exact reduced fraction of a second when it fits 16 bits; otherwise the cumulative timeline rounded to 1/1000 … 1 s. + private static (ushort Num, ushort Den) Delay(long start, long end) + { + var duration = end - start; + var g = Gcd(duration, TimeScale); + if (duration / g <= ushort.MaxValue && TimeScale / g <= ushort.MaxValue) return ((ushort)(duration / g), (ushort)(TimeScale / g)); + foreach (var den in (ReadOnlySpan)[1000, 100, 10, 1]) + { + var num = Rounded(end, den) - Rounded(start, den); + if (num <= ushort.MaxValue) return ((ushort)num, (ushort)den); + } + return (ushort.MaxValue, 1); + + static long Rounded(long ticks, int den) => (ticks * den + TimeScale / 2) / TimeScale; + } + + private static long Gcd(long a, long b) + { + while (b != 0) (a, b) = (b, a % b); + return a; + } + + private void WriteAnimationControl(int frames) + { + Span actl = stackalloc byte[8]; + BinaryPrimitives.WriteInt32BigEndian(actl, frames); + BinaryPrimitives.WriteInt32BigEndian(actl[4..], _options.LoopCount); + PngEncoder.WriteChunk(_output, "acTL", actl); + } +} diff --git a/tests/Titano.Core.Tests/Video/ApngEncoderTests.cs b/tests/Titano.Core.Tests/Video/ApngEncoderTests.cs new file mode 100644 index 00000000..db4eba39 --- /dev/null +++ b/tests/Titano.Core.Tests/Video/ApngEncoderTests.cs @@ -0,0 +1,221 @@ +using System.Buffers.Binary; +using System.IO.Compression; +using System.Text; +using Titano.Core.Codecs.Apng; +using Titano.Core.Hashing; +using Titano.Core.Imaging; +using Titano.Testing; + +namespace Titano.Core.Tests.Video; + +public sealed class ApngEncoderTests +{ + private const int W = 40, H = 30; + + [Test] + public void ChunkOrderSequenceNumbersAndCrcsAreValid() + { + var frames = Enumerable.Range(0, 5).Select(i => Pattern(i, 3)).ToList(); + var bytes = Encode(frames, new ApngEncoderOptions(), 3000); + var chunks = Chunks(bytes); + Assert.SequenceEqual( + ["IHDR", "acTL", "fcTL", "IDAT", "fcTL", "fdAT", "fcTL", "fdAT", "fcTL", "fdAT", "fcTL", "fdAT", "IEND"], + chunks.Select(c => c.Type)); + var actl = chunks[1].Data; + Assert.Equal(5, BinaryPrimitives.ReadInt32BigEndian(actl)); + Assert.Equal(0, BinaryPrimitives.ReadInt32BigEndian(actl.AsSpan(4))); + var sequence = chunks.Where(c => c.Type is "fcTL" or "fdAT").Select(c => BinaryPrimitives.ReadUInt32BigEndian(c.Data)).ToList(); + Assert.SequenceEqual(Enumerable.Range(0, sequence.Count).Select(i => (uint)i), sequence); + var first = chunks[2].Data; + Assert.Equal((W, H, 0, 0), (BinaryPrimitives.ReadInt32BigEndian(first.AsSpan(4)), BinaryPrimitives.ReadInt32BigEndian(first.AsSpan(8)), + BinaryPrimitives.ReadInt32BigEndian(first.AsSpan(12)), BinaryPrimitives.ReadInt32BigEndian(first.AsSpan(16)))); + Assert.Equal((1, 30), Delay(first)); + } + + [Test] + public void FirstFrameIsTheDefaultImageAndAllFramesComposeExactly() + { + var frames = new List { Pattern(0, 3), Pattern(0, 3), Pattern(1, 3) }; + // Frame 3 changes only a small block of frame 2. + var last = (byte[])frames[2].Clone(); + for (var y = 5; y < 9; y++) + { + for (var x = 20; x < 26; x++) last[(y * W + x) * 3] ^= 0xFF; + } + frames.Add(last); + var bytes = Encode(frames, new ApngEncoderOptions(), 1500); + var decoded = Decode(bytes, 3); + Assert.Equal(3, decoded.Count); // the duplicate merged into the first frame + Assert.BytesEqual(frames[0], decoded[0].Canvas); + Assert.BytesEqual(frames[2], decoded[1].Canvas); + Assert.BytesEqual(frames[3], decoded[2].Canvas); + Assert.Equal((1, 30), decoded[0].Delay); + Assert.Equal((20, 5, 6, 4), decoded[2].Rect); + Assert.Equal(0, decoded[2].Blend); // RGB output always uses SOURCE + } + + [Test] + public void RgbaDifferencingUsesOverWithTransparentUnchangedPixels() + { + var a = Pattern(2, 4); + var b = (byte[])a.Clone(); + b[(10 * W + 10) * 4] ^= 0x80; + b[(12 * W + 14) * 4 + 1] ^= 0x80; + var bytes = Encode([a, b], new ApngEncoderOptions { Alpha = true }, 3000); + var decoded = Decode(bytes, 4); + Assert.Equal(2, decoded.Count); + Assert.Equal(1, decoded[1].Blend); + Assert.Equal((10, 10, 5, 3), decoded[1].Rect); + Assert.BytesEqual(b, decoded[1].Canvas); + } + + [Theory] + [Case(3003L, 1001, 30000)] + [Case(90000L * 30, 30, 1)] + [Case(3001L, 33, 1000)] + public void DelaysAreExactFractionsOrCumulativelyRounded(long duration, int num, int den) + { + var bytes = Encode([Pattern(0, 3), Pattern(1, 3)], new ApngEncoderOptions(), duration); + var fctl = Chunks(bytes).Where(c => c.Type == "fcTL").ToList(); + Assert.Equal((num, den), Delay(fctl[0].Data)); + } + + [Test] + public void LinearBuffersAreEncodedAsSrgb() + { + using var image = new ImageBuffer(W, H, 4); + image.Plane(0).Fill(ColorSpace.Linear8(200)); + image.Plane(1).Fill(ColorSpace.Linear8(100)); + image.Plane(3).Fill(0.5f); + using var ms = new MemoryStream(); + var encoder = new ApngEncoder(ms, W, H, new ApngEncoderOptions { Alpha = true, LoopCount = 2 }); + encoder.AddFrame(image, 9000); + encoder.Complete(); + var decoded = Decode(ms.ToArray(), 4); + Assert.SequenceEqual(new byte[] { 200, 100, 0, 128 }, decoded[0].Canvas.Take(4)); + Assert.Equal(2, BinaryPrimitives.ReadInt32BigEndian(Chunks(ms.ToArray())[1].Data.AsSpan(4))); + } + + private static byte[] Encode(List frames, ApngEncoderOptions options, long duration) + { + using var ms = new MemoryStream(); + var encoder = new ApngEncoder(ms, W, H, options); + foreach (var f in frames) encoder.AddFrame(f, duration); + encoder.Complete(); + return ms.ToArray(); + } + + private static (int, int) Delay(byte[] fctl) => + (BinaryPrimitives.ReadUInt16BigEndian(fctl.AsSpan(20)), BinaryPrimitives.ReadUInt16BigEndian(fctl.AsSpan(22))); + + private sealed record Chunk(string Type, byte[] Data); + + /// Splits the PNG into chunks, checking the signature, lengths and every CRC. + private static List Chunks(byte[] png) + { + Assert.BytesEqual([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], png.AsSpan(0, 8)); + var list = new List(); + var pos = 8; + while (pos < png.Length) + { + var length = BinaryPrimitives.ReadInt32BigEndian(png.AsSpan(pos)); + var type = Encoding.ASCII.GetString(png, pos + 4, 4); + Assert.Equal(BinaryPrimitives.ReadUInt32BigEndian(png.AsSpan(pos + 8 + length)), Crc32.Compute(png.AsSpan(pos + 4, length + 4)), $"CRC of {type}"); + list.Add(new Chunk(type, png.AsSpan(pos + 8, length).ToArray())); + pos += 12 + length; + } + Assert.Equal(png.Length, pos); + return list; + } + + private sealed record Frame(byte[] Canvas, (int, int) Delay, (int X, int Y, int W, int H) Rect, int Blend); + + /// Inflates, unfilters and composites every frame (dispose NONE, blend SOURCE/OVER with binary alpha). + private static List Decode(byte[] png, int bpp) + { + var chunks = Chunks(png); + var canvas = new byte[W * H * bpp]; + var frames = new List(); + for (var i = 0; i < chunks.Count; i++) + { + if (chunks[i].Type != "fcTL") continue; + var f = chunks[i].Data; + int w = BinaryPrimitives.ReadInt32BigEndian(f.AsSpan(4)), h = BinaryPrimitives.ReadInt32BigEndian(f.AsSpan(8)); + int x = BinaryPrimitives.ReadInt32BigEndian(f.AsSpan(12)), y = BinaryPrimitives.ReadInt32BigEndian(f.AsSpan(16)); + Assert.Equal((byte)0, f[24]); + var data = chunks[i + 1].Type == "IDAT" ? chunks[i + 1].Data : chunks[i + 1].Data[4..]; + var pixels = Unfilter(Inflate(data), w, h, bpp); + for (var row = 0; row < h; row++) + { + for (var col = 0; col < w; col++) + { + var s = (row * w + col) * bpp; + var d = ((y + row) * W + x + col) * bpp; + if (f[25] == 1 && bpp == 4 && pixels[s + 3] == 0) continue; + pixels.AsSpan(s, bpp).CopyTo(canvas.AsSpan(d)); + } + } + frames.Add(new Frame((byte[])canvas.Clone(), Delay(f), (x, y, w, h), f[25])); + } + return frames; + } + + private static byte[] Inflate(byte[] zlib) + { + using var z = new ZLibStream(new MemoryStream(zlib), CompressionMode.Decompress); + using var ms = new MemoryStream(); + z.CopyTo(ms); + return ms.ToArray(); + } + + private static byte[] Unfilter(byte[] filtered, int w, int h, int bpp) + { + var stride = w * bpp; + Assert.Equal((stride + 1) * h, filtered.Length); + var output = new byte[stride * h]; + for (var y = 0; y < h; y++) + { + var filter = filtered[y * (stride + 1)]; + for (var i = 0; i < stride; i++) + { + int raw = filtered[y * (stride + 1) + 1 + i]; + int a = i >= bpp ? output[y * stride + i - bpp] : 0; + int b = y > 0 ? output[(y - 1) * stride + i] : 0; + int c = i >= bpp && y > 0 ? output[(y - 1) * stride + i - bpp] : 0; + output[y * stride + i] = (byte)(filter switch + { + 0 => raw, + 1 => raw + a, + 2 => raw + b, + 3 => raw + ((a + b) >> 1), + _ => raw + Paeth(a, b, c), + }); + } + } + return output; + } + + private static int Paeth(int a, int b, int c) + { + var p = a + b - c; + int pa = Math.Abs(p - a), pb = Math.Abs(p - b), pc = Math.Abs(p - c); + return pa <= pb && pa <= pc ? a : pb <= pc ? b : c; + } + + private static byte[] Pattern(int variant, int bpp) + { + var pixels = new byte[W * H * bpp]; + for (var y = 0; y < H; y++) + { + for (var x = 0; x < W; x++) + { + var o = (y * W + x) * bpp; + pixels[o] = (byte)(x * 6 + variant * 40); + pixels[o + 1] = (byte)(y * 8); + pixels[o + 2] = (byte)((x ^ y) * 4 + variant); + if (bpp == 4) pixels[o + 3] = 255; + } + } + return pixels; + } +} From 45518e8e4e068a72788d91925b0be754006e701a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 10:05:36 +0000 Subject: [PATCH 8/9] feat(windows): add Media Foundation encoder provider/session and DXGI adapter enumeration Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core.Windows/Dxgi/GpuAdapters.cs | 131 ++++ src/Titano.Core.Windows/Interop/ComObjects.cs | 60 ++ .../MediaFoundationEncoderProvider.cs | 124 ++++ .../MediaFoundationEncoderSession.cs | 640 ++++++++++++++++++ .../MediaFoundationEncoders.cs | 95 +++ .../MediaFoundation/MediaFoundationRuntime.cs | 44 ++ .../MediaFoundation/MfInterop.cs | 314 +++++++++ .../MediaFoundation/GpuAdaptersTests.cs | 18 + .../MediaFoundationEncoderTests.cs | 153 +++++ 9 files changed, 1579 insertions(+) create mode 100644 src/Titano.Core.Windows/Dxgi/GpuAdapters.cs create mode 100644 src/Titano.Core.Windows/Interop/ComObjects.cs create mode 100644 src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoderProvider.cs create mode 100644 src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoderSession.cs create mode 100644 src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoders.cs create mode 100644 src/Titano.Core.Windows/MediaFoundation/MediaFoundationRuntime.cs create mode 100644 src/Titano.Core.Windows/MediaFoundation/MfInterop.cs create mode 100644 tests/Titano.Core.Windows.Tests/MediaFoundation/GpuAdaptersTests.cs create mode 100644 tests/Titano.Core.Windows.Tests/MediaFoundation/MediaFoundationEncoderTests.cs diff --git a/src/Titano.Core.Windows/Dxgi/GpuAdapters.cs b/src/Titano.Core.Windows/Dxgi/GpuAdapters.cs new file mode 100644 index 00000000..ae6aaa28 --- /dev/null +++ b/src/Titano.Core.Windows/Dxgi/GpuAdapters.cs @@ -0,0 +1,131 @@ +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; +using Titano.Core.Diagnostics; +using Titano.Core.Windows.Interop; + +namespace Titano.Core.Windows.Dxgi; + +/// A graphics adapter as reported by DXGI (shown next to the encoders in Settings › Rendering). +/// PCI vendor id (0x10DE NVIDIA, 0x8086 Intel, 0x1002 AMD…). +/// The Microsoft Basic Render Driver or another software adapter. +public sealed record GpuAdapterInfo(string Description, uint VendorId, uint DeviceId, ulong DedicatedVideoMemory, ulong SharedSystemMemory, bool IsSoftware); + +/// Enumerates graphics adapters with CreateDXGIFactory1 → EnumAdapters1 → GetDesc1. +public static partial class GpuAdapters +{ + private const int NotFound = unchecked((int)0x887A0002); // DXGI_ERROR_NOT_FOUND + private const uint SoftwareFlag = 2; // DXGI_ADAPTER_FLAG_SOFTWARE + + /// All adapters in DXGI order (the primary adapter first); empty when DXGI is unavailable. + public static IReadOnlyList Enumerate() + { + var adapters = new List(); + if (!OperatingSystem.IsWindows()) return adapters; + IDXGIFactory1? factory = null; + try + { + ComObjects.Check(CreateDXGIFactory1(typeof(IDXGIFactory1).GUID, out var pointer), "CreateDXGIFactory1"); + factory = ComObjects.Attach(pointer); + for (uint index = 0; ; index++) + { + var hr = factory.EnumAdapters1(index, out var adapterPointer); + if (hr == NotFound) break; + ComObjects.Check(hr, "IDXGIFactory1.EnumAdapters1"); + var adapter = ComObjects.Attach(adapterPointer); + try + { + ComObjects.Check(adapter.GetDesc1(out var desc), "IDXGIAdapter1.GetDesc1"); + adapters.Add(desc.ToInfo()); + } + finally + { + ComObjects.Release(adapter); + } + } + } + catch (Exception ex) when (ex is COMException or DllNotFoundException or EntryPointNotFoundException) + { + Log.Warning("Dxgi", "Graphics adapters could not be enumerated", ex); + } + finally + { + ComObjects.Release(factory); + } + return adapters; + } + + [LibraryImport("dxgi.dll")] + private static partial int CreateDXGIFactory1(in Guid iid, out nint factory); + + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct AdapterDesc1 + { + public fixed char Description[128]; + public uint VendorId; + public uint DeviceId; + public uint SubSysId; + public uint Revision; + public nuint DedicatedVideoMemory; + public nuint DedicatedSystemMemory; + public nuint SharedSystemMemory; + public uint AdapterLuidLow; + public int AdapterLuidHigh; + public uint Flags; + + public readonly GpuAdapterInfo ToInfo() + { + fixed (char* text = Description) + { + var description = new string(text, 0, new ReadOnlySpan(text, 128).IndexOf('\0') is var n and >= 0 ? n : 128); + return new GpuAdapterInfo(description, VendorId, DeviceId, DedicatedVideoMemory, SharedSystemMemory, (Flags & SoftwareFlag) != 0); + } + } + } +} + +// DXGI interfaces in vtable order (dxgi.h); unused slots keep their real signatures with raw pointers. + +[GeneratedComInterface] +[Guid("aec22fb8-76f3-4639-9be0-28eb43a67a2e")] +internal partial interface IDXGIObject +{ + [PreserveSig] int SetPrivateData(in Guid name, uint size, nint data); + [PreserveSig] int SetPrivateDataInterface(in Guid name, nint unknown); + [PreserveSig] int GetPrivateData(in Guid name, ref uint size, nint data); + [PreserveSig] int GetParent(in Guid iid, out nint parent); +} + +[GeneratedComInterface] +[Guid("7b7166ec-21c7-44ae-b21a-c9ae321ae369")] +internal partial interface IDXGIFactory : IDXGIObject +{ + [PreserveSig] int EnumAdapters(uint index, out nint adapter); + [PreserveSig] int MakeWindowAssociation(nint window, uint flags); + [PreserveSig] int GetWindowAssociation(out nint window); + [PreserveSig] int CreateSwapChain(nint device, nint description, out nint swapChain); + [PreserveSig] int CreateSoftwareAdapter(nint module, out nint adapter); +} + +[GeneratedComInterface] +[Guid("770aae78-f26f-4dba-a829-253c83d1b387")] +internal partial interface IDXGIFactory1 : IDXGIFactory +{ + [PreserveSig] int EnumAdapters1(uint index, out nint adapter); + [PreserveSig] int IsCurrent(); +} + +[GeneratedComInterface] +[Guid("2411e7e1-12ac-4ccf-bd14-9798e8534dc0")] +internal partial interface IDXGIAdapter : IDXGIObject +{ + [PreserveSig] int EnumOutputs(uint index, out nint output); + [PreserveSig] int GetDesc(nint description); + [PreserveSig] int CheckInterfaceSupport(in Guid iid, out long umdVersion); +} + +[GeneratedComInterface] +[Guid("29038f61-3839-4626-91fd-086879011a05")] +internal partial interface IDXGIAdapter1 : IDXGIAdapter +{ + [PreserveSig] int GetDesc1(out GpuAdapters.AdapterDesc1 description); +} diff --git a/src/Titano.Core.Windows/Interop/ComObjects.cs b/src/Titano.Core.Windows/Interop/ComObjects.cs new file mode 100644 index 00000000..db6a7bc6 --- /dev/null +++ b/src/Titano.Core.Windows/Interop/ComObjects.cs @@ -0,0 +1,60 @@ +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; + +namespace Titano.Core.Windows.Interop; + +/// +/// Lifetime helpers for source-generated COM interop. Interface pointers received as out nint are wrapped in +/// unique runtime-callable wrappers so they can be released deterministically (Media Foundation encoders recycle a +/// small pool of samples and stall when references linger until finalization). +/// +internal static class ComObjects +{ + private static readonly StrategyBasedComWrappers Wrappers = new(); + + /// Wraps an owned interface pointer; the caller's reference is released (the wrapper holds its own). + public static T Attach(nint pointer) where T : class + { + if (pointer == 0) throw new ArgumentNullException(nameof(pointer)); + try + { + return (T)Wrappers.GetOrCreateObjectForComInstance(pointer, CreateObjectFlags.UniqueInstance); + } + finally + { + Marshal.Release(pointer); + } + } + + /// Like but returns null for a null pointer. + public static T? AttachOrNull(nint pointer) where T : class => pointer == 0 ? null : Attach(pointer); + + /// An AddRef'ed pointer of a wrapper, for structures; release it with . + public static unsafe nint GetPointer(T wrapper) where T : class => (nint)ComInterfaceMarshaller.ConvertToUnmanaged(wrapper); + + /// Releases the COM object behind a wrapper created by now rather than at finalization. + public static void Release(object? wrapper) + { + if (wrapper is ComObject com) com.FinalRelease(); + } + + /// Throws a naming the operation when is a failure code. + public static void Check(int hr, string operation) + { + if (hr < 0) throw new COMException($"{operation} failed with HRESULT 0x{hr:X8}", hr); + } + + /// Reads and frees a CoTaskMem-allocated UTF-16 string. + public static string? TakeString(nint pointer) + { + if (pointer == 0) return null; + try + { + return Marshal.PtrToStringUni(pointer); + } + finally + { + Marshal.FreeCoTaskMem(pointer); + } + } +} diff --git a/src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoderProvider.cs b/src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoderProvider.cs new file mode 100644 index 00000000..d7bc3752 --- /dev/null +++ b/src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoderProvider.cs @@ -0,0 +1,124 @@ +using System.Runtime.InteropServices; +using Titano.Core.Diagnostics; +using Titano.Core.Video; +using Titano.Core.Windows.Interop; + +namespace Titano.Core.Windows.MediaFoundation; + +/// +/// H.264, HEVC and AV1 encoding through Media Foundation MFTs (BRIEF §11.3): hardware encoders (NVENC, Quick Sync, +/// AMF…) first, the Microsoft software encoder as the H.264 fallback; HEVC and AV1 are offered only with hardware. +/// Codecs without a usable encoder are absent from . +/// +public sealed class MediaFoundationEncoderProvider : IVideoEncoderProvider +{ + private const string LogCategory = MediaFoundationEncoders.LogCategory; + private const int SoftwareH264MaxWidth = 4096, SoftwareH264MaxHeight = 2304; // Microsoft H.264 encoder, level 5.2 + private readonly Lazy> _capabilities = new(BuildCapabilities); + + public string Name => "Media Foundation"; + + public IReadOnlyList Capabilities => _capabilities.Value; + + public bool CanEncode(EncodeSettings settings) + { + if (!SettingsSupported(settings)) return false; + foreach (var c in Capabilities) + { + if (Fits(c.Codec, c.Name, c.Hardware, c.SupportsTenBit, settings) && settings.Width <= c.MaxWidth && settings.Height <= c.MaxHeight) return true; + } + return false; + } + + public IVideoEncoderSession Open(EncodeSettings settings) + { + if (!SettingsSupported(settings)) throw new NotSupportedException($"Media Foundation cannot encode {settings.Codec} {settings.Width}×{settings.Height} {settings.BitDepth}-bit"); + var runtime = MediaFoundationRuntime.Acquire(); + var candidates = new List<(MediaFoundationEncoderInfo Info, IMFActivate Activate)>(); + try + { + foreach (var (info, activate) in MediaFoundationEncoders.Find(settings.Codec)) + { + if (Fits(info.Codec, info.Name, info.IsHardware, info.SupportsTenBit, settings)) candidates.Add((info, activate)); + else ComObjects.Release(activate); + } + // Named encoder first, then the preferred kind; MFTEnumEx merit order otherwise. + var ordered = candidates + .OrderByDescending(c => settings.EncoderName is not null && c.Info.Name == settings.EncoderName) + .ThenByDescending(c => c.Info.IsHardware == settings.PreferHardware) + .ToList(); + Exception? failure = null; + for (var i = 0; i < ordered.Count; i++) + { + var (info, activate) = ordered[i]; + try + { + var session = MediaFoundationEncoderSession.Open(settings, info, activate, runtime); + for (var j = i + 1; j < ordered.Count; j++) ComObjects.Release(ordered[j].Activate); + candidates.Clear(); + return session; + } + catch (Exception ex) when (ex is COMException or NotSupportedException or InvalidCastException) + { + Log.Warning(LogCategory, $"\"{info.Name}\" could not be configured for {settings.Codec} {settings.Width}×{settings.Height}: {ex.Message}"); + candidates.Remove(ordered[i]); + activate.ShutdownObject(); + ComObjects.Release(activate); + failure = ex; + } + } + throw new NotSupportedException($"No Media Foundation encoder accepted {settings.Codec} {settings.Width}×{settings.Height} {settings.BitDepth}-bit", failure); + } + catch + { + foreach (var (_, activate) in candidates) ComObjects.Release(activate); + runtime.Dispose(); + throw; + } + } + + private static bool SettingsSupported(EncodeSettings s) => + s.Codec is VideoCodec.H264 or VideoCodec.Hevc or VideoCodec.Av1 + && s.Width > 0 && s.Height > 0 && s.Width % 2 == 0 && s.Height % 2 == 0 + && s.FrameRate.Numerator > 0 && s.FrameRate.Denominator > 0 + && (s.BitDepth == 8 || (s.BitDepth == 10 && s.Codec != VideoCodec.H264)); + + /// §11.3: HEVC and AV1 only in hardware; 10-bit needs P010 input; a named encoder must match. + private static bool Fits(VideoCodec codec, string name, bool hardware, bool tenBit, EncodeSettings s) => + codec == s.Codec + && (hardware || codec == VideoCodec.H264) + && (s.BitDepth == 8 || tenBit) + && (s.EncoderName is null || s.EncoderName == name); + + private static IReadOnlyList BuildCapabilities() + { + var capabilities = new List(); + try + { + foreach (var info in MediaFoundationEncoders.Enumerate()) + { + if (!info.IsHardware && info.Codec != VideoCodec.H264) continue; + var (width, height) = info.IsHardware ? (info.Codec == VideoCodec.H264 ? (4096, 4096) : (8192, 8192)) : (SoftwareH264MaxWidth, SoftwareH264MaxHeight); + capabilities.Add(new EncoderCapability(info.Codec, info.Name, info.IsHardware, width, height, + info.SupportsTenBit && info.Codec != VideoCodec.H264, VendorName(info.VendorId))); + Log.Info(LogCategory, $"{info.Codec} encoder \"{info.Name}\" ({(info.IsHardware ? "hardware" : "software")}{(info.SupportsTenBit ? ", 10-bit" : string.Empty)})"); + } + } + catch (Exception ex) when (ex is COMException or DllNotFoundException or EntryPointNotFoundException or PlatformNotSupportedException) + { + Log.Warning(LogCategory, "Media Foundation encoders could not be enumerated", ex); + } + return capabilities; + } + + /// Readable vendor for PCI vendor ids reported as "VEN_xxxx". + internal static string? VendorName(string? vendorId) => vendorId?.ToUpperInvariant() switch + { + null => null, + "VEN_10DE" => "NVIDIA", + "VEN_8086" => "Intel", + "VEN_1002" or "VEN_1022" => "AMD", + "VEN_5143" or "VEN_4D4F4351" => "Qualcomm", + var other => other, + }; +} diff --git a/src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoderSession.cs b/src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoderSession.cs new file mode 100644 index 00000000..f82354aa --- /dev/null +++ b/src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoderSession.cs @@ -0,0 +1,640 @@ +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; +using Titano.Core.Diagnostics; +using Titano.Core.Imaging; +using Titano.Core.IO; +using Titano.Core.Video; +using Titano.Core.Video.Bitstream; +using Titano.Core.Video.Mp4; +using Titano.Core.Video.Yuv; +using Titano.Core.Windows.Interop; + +namespace Titano.Core.Windows.MediaFoundation; + +/// +/// One encode through a Media Foundation encoder MFT driven directly (no SinkWriter): output type first, then NV12/P010 +/// input, ICodecAPI rate control/GOP/B-frames, synchronous ProcessInput/ProcessOutput or the asynchronous event loop of +/// hardware MFTs, and Annex-B/OBU output muxed by into a temporary file that +/// commits. Disposing without completing deletes the partial file. +/// +public sealed class MediaFoundationEncoderSession : IVideoEncoderSession +{ + private const string LogCategory = MediaFoundationEncoders.LogCategory; + + private readonly EncodeSettings _settings; + private readonly IDisposable _runtime; + private readonly IMFActivate _activate; + private readonly IMFTransform _transform; + private readonly IMFMediaEventGenerator? _events; + private readonly bool _tenBit; + private readonly string _tempPath; + private readonly FileStream _file; + private readonly Mp4Muxer _muxer; + private readonly Dictionary _ticksByHns = []; + private readonly Dictionary _durationByTicks = []; + private readonly Queue _inputStarts = new(); + private bool _providesSamples; + private uint _outputBufferSize; + private int _needInput; + private bool _drained; + private long _time; + private Encoded? _pending; + private int _framesIn; + private int _framesOut; + private bool _completed; + private bool _released; + private bool _disposed; + + private MediaFoundationEncoderSession(EncodeSettings settings, MediaFoundationEncoderInfo info, IMFActivate activate, IMFTransform transform, IDisposable runtime) + { + _settings = settings; + _activate = activate; + _transform = transform; + _runtime = runtime; + _tenBit = settings.BitDepth == 10; + EncoderName = info.Name; + IsHardware = info.IsHardware; + + if (UnlockAsync(transform)) + _events = transform as IMFMediaEventGenerator ?? throw new NotSupportedException("Asynchronous MFT without an event generator"); + if (transform as ICodecAPI is { } codecApi) ApplyCodecSettings(codecApi); + SetOutputType(); + SetInputType(); + ReadOutputStreamInfo(); + Message(MfNative.MessageNotifyBeginStreaming); + Message(MfNative.MessageNotifyStartOfStream); + + _tempPath = AtomicFile.TempPathFor(settings.OutputPath); + _file = new FileStream(_tempPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 1 << 20); + try + { + _muxer = new Mp4Muxer(_file, new VideoTrackConfig { Codec = settings.Codec, Width = settings.Width, Height = settings.Height, FastStart = true }); + ReadSequenceHeader(); + } + catch + { + _file.Dispose(); + File.Delete(_tempPath); + throw; + } + Log.Info(LogCategory, $"Encoding {settings.Codec} {settings.Width}×{settings.Height} {settings.BitDepth}-bit with \"{info.Name}\" ({(info.IsHardware ? "hardware" : "software")}, {(_events is null ? "sync" : "async")})"); + } + + public string EncoderName { get; } + + public bool IsHardware { get; } + + /// Activates the MFT behind and configures it; the session then owns the activate and the runtime lease. + internal static MediaFoundationEncoderSession Open(EncodeSettings settings, MediaFoundationEncoderInfo info, IMFActivate activate, IDisposable runtime) + { + ComObjects.Check(activate.ActivateObject(MfGuids.IidTransform, out var pointer), "IMFActivate.ActivateObject"); + var transform = ComObjects.Attach(pointer); + try + { + return new MediaFoundationEncoderSession(settings, info, activate, transform, runtime); + } + catch + { + ComObjects.Release(transform); + throw; + } + } + + public void WriteFrame(ImageBuffer frame, long duration90k) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_completed) throw new InvalidOperationException("The session is complete"); + if (frame.Width != _settings.Width || frame.Height != _settings.Height) throw new ArgumentException("Frame size differs from the encode size", nameof(frame)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(duration90k); + + var start = _time; + _time += duration90k; + var hns = ToHns(start); + _ticksByHns[hns] = start; + _durationByTicks[start] = duration90k; + _inputStarts.Enqueue(start); + var sample = CreateInputSample(frame, hns, ToHns(_time) - hns); + try + { + if (_events is not null) + { + Pump(() => _needInput > 0); + _needInput--; + ComObjects.Check(_transform.ProcessInput(0, sample, 0), "ProcessInput"); + } + else + { + SubmitSync(sample); + } + } + finally + { + ComObjects.Release(sample); + } + _framesIn++; + } + + public void Complete() + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_completed) throw new InvalidOperationException("The session is already complete"); + if (_framesIn == 0) throw new InvalidOperationException("No frames were written"); + Message(MfNative.MessageNotifyEndOfStream); + Message(MfNative.MessageCommandDrain); + if (_events is not null) Pump(() => _drained); + else while (ProcessOutput()) { } + if (_pending is { } last) + { + // The last decoded picture keeps the duration of its own presentation slot. + WritePending(last, _durationByTicks.GetValueOrDefault(last.Pts, 1)); + _pending = null; + } + Message(MfNative.MessageNotifyEndStreaming); + ReleaseEncoder(); + + _muxer.Complete(); + _file.Flush(flushToDisk: true); + _file.Dispose(); + AtomicFile.Commit(_tempPath, _settings.OutputPath); + _completed = true; + Log.Info(LogCategory, $"\"{EncoderName}\" encoded {_framesIn} frames into {_framesOut} samples: {_settings.OutputPath}"); + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + if (!_completed) + { + _file.Dispose(); + try + { + File.Delete(_tempPath); + } + catch (IOException ex) + { + Log.Warning(LogCategory, "Could not delete the partial output " + _tempPath, ex); + } + } + ReleaseEncoder(); + _runtime.Dispose(); + } + + private sealed record Encoded(byte[] Data, long Pts, long Dts, bool Key); + + private static bool UnlockAsync(IMFTransform transform) + { + if (transform.GetAttributes(out var pointer) < 0 || pointer == 0) return false; + var attributes = ComObjects.Attach(pointer); + try + { + if (attributes.GetUINT32(MfGuids.TransformAsync, out var isAsync) < 0 || isAsync == 0) return false; + ComObjects.Check(attributes.SetUINT32(MfGuids.TransformAsyncUnlock, 1), "MF_TRANSFORM_ASYNC_UNLOCK"); + return true; + } + finally + { + ComObjects.Release(attributes); + } + } + + private void ApplyCodecSettings(ICodecAPI api) + { + var mode = _settings.RateControl switch + { + RateControl.Cbr => MfNative.RateControlCbr, + RateControl.Quality => MfNative.RateControlQuality, + _ => MfNative.RateControlUnconstrainedVbr, + }; + SetCodecValue(api, MfGuids.CodecRateControlMode, ComVariant.Create(mode), "RateControlMode"); + if (_settings.RateControl == RateControl.Quality) + SetCodecValue(api, MfGuids.CodecQuality, ComVariant.Create((uint)Math.Clamp(_settings.Quality, 1, 100)), "Quality"); + else + SetCodecValue(api, MfGuids.CodecMeanBitRate, ComVariant.Create(BitRate), "MeanBitRate"); + var gop = _settings.GopLength > 0 ? _settings.GopLength : (int)Math.Max(1, Math.Round(_settings.FrameRate.Value)); + SetCodecValue(api, MfGuids.CodecGopSize, ComVariant.Create((uint)gop), "GOPSize"); + var bFrames = _settings.Profile == CodecProfile.H264Baseline ? 0 : Math.Max(0, _settings.BFrames); + SetCodecValue(api, MfGuids.CodecBPictureCount, ComVariant.Create((uint)bFrames), "BPictureCount"); + SetCodecValue(api, MfGuids.CodecLowLatencyMode, ComVariant.Create(false), "LowLatencyMode"); + } + + private static unsafe void SetCodecValue(ICodecAPI api, in Guid property, ComVariant value, string name) + { + if (api.IsSupported(property) != 0) + { + Log.Debug(LogCategory, $"ICodecAPI {name} not supported by the encoder"); + return; + } + var hr = api.SetValue(property, (nint)(&value)); + if (hr < 0) Log.Debug(LogCategory, $"ICodecAPI {name} rejected (0x{hr:X8})"); + } + + /// Requested bit rate, or the preset density (§11.4: 0.10/0.07/0.05 bits per pixel per frame) when unset. + private uint BitRate + { + get + { + if (_settings.BitRate > 0) return (uint)Math.Min(uint.MaxValue, _settings.BitRate); + var bpp = _settings.Codec switch { VideoCodec.H264 => 0.10, VideoCodec.Hevc => 0.07, _ => 0.05 }; + return (uint)Math.Min(uint.MaxValue, _settings.Width * (double)_settings.Height * _settings.FrameRate.Value * bpp); + } + } + + private void SetOutputType() + { + var level = CodecLimits.Check(_settings.Codec, _settings.Width, _settings.Height, _settings.FrameRate.Value, BitRate, _settings.Profile).Level; + var hr = TrySetOutputType(level); + if (hr < 0 && level is not null) + { + Log.Debug(LogCategory, $"Output type with level {level} rejected (0x{hr:X8}); letting the encoder choose"); + hr = TrySetOutputType(null); + } + ComObjects.Check(hr, "SetOutputType"); + } + + private int TrySetOutputType(CodecLevel? level) + { + var type = CreateMediaType(); + try + { + ComObjects.Check(type.SetGUID(MfGuids.Subtype, MediaFoundationEncoders.Subtype(_settings.Codec)), "MF_MT_SUBTYPE"); + ComObjects.Check(type.SetUINT32(MfGuids.AverageBitrate, BitRate), "MF_MT_AVG_BITRATE"); + SetVideoFormat(type); + var profile = Profile(); + if (profile > 0) ComObjects.Check(type.SetUINT32(MfGuids.Mpeg2Profile, profile), "MF_MT_MPEG2_PROFILE"); + if (level is { } l && _settings.Codec != VideoCodec.Av1) + { + var idc = _settings.Codec == VideoCodec.H264 ? l.H264LevelIdc : l.HevcLevelIdc; + ComObjects.Check(type.SetUINT32(MfGuids.Mpeg2Level, (uint)idc), "MF_MT_MPEG2_LEVEL"); + } + return _transform.SetOutputType(0, type, 0); + } + finally + { + ComObjects.Release(type); + } + } + + /// eAVEncH264VProfile / eAVEncH265VProfile value; AV1 encoders derive the profile from the input format. + private uint Profile() => _settings.Codec switch + { + VideoCodec.H264 => _settings.Profile switch + { + CodecProfile.H264Baseline => 66u, + CodecProfile.H264Main => 77u, + _ => 100u, + }, + VideoCodec.Hevc => _tenBit ? 2u : 1u, + _ => 0u, + }; + + private void SetInputType() + { + var subtype = _tenBit ? MfGuids.FormatP010 : MfGuids.FormatNv12; + var type = CreateMediaType(); + int hr; + try + { + ComObjects.Check(type.SetGUID(MfGuids.Subtype, subtype), "MF_MT_SUBTYPE"); + ComObjects.Check(type.SetUINT32(MfGuids.DefaultStride, (uint)(_settings.Width * (_tenBit ? 2 : 1))), "MF_MT_DEFAULT_STRIDE"); + SetVideoFormat(type); + hr = _transform.SetInputType(0, type, 0); + } + finally + { + ComObjects.Release(type); + } + + // Some encoders only accept their own enumerated input types: complete one with our format. + for (uint index = 0; hr < 0; index++) + { + var available = _transform.GetInputAvailableType(0, index, out var pointer); + if (available < 0) break; + var candidate = ComObjects.Attach(pointer); + try + { + if (candidate.GetGUID(MfGuids.Subtype, out var s) < 0 || s != subtype) continue; + SetVideoFormat(candidate); + hr = _transform.SetInputType(0, candidate, 0); + } + finally + { + ComObjects.Release(candidate); + } + } + ComObjects.Check(hr, $"SetInputType ({(_tenBit ? "P010" : "NV12")})"); + } + + private static IMFMediaType CreateMediaType() + { + ComObjects.Check(MfNative.MFCreateMediaType(out var pointer), "MFCreateMediaType"); + var type = ComObjects.Attach(pointer); + ComObjects.Check(type.SetGUID(MfGuids.MajorType, MfGuids.MediaTypeVideo), "MF_MT_MAJOR_TYPE"); + return type; + } + + /// Frame size and rate, square pixels, progressive, BT.709 limited range. + private void SetVideoFormat(IMFMediaType type) + { + ComObjects.Check(type.SetUINT64(MfGuids.FrameSize, Pack(_settings.Width, _settings.Height)), "MF_MT_FRAME_SIZE"); + ComObjects.Check(type.SetUINT64(MfGuids.FrameRate, Pack(_settings.FrameRate.Numerator, _settings.FrameRate.Denominator)), "MF_MT_FRAME_RATE"); + ComObjects.Check(type.SetUINT64(MfGuids.PixelAspectRatio, Pack(1, 1)), "MF_MT_PIXEL_ASPECT_RATIO"); + ComObjects.Check(type.SetUINT32(MfGuids.InterlaceMode, MfNative.InterlaceProgressive), "MF_MT_INTERLACE_MODE"); + type.SetUINT32(MfGuids.NominalRange, MfNative.NominalRange16To235); + type.SetUINT32(MfGuids.VideoPrimaries, MfNative.PrimariesBt709); + type.SetUINT32(MfGuids.TransferFunction, MfNative.TransferBt709); + type.SetUINT32(MfGuids.YuvMatrix, MfNative.MatrixBt709); + } + + private static ulong Pack(int high, int low) => (ulong)(uint)high << 32 | (uint)low; + + private void ReadOutputStreamInfo() + { + ComObjects.Check(_transform.GetOutputStreamInfo(0, out var info), "GetOutputStreamInfo"); + _providesSamples = (info.Flags & (MfNative.OutputStreamProvidesSamples | MfNative.OutputStreamCanProvideSamples)) != 0; + _outputBufferSize = Math.Max(info.Size, (uint)(_settings.Width * _settings.Height * 3 / 2)); + } + + /// Hands MF_MT_MPEG_SEQUENCE_HEADER (parameter sets / sequence header OBU) of the current output type to the muxer. + private void ReadSequenceHeader() + { + if (_transform.GetOutputCurrentType(0, out var pointer) < 0) return; + var type = ComObjects.Attach(pointer); + try + { + if (type.GetAllocatedBlob(MfGuids.MpegSequenceHeader, out var blob, out var size) < 0) return; + try + { + var bytes = new byte[size]; + Marshal.Copy(blob, bytes, 0, (int)size); + _muxer.AddParameterSets(bytes); + } + finally + { + Marshal.FreeCoTaskMem(blob); + } + } + finally + { + ComObjects.Release(type); + } + } + + private unsafe IMFSample CreateInputSample(ImageBuffer frame, long time, long duration) + { + var fourCc = _tenBit ? 0x30313050u : 0x3231564Eu; // 'P010' / 'NV12' + ComObjects.Check(MfNative.MFCreate2DMediaBuffer((uint)_settings.Width, (uint)_settings.Height, fourCc, false, out var bufferPointer), "MFCreate2DMediaBuffer"); + var buffer = ComObjects.Attach(bufferPointer); + try + { + var buffer2D = buffer as IMF2DBuffer ?? throw new NotSupportedException("Media buffer without IMF2DBuffer"); + ComObjects.Check(buffer2D.Lock2D(out var scanline0, out var pitch), "IMF2DBuffer.Lock2D"); + try + { + if (pitch <= 0) throw new NotSupportedException("Bottom-up input buffers are not supported"); + var rows = _settings.Height * 3 / 2; + if (_tenBit) Nv12Converter.ToP010(frame, new Span((void*)scanline0, pitch / 2 * rows), ChromaSiting.Left, pitch / 2); + else Nv12Converter.ToNv12(frame, new Span((void*)scanline0, pitch * rows), ChromaSiting.Left, pitch); + } + finally + { + buffer2D.Unlock2D(); + } + // 2D buffers report their contiguous size; setting it explicitly helps encoders that read the plain length. + if (buffer2D.GetContiguousLength(out var length) >= 0 && buffer.SetCurrentLength(length) < 0) + Log.Debug(LogCategory, "SetCurrentLength on the 2D input buffer was rejected"); + + ComObjects.Check(MfNative.MFCreateSample(out var samplePointer), "MFCreateSample"); + var sample = ComObjects.Attach(samplePointer); + try + { + ComObjects.Check(sample.AddBuffer(buffer), "AddBuffer"); + ComObjects.Check(sample.SetSampleTime(time), "SetSampleTime"); + ComObjects.Check(sample.SetSampleDuration(duration), "SetSampleDuration"); + return sample; + } + catch + { + ComObjects.Release(sample); + throw; + } + } + finally + { + ComObjects.Release(buffer); + } + } + + private void SubmitSync(IMFSample sample) + { + while (true) + { + var hr = _transform.ProcessInput(0, sample, 0); + if (hr != MfNative.NotAccepting) + { + ComObjects.Check(hr, "ProcessInput"); + break; + } + // Full: collect output first; an encoder that neither accepts input nor yields output is stuck. + if (!ProcessOutput()) throw new COMException("Encoder refuses input without producing output", hr); + while (ProcessOutput()) { } + } + while (ProcessOutput()) { } + } + + /// Runs the asynchronous MFT event loop until holds. + private void Pump(Func done) + { + while (!done()) + { + ComObjects.Check(_events!.GetEvent(0, out var pointer), "IMFMediaEventGenerator.GetEvent"); + var mediaEvent = ComObjects.Attach(pointer); + uint type; + int status; + try + { + ComObjects.Check(mediaEvent.GetEventType(out type), "IMFMediaEvent.GetType"); + ComObjects.Check(mediaEvent.GetStatus(out status), "IMFMediaEvent.GetStatus"); + } + finally + { + ComObjects.Release(mediaEvent); + } + if (status < 0 || type == MfNative.EventError) throw new COMException($"Encoder reported event {type} with failure", status); + switch (type) + { + case MfNative.EventNeedInput: + _needInput++; + break; + case MfNative.EventHaveOutput: + ProcessOutput(); + break; + case MfNative.EventDrainComplete: + _drained = true; + break; + } + } + } + + /// Pulls one output sample; false when the encoder needs more input. + private bool ProcessOutput() + { + IMFSample? own = null; + nint ownPointer = 0; + var data = new MftOutputDataBuffer(); + if (!_providesSamples) + { + own = CreateOutputSample(); + data.Sample = ownPointer = ComObjects.GetPointer(own); + } + int hr; + try + { + hr = _transform.ProcessOutput(0, 1, ref data, out _); + } + finally + { + if (data.Events != 0) Marshal.Release(data.Events); + if (ownPointer != 0) Marshal.Release(ownPointer); + } + + if (hr < 0) + { + ComObjects.Release(own); + if (hr == MfNative.TransformNeedMoreInput) return false; + if (hr == MfNative.TransformStreamChange) + { + Renegotiate(); + return true; + } + ComObjects.Check(hr, "ProcessOutput"); + } + + var output = own ?? ComObjects.AttachOrNull(data.Sample); + if (output is null) return true; + try + { + Deliver(output); + } + finally + { + ComObjects.Release(output); + } + return true; + } + + private IMFSample CreateOutputSample() + { + ComObjects.Check(MfNative.MFCreateMemoryBuffer(_outputBufferSize, out var bufferPointer), "MFCreateMemoryBuffer"); + var buffer = ComObjects.Attach(bufferPointer); + try + { + ComObjects.Check(MfNative.MFCreateSample(out var samplePointer), "MFCreateSample"); + var sample = ComObjects.Attach(samplePointer); + ComObjects.Check(sample.AddBuffer(buffer), "AddBuffer"); + return sample; + } + finally + { + ComObjects.Release(buffer); + } + } + + /// MF_E_TRANSFORM_STREAM_CHANGE: adopt the encoder's new output type and its sequence header. + private void Renegotiate() + { + ComObjects.Check(_transform.GetOutputAvailableType(0, 0, out var pointer), "GetOutputAvailableType"); + var type = ComObjects.Attach(pointer); + try + { + ComObjects.Check(_transform.SetOutputType(0, type, 0), "SetOutputType (stream change)"); + } + finally + { + ComObjects.Release(type); + } + ReadOutputStreamInfo(); + ReadSequenceHeader(); + Log.Debug(LogCategory, $"\"{EncoderName}\" renegotiated its output type"); + } + + private void Deliver(IMFSample sample) + { + ComObjects.Check(sample.GetSampleTime(out var pts), "GetSampleTime"); + bool? cleanPoint = sample.GetUINT32(MfGuids.SampleCleanPoint, out var clean) >= 0 ? clean != 0 : null; + long? dts = sample.GetUINT64(MfGuids.SampleDecodeTimestamp, out var decode) >= 0 ? (long)decode : null; + ComObjects.Check(sample.ConvertToContiguousBuffer(out var bufferPointer), "ConvertToContiguousBuffer"); + var buffer = ComObjects.Attach(bufferPointer); + byte[] bytes; + try + { + ComObjects.Check(buffer.Lock(out var data, out _, out var length), "IMFMediaBuffer.Lock"); + try + { + bytes = new byte[length]; + Marshal.Copy(data, bytes, 0, (int)length); + } + finally + { + buffer.Unlock(); + } + } + finally + { + ComObjects.Release(buffer); + } + if (bytes.Length == 0) return; + var key = cleanPoint ?? IsRandomAccess(bytes); + + var ptsTicks = _ticksByHns.Remove(pts, out var exact) ? exact : To90k(pts); + // Without MFSampleExtension_DecodeTimestamp the n-th output decodes at the n-th input's time. + _inputStarts.TryDequeue(out var inputStart); + var dtsTicks = dts is { } d ? To90k(d) : inputStart; + var encoded = new Encoded(bytes, ptsTicks, dtsTicks, key); + if (_pending is { } previous) WritePending(previous, Math.Max(1, encoded.Dts - previous.Dts)); + _pending = encoded; + _framesOut++; + } + + /// Sync detection for encoders that do not set MFSampleExtension_CleanPoint: IDR/IRAP NAL units or an AV1 sequence header. + private bool IsRandomAccess(byte[] data) + { + if (_settings.Codec == VideoCodec.Av1) + { + foreach (var obu in Av1ObuParser.Split(data)) + { + if (obu.Type == Av1ObuType.SequenceHeader) return true; + } + return false; + } + foreach (var unit in AnnexBParser.SplitAccessUnit(data, out _)) + { + var header = data[unit.Offset]; + if (_settings.Codec == VideoCodec.H264 ? AnnexBParser.H264Type(header) == H264NalUnitType.IdrSlice + : AnnexBParser.HevcType(header) is >= HevcNalUnitType.BlaWLp and <= HevcNalUnitType.Cra) + return true; + } + return false; + } + + private void WritePending(Encoded sample, long duration) + { + _muxer.WriteSample(sample.Data, duration, sample.Key, sample.Pts - sample.Dts); + _durationByTicks.Remove(sample.Pts); + } + + private void Message(uint message) => ComObjects.Check(_transform.ProcessMessage(message, 0), $"ProcessMessage(0x{message:X})"); + + private void ReleaseEncoder() + { + if (_released) return; + _released = true; + ComObjects.Release(_transform); + _activate.ShutdownObject(); + ComObjects.Release(_activate); + } + + private static long ToHns(long ticks90k) => (ticks90k * 1000 + 4) / 9; + + private static long To90k(long hns) => (long)Math.Round(hns * 9 / 1000.0); +} diff --git a/src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoders.cs b/src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoders.cs new file mode 100644 index 00000000..b8a05dc8 --- /dev/null +++ b/src/Titano.Core.Windows/MediaFoundation/MediaFoundationEncoders.cs @@ -0,0 +1,95 @@ +using System.Runtime.InteropServices; +using Titano.Core.Diagnostics; +using Titano.Core.Video; +using Titano.Core.Windows.Interop; + +namespace Titano.Core.Windows.MediaFoundation; + +/// A Media Foundation video encoder MFT available on this machine. +/// MFT_FRIENDLY_NAME_Attribute, e.g. "NVIDIA HEVC Encoder MFT". +/// MFT_ENUM_HARDWARE_VENDOR_ID_Attribute of hardware MFTs, e.g. "VEN_10DE". +/// The MFT accepts P010 input (HEVC Main10 / AV1 10-bit). +public sealed record MediaFoundationEncoderInfo(VideoCodec Codec, string Name, bool IsHardware, bool IsAsync, string? VendorId, bool SupportsTenBit); + +/// Discovers H.264, HEVC and AV1 encoder MFTs with MFTEnumEx in the system's merit order (hardware first). +public static class MediaFoundationEncoders +{ + internal const string LogCategory = "MediaFoundation"; + private static readonly VideoCodec[] Codecs = [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Av1]; + + /// Every encoder MFT for H.264, HEVC and AV1; empty when Media Foundation is unavailable. + public static IReadOnlyList Enumerate() + { + if (!MediaFoundationRuntime.IsAvailable) return []; + using var lease = MediaFoundationRuntime.Acquire(); + var result = new List(); + foreach (var codec in Codecs) + { + foreach (var (info, activate) in Find(codec)) + { + result.Add(info); + ComObjects.Release(activate); + } + } + return result; + } + + /// Encoder activation objects for ; Media Foundation must be started and the caller releases them. + internal static List<(MediaFoundationEncoderInfo Info, IMFActivate Activate)> Find(VideoCodec codec) + { + var tenBit = new HashSet(StringComparer.Ordinal); + if (codec != VideoCodec.H264) + { + foreach (var activate in Activates(codec, MfGuids.FormatP010)) + { + if (FriendlyName(activate) is { } name) tenBit.Add(name); + ComObjects.Release(activate); + } + } + var found = new List<(MediaFoundationEncoderInfo, IMFActivate)>(); + foreach (var activate in Activates(codec, null)) + { + var name = FriendlyName(activate) ?? "Unnamed encoder"; + var flags = activate.GetUINT32(MfGuids.TransformFlags, out var f) >= 0 ? f : 0; + string? vendor = null; + if (activate.GetAllocatedString(MfGuids.HardwareVendorId, out var vendorPointer, out _) >= 0) vendor = ComObjects.TakeString(vendorPointer); + var info = new MediaFoundationEncoderInfo(codec, name, (flags & MfNative.EnumHardware) != 0, (flags & MfNative.EnumAsync) != 0, vendor, tenBit.Contains(name)); + found.Add((info, activate)); + } + return found; + } + + internal static Guid Subtype(VideoCodec codec) => codec switch + { + VideoCodec.H264 => MfGuids.FormatH264, + VideoCodec.Hevc => MfGuids.FormatHevc, + VideoCodec.Av1 => MfGuids.FormatAv1, + _ => throw new ArgumentException($"{codec} has no Media Foundation encoder", nameof(codec)), + }; + + private static string? FriendlyName(IMFActivate activate) => + activate.GetAllocatedString(MfGuids.FriendlyName, out var pointer, out _) >= 0 ? ComObjects.TakeString(pointer) : null; + + private static unsafe List Activates(VideoCodec codec, Guid? inputSubtype) + { + var output = new MftRegisterTypeInfo { MajorType = MfGuids.MediaTypeVideo, Subtype = Subtype(codec) }; + var input = new MftRegisterTypeInfo { MajorType = MfGuids.MediaTypeVideo, Subtype = inputSubtype ?? Guid.Empty }; + const uint flags = MfNative.EnumHardware | MfNative.EnumAsync | MfNative.EnumSync | MfNative.EnumSortAndFilter; + var hr = MfNative.MFTEnumEx(MfGuids.VideoEncoderCategory, flags, inputSubtype.HasValue ? &input : null, &output, out var array, out var count); + var list = new List((int)count); + if (hr < 0) + { + Log.Warning(LogCategory, $"MFTEnumEx for {codec} failed with HRESULT 0x{hr:X8}"); + return list; + } + try + { + for (var i = 0; i < count; i++) list.Add(ComObjects.Attach(Marshal.ReadIntPtr(array, i * IntPtr.Size))); + } + finally + { + Marshal.FreeCoTaskMem(array); + } + return list; + } +} diff --git a/src/Titano.Core.Windows/MediaFoundation/MediaFoundationRuntime.cs b/src/Titano.Core.Windows/MediaFoundation/MediaFoundationRuntime.cs new file mode 100644 index 00000000..a8cfb3a1 --- /dev/null +++ b/src/Titano.Core.Windows/MediaFoundation/MediaFoundationRuntime.cs @@ -0,0 +1,44 @@ +using System.Runtime.InteropServices; +using Titano.Core.Windows.Interop; + +namespace Titano.Core.Windows.MediaFoundation; + +/// Reference-counted MFStartup/MFShutdown shared by every Media Foundation user in the process. +internal static class MediaFoundationRuntime +{ + private static readonly Lock Gate = new(); + private static int _references; + + /// True when mfplat.dll can be loaded (absent on Windows "N" editions without the Media Feature Pack). + public static bool IsAvailable { get; } = OperatingSystem.IsWindows() && NativeLibrary.TryLoad("mfplat.dll", out _); + + /// Starts Media Foundation for the lifetime of the returned lease. + public static IDisposable Acquire() + { + if (!IsAvailable) throw new PlatformNotSupportedException("Media Foundation is not available on this system"); + lock (Gate) + { + if (_references == 0) ComObjects.Check(MfNative.MFStartup(MfNative.Version, MfNative.StartupLite), "MFStartup"); + _references++; + } + return new Lease(); + } + + private static void Release() + { + lock (Gate) + { + if (--_references == 0) MfNative.MFShutdown(); + } + } + + private sealed class Lease : IDisposable + { + private int _released; + + public void Dispose() + { + if (Interlocked.Exchange(ref _released, 1) == 0) Release(); + } + } +} diff --git a/src/Titano.Core.Windows/MediaFoundation/MfInterop.cs b/src/Titano.Core.Windows/MediaFoundation/MfInterop.cs new file mode 100644 index 00000000..a6ff08f1 --- /dev/null +++ b/src/Titano.Core.Windows/MediaFoundation/MfInterop.cs @@ -0,0 +1,314 @@ +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; + +namespace Titano.Core.Windows.MediaFoundation; + +// Media Foundation COM interfaces in vtable order (mfobjects.h, mftransform.h, strmif.h). Every method returns the raw +// HRESULT because MF uses failure codes for flow control (MF_E_TRANSFORM_NEED_MORE_INPUT, MF_E_NOTACCEPTING…). +// Interface pointers come back as nint and are wrapped with ComObjects.Attach for deterministic release; VARIANT and +// PROPVARIANT arguments are raw pointers so the assembly keeps runtime marshalling enabled. + +[GeneratedComInterface] +[Guid("2cd2d921-c447-44a7-a13c-4adabfc247e3")] +internal partial interface IMFAttributes +{ + [PreserveSig] int GetItem(in Guid key, nint value); + [PreserveSig] int GetItemType(in Guid key, out int type); + [PreserveSig] int CompareItem(in Guid key, nint value, out int result); + [PreserveSig] int Compare(nint theirs, int matchType, out int result); + [PreserveSig] int GetUINT32(in Guid key, out uint value); + [PreserveSig] int GetUINT64(in Guid key, out ulong value); + [PreserveSig] int GetDouble(in Guid key, out double value); + [PreserveSig] int GetGUID(in Guid key, out Guid value); + [PreserveSig] int GetStringLength(in Guid key, out uint length); + [PreserveSig] int GetString(in Guid key, nint buffer, uint bufferSize, out uint length); + [PreserveSig] int GetAllocatedString(in Guid key, out nint value, out uint length); + [PreserveSig] int GetBlobSize(in Guid key, out uint size); + [PreserveSig] int GetBlob(in Guid key, nint buffer, uint bufferSize, out uint size); + [PreserveSig] int GetAllocatedBlob(in Guid key, out nint buffer, out uint size); + [PreserveSig] int GetUnknown(in Guid key, in Guid iid, out nint value); + [PreserveSig] int SetItem(in Guid key, nint value); + [PreserveSig] int DeleteItem(in Guid key); + [PreserveSig] int DeleteAllItems(); + [PreserveSig] int SetUINT32(in Guid key, uint value); + [PreserveSig] int SetUINT64(in Guid key, ulong value); + [PreserveSig] int SetDouble(in Guid key, double value); + [PreserveSig] int SetGUID(in Guid key, in Guid value); + [PreserveSig] int SetString(in Guid key, nint value); + [PreserveSig] int SetBlob(in Guid key, nint buffer, uint size); + [PreserveSig] int SetUnknown(in Guid key, nint unknown); + [PreserveSig] int LockStore(); + [PreserveSig] int UnlockStore(); + [PreserveSig] int GetCount(out uint count); + [PreserveSig] int GetItemByIndex(uint index, out Guid key, nint value); + [PreserveSig] int CopyAllItems(nint destination); +} + +[GeneratedComInterface] +[Guid("44ae0fa8-ea31-4109-8d2e-4cae4997c555")] +internal partial interface IMFMediaType : IMFAttributes +{ + [PreserveSig] int GetMajorType(out Guid type); + [PreserveSig] int IsCompressedFormat(out int compressed); + [PreserveSig] int IsEqual(nint other, out uint flags); + [PreserveSig] int GetRepresentation(Guid representation, out nint value); + [PreserveSig] int FreeRepresentation(Guid representation, nint value); +} + +[GeneratedComInterface] +[Guid("045fa593-8799-42b8-bc8d-8968c6453507")] +internal partial interface IMFMediaBuffer +{ + [PreserveSig] int Lock(out nint buffer, out uint maxLength, out uint currentLength); + [PreserveSig] int Unlock(); + [PreserveSig] int GetCurrentLength(out uint length); + [PreserveSig] int SetCurrentLength(uint length); + [PreserveSig] int GetMaxLength(out uint length); +} + +[GeneratedComInterface] +[Guid("7dc9d5f9-9ed9-44ec-9bbf-0600bb589fbb")] +internal partial interface IMF2DBuffer +{ + [PreserveSig] int Lock2D(out nint scanline0, out int pitch); + [PreserveSig] int Unlock2D(); + [PreserveSig] int GetScanline0AndPitch(out nint scanline0, out int pitch); + [PreserveSig] int IsContiguousFormat(out int contiguous); + [PreserveSig] int GetContiguousLength(out uint length); + [PreserveSig] int ContiguousCopyTo(nint destination, uint size); + [PreserveSig] int ContiguousCopyFrom(nint source, uint size); +} + +[GeneratedComInterface] +[Guid("c40a00f2-b93a-4d80-ae8c-5a1c634f58e4")] +internal partial interface IMFSample : IMFAttributes +{ + [PreserveSig] int GetSampleFlags(out uint flags); + [PreserveSig] int SetSampleFlags(uint flags); + [PreserveSig] int GetSampleTime(out long time); + [PreserveSig] int SetSampleTime(long time); + [PreserveSig] int GetSampleDuration(out long duration); + [PreserveSig] int SetSampleDuration(long duration); + [PreserveSig] int GetBufferCount(out uint count); + [PreserveSig] int GetBufferByIndex(uint index, out nint buffer); + [PreserveSig] int ConvertToContiguousBuffer(out nint buffer); + [PreserveSig] int AddBuffer(IMFMediaBuffer buffer); + [PreserveSig] int RemoveBufferByIndex(uint index); + [PreserveSig] int RemoveAllBuffers(); + [PreserveSig] int GetTotalLength(out uint length); + [PreserveSig] int CopyToBuffer(IMFMediaBuffer buffer); +} + +[GeneratedComInterface] +[Guid("7fee9e9a-4a89-47a6-899c-b6a53a70fb67")] +internal partial interface IMFActivate : IMFAttributes +{ + [PreserveSig] int ActivateObject(in Guid iid, out nint instance); + [PreserveSig] int ShutdownObject(); + [PreserveSig] int DetachObject(); +} + +[GeneratedComInterface] +[Guid("df598932-f10c-4e39-bba2-c308f101daa3")] +internal partial interface IMFMediaEvent : IMFAttributes +{ + [PreserveSig] int GetEventType(out uint type); + [PreserveSig] int GetExtendedType(out Guid type); + [PreserveSig] int GetStatus(out int status); + [PreserveSig] int GetValue(nint propVariant); +} + +[GeneratedComInterface] +[Guid("2cd0bd52-bcd5-4b89-b62c-eadc0c031e7d")] +internal partial interface IMFMediaEventGenerator +{ + [PreserveSig] int GetEvent(uint flags, out nint mediaEvent); + [PreserveSig] int BeginGetEvent(nint callback, nint state); + [PreserveSig] int EndGetEvent(nint result, out nint mediaEvent); + [PreserveSig] int QueueEvent(uint type, in Guid extendedType, int status, nint value); +} + +[GeneratedComInterface] +[Guid("bf94c121-5b05-4e6f-8000-ba598961414d")] +internal partial interface IMFTransform +{ + [PreserveSig] int GetStreamLimits(out uint inputMinimum, out uint inputMaximum, out uint outputMinimum, out uint outputMaximum); + [PreserveSig] int GetStreamCount(out uint inputStreams, out uint outputStreams); + [PreserveSig] int GetStreamIDs(uint inputSize, nint inputIds, uint outputSize, nint outputIds); + [PreserveSig] int GetInputStreamInfo(uint stream, out MftInputStreamInfo info); + [PreserveSig] int GetOutputStreamInfo(uint stream, out MftOutputStreamInfo info); + [PreserveSig] int GetAttributes(out nint attributes); + [PreserveSig] int GetInputStreamAttributes(uint stream, out nint attributes); + [PreserveSig] int GetOutputStreamAttributes(uint stream, out nint attributes); + [PreserveSig] int DeleteInputStream(uint stream); + [PreserveSig] int AddInputStreams(uint count, nint ids); + [PreserveSig] int GetInputAvailableType(uint stream, uint index, out nint type); + [PreserveSig] int GetOutputAvailableType(uint stream, uint index, out nint type); + [PreserveSig] int SetInputType(uint stream, IMFMediaType? type, uint flags); + [PreserveSig] int SetOutputType(uint stream, IMFMediaType? type, uint flags); + [PreserveSig] int GetInputCurrentType(uint stream, out nint type); + [PreserveSig] int GetOutputCurrentType(uint stream, out nint type); + [PreserveSig] int GetInputStatus(uint stream, out uint flags); + [PreserveSig] int GetOutputStatus(out uint flags); + [PreserveSig] int SetOutputBounds(long lower, long upper); + [PreserveSig] int ProcessEvent(uint stream, nint mediaEvent); + [PreserveSig] int ProcessMessage(uint message, nint parameter); + [PreserveSig] int ProcessInput(uint stream, IMFSample sample, uint flags); + [PreserveSig] int ProcessOutput(uint flags, uint count, ref MftOutputDataBuffer buffers, out uint status); +} + +[GeneratedComInterface] +[Guid("901db4c7-31ce-41a2-85dc-8fa0bf41b8da")] +internal partial interface ICodecAPI +{ + [PreserveSig] int IsSupported(in Guid api); + [PreserveSig] int IsModifiable(in Guid api); + [PreserveSig] int GetParameterRange(in Guid api, nint minimum, nint maximum, nint step); + [PreserveSig] int GetParameterValues(in Guid api, out nint values, out uint count); + [PreserveSig] int GetDefaultValue(in Guid api, nint value); + [PreserveSig] int GetValue(in Guid api, nint value); + + /// Sets a property; points to a VARIANT (a ComVariant). + [PreserveSig] int SetValue(in Guid api, nint value); + [PreserveSig] int RegisterForEvent(in Guid api, nint userData); + [PreserveSig] int UnregisterForEvent(in Guid api); + [PreserveSig] int SetAllDefaults(); + [PreserveSig] int SetValueWithNotify(in Guid api, nint value, out nint changed, out uint count); + [PreserveSig] int SetAllDefaultsWithNotify(out nint changed, out uint count); + [PreserveSig] int GetAllSettings(nint stream); + [PreserveSig] int SetAllSettings(nint stream); + [PreserveSig] int SetAllSettingsWithNotify(nint stream, out nint changed, out uint count); +} + +[StructLayout(LayoutKind.Sequential)] +internal struct MftInputStreamInfo +{ + public long MaxLatency; + public uint Flags; + public uint Size; + public uint MaxLookahead; + public uint Alignment; +} + +[StructLayout(LayoutKind.Sequential)] +internal struct MftOutputStreamInfo +{ + public uint Flags; + public uint Size; + public uint Alignment; +} + +[StructLayout(LayoutKind.Sequential)] +internal struct MftOutputDataBuffer +{ + public uint StreamId; + public nint Sample; + public uint Status; + public nint Events; +} + +[StructLayout(LayoutKind.Sequential)] +internal struct MftRegisterTypeInfo +{ + public Guid MajorType; + public Guid Subtype; +} + +/// mfplat.dll entry points and the constants this assembly uses. +internal static unsafe partial class MfNative +{ + public const uint Version = 0x0002_0070; // MF_SDK_VERSION << 16 | MF_API_VERSION + public const uint StartupLite = 1; // MFSTARTUP_NOSOCKET + + public const uint EnumSync = 0x1, EnumAsync = 0x2, EnumHardware = 0x4, EnumSortAndFilter = 0x40; + + public const uint MessageCommandDrain = 0x1; + public const uint MessageNotifyBeginStreaming = 0x1000_0000; + public const uint MessageNotifyEndStreaming = 0x1000_0001; + public const uint MessageNotifyEndOfStream = 0x1000_0002; + public const uint MessageNotifyStartOfStream = 0x1000_0003; + + public const uint OutputStreamProvidesSamples = 0x100, OutputStreamCanProvideSamples = 0x200; + + public const uint EventError = 1, EventNeedInput = 601, EventHaveOutput = 602, EventDrainComplete = 603; + + public const int TransformNeedMoreInput = unchecked((int)0xC00D6D72); + public const int TransformStreamChange = unchecked((int)0xC00D6D61); + public const int NotAccepting = unchecked((int)0xC00D36B5); + public const int NoMoreTypes = unchecked((int)0xC00D36B9); + + public const uint InterlaceProgressive = 2; + public const uint NominalRange16To235 = 2; + public const uint PrimariesBt709 = 2; + public const uint TransferBt709 = 5; + public const uint MatrixBt709 = 1; + + public const uint RateControlCbr = 0, RateControlUnconstrainedVbr = 2, RateControlQuality = 3; + + [LibraryImport("mfplat.dll")] + public static partial int MFStartup(uint version, uint flags); + + [LibraryImport("mfplat.dll")] + public static partial int MFShutdown(); + + [LibraryImport("mfplat.dll")] + public static partial int MFTEnumEx(Guid category, uint flags, MftRegisterTypeInfo* inputType, MftRegisterTypeInfo* outputType, out nint activates, out uint count); + + [LibraryImport("mfplat.dll")] + public static partial int MFCreateMediaType(out nint mediaType); + + [LibraryImport("mfplat.dll")] + public static partial int MFCreateSample(out nint sample); + + [LibraryImport("mfplat.dll")] + public static partial int MFCreateMemoryBuffer(uint maxLength, out nint buffer); + + [LibraryImport("mfplat.dll")] + public static partial int MFCreate2DMediaBuffer(uint width, uint height, uint fourCc, [MarshalAs(UnmanagedType.Bool)] bool bottomUp, out nint buffer); +} + +/// Media Foundation GUIDs (mfapi.h, mftransform.h, codecapi.h). +internal static class MfGuids +{ + public static readonly Guid VideoEncoderCategory = new("f79eac7d-e545-4387-bdee-d647d7bde42a"); + public static readonly Guid MediaTypeVideo = new("73646976-0000-0010-8000-00aa00389b71"); + public static readonly Guid FormatH264 = new("34363248-0000-0010-8000-00aa00389b71"); + public static readonly Guid FormatHevc = new("43564548-0000-0010-8000-00aa00389b71"); + public static readonly Guid FormatAv1 = new("31305641-0000-0010-8000-00aa00389b71"); + public static readonly Guid FormatNv12 = new("3231564e-0000-0010-8000-00aa00389b71"); + public static readonly Guid FormatP010 = new("30313050-0000-0010-8000-00aa00389b71"); + + public static readonly Guid MajorType = new("48eba18e-f8c9-4687-bf11-0a74c9f96a8f"); + public static readonly Guid Subtype = new("f7e34c9a-42e8-4714-b74b-cb29d72c35e5"); + public static readonly Guid FrameSize = new("1652c33d-d6b2-4012-b834-72030849a37d"); + public static readonly Guid FrameRate = new("c459a2e8-3d2c-4e44-b132-fee5156c7bb0"); + public static readonly Guid PixelAspectRatio = new("c6376a1e-8d0a-4027-be45-6d9a0ad39bb6"); + public static readonly Guid InterlaceMode = new("e2724bb8-e676-4806-b4b2-a8d6efb44ccd"); + public static readonly Guid AverageBitrate = new("20332624-fb0d-4d9e-bd0d-cbf6786c102e"); + public static readonly Guid Mpeg2Profile = new("ad76a80b-2d5c-4e0b-b375-64e520137036"); + public static readonly Guid Mpeg2Level = new("96f66574-11c5-4015-8666-bff516436da7"); + public static readonly Guid MpegSequenceHeader = new("3c036de7-3ad0-4c9e-9216-ee6d6ac21cb3"); + public static readonly Guid DefaultStride = new("644b4e48-1e02-4516-b0eb-c01ca9d49ac6"); + public static readonly Guid NominalRange = new("c21b8ee5-b956-4071-8daf-325edf5cab11"); + public static readonly Guid VideoPrimaries = new("dbfbe4d7-0740-4ee0-8192-850ab0e21935"); + public static readonly Guid TransferFunction = new("5fb0fce9-be5c-4935-a811-ec838f8eed93"); + public static readonly Guid YuvMatrix = new("3e23d450-2c75-4d25-a00e-b91670d12327"); + + public static readonly Guid TransformAsync = new("f81a699a-649a-497d-8c73-29f8fed6ad7a"); + public static readonly Guid TransformAsyncUnlock = new("e5666d6b-3422-4eb6-a421-da7db1f8e207"); + public static readonly Guid TransformFlags = new("9359bb7e-6275-46c4-a025-1c01e45f1a86"); + public static readonly Guid FriendlyName = new("314ffbae-5b41-4c95-9c19-4e7d586face3"); + public static readonly Guid HardwareVendorId = new("3aecb0cc-035b-4bcc-8185-2b8d551ef3af"); + + public static readonly Guid SampleCleanPoint = new("9cdf01d8-a0f0-43ba-b077-eaa06cbd728a"); + public static readonly Guid SampleDecodeTimestamp = new("73a954d4-09e2-4861-befc-94bd97c08e6e"); + + public static readonly Guid CodecRateControlMode = new("1c0608e9-370c-4710-8a58-cb6181c42423"); + public static readonly Guid CodecMeanBitRate = new("f7222374-2144-4815-b550-a37f8e12ee52"); + public static readonly Guid CodecQuality = new("fcbf57a3-7ea5-4b0c-9644-69b40c39c391"); + public static readonly Guid CodecGopSize = new("95f31b26-95a4-41aa-9303-246a7fc6eef1"); + public static readonly Guid CodecBPictureCount = new("8d390aac-dc5c-4200-b57f-814d04babab2"); + public static readonly Guid CodecLowLatencyMode = new("9c27891a-ed7a-40e1-88e8-b22727a024ee"); + + public static readonly Guid IidTransform = typeof(IMFTransform).GUID; +} diff --git a/tests/Titano.Core.Windows.Tests/MediaFoundation/GpuAdaptersTests.cs b/tests/Titano.Core.Windows.Tests/MediaFoundation/GpuAdaptersTests.cs new file mode 100644 index 00000000..a84fee47 --- /dev/null +++ b/tests/Titano.Core.Windows.Tests/MediaFoundation/GpuAdaptersTests.cs @@ -0,0 +1,18 @@ +using Titano.Core.Windows.Dxgi; +using Titano.Testing; + +namespace Titano.Core.Windows.Tests.MediaFoundation; + +public sealed class GpuAdaptersTests(TestContext context) +{ + [Test] + [WindowsOnly] + public void EnumeratesAtLeastTheBasicRenderDriver() + { + var adapters = GpuAdapters.Enumerate(); + foreach (var a in adapters) + context.Log($"{a.Description} vendor=0x{a.VendorId:X4} device=0x{a.DeviceId:X4} vram={a.DedicatedVideoMemory >> 20} MiB software={a.IsSoftware}"); + Assert.NotEmpty(adapters); + Assert.True(adapters.All(a => a.Description.Length > 0 && !a.Description.Contains('\0'))); + } +} diff --git a/tests/Titano.Core.Windows.Tests/MediaFoundation/MediaFoundationEncoderTests.cs b/tests/Titano.Core.Windows.Tests/MediaFoundation/MediaFoundationEncoderTests.cs new file mode 100644 index 00000000..7ec92670 --- /dev/null +++ b/tests/Titano.Core.Windows.Tests/MediaFoundation/MediaFoundationEncoderTests.cs @@ -0,0 +1,153 @@ +using Titano.Core.Imaging; +using Titano.Core.Video; +using Titano.Core.Windows.MediaFoundation; +using Titano.Testing; + +namespace Titano.Core.Windows.Tests.MediaFoundation; + +[Serial] // encoders share GPU sessions and the MF runtime +public sealed class MediaFoundationEncoderTests(TestContext context) +{ + private const long Frame30 = 3000; // 1/30 s at 90 kHz + + [Test] + public void VendorIdsMapToNames() + { + Assert.Equal("NVIDIA", MediaFoundationEncoderProvider.VendorName("VEN_10DE")); + Assert.Equal("Intel", MediaFoundationEncoderProvider.VendorName("ven_8086")); + Assert.Equal("AMD", MediaFoundationEncoderProvider.VendorName("VEN_1002")); + Assert.Equal("VEN_ABCD", MediaFoundationEncoderProvider.VendorName("VEN_ABCD")); + Assert.Null(MediaFoundationEncoderProvider.VendorName(null)); + } + + [Test] + [WindowsOnly] + public void EnumeratesEncodersAndHidesSoftwareHevcAndAv1() + { + RequireMediaFoundation(); + var encoders = MediaFoundationEncoders.Enumerate(); + foreach (var e in encoders) context.Log($"{e.Codec} \"{e.Name}\" hardware={e.IsHardware} async={e.IsAsync} vendor={e.VendorId} 10-bit={e.SupportsTenBit}"); + Assert.True(encoders.All(e => !string.IsNullOrWhiteSpace(e.Name))); + + var capabilities = new MediaFoundationEncoderProvider().Capabilities; + Assert.True(capabilities.All(c => c.Hardware || c.Codec == VideoCodec.H264), "HEVC/AV1 are hardware-only"); + Assert.True(capabilities.All(c => !c.SupportsTenBit || c.Codec != VideoCodec.H264)); + Assert.Equal(encoders.Count(e => e.IsHardware || e.Codec == VideoCodec.H264), capabilities.Count); + } + + [Test] + [WindowsOnly] + [Timeout(60_000)] + public void SoftwareH264EncodeProducesAValidMp4() + { + RequireEncoder(VideoCodec.H264, hardware: false, tenBit: false); + var dir = context.CreateTempDirectory(); + var settings = Settings(Path.Combine(dir, "software.mp4"), VideoCodec.H264, 8) with { PreferHardware = false }; + var provider = new MediaFoundationEncoderProvider(); + Assert.True(provider.CanEncode(settings)); + using (var session = provider.Open(settings)) + { + Assert.False(session.IsHardware); + context.Log("encoder: " + session.EncoderName); + WriteFrames(session, 60, 8); + session.Complete(); + } + + var report = Mp4Inspector.Inspect(settings.OutputPath); + context.Log(report.ToString()); + Assert.True(report.IsValid, string.Join("; ", report.Errors)); + Assert.Equal(VideoCodec.H264, report.Codec); + Assert.Equal(1280, report.Width); + Assert.Equal(720, report.Height); + Assert.Equal(60, report.FrameCount); + Assert.Approximately(2.0, report.Duration.TotalSeconds, 0.034); + Assert.True(report.FastStart); + Assert.Count(1, Directory.GetFiles(dir)); // no temporary file left behind + } + + [Test] + [WindowsOnly] + [Timeout(60_000)] + public void HardwareHevcMain10EncodeWhenAvailable() + { + RequireEncoder(VideoCodec.Hevc, hardware: true, tenBit: true); + var dir = context.CreateTempDirectory(); + var settings = Settings(Path.Combine(dir, "main10.mp4"), VideoCodec.Hevc, 10); + using (var session = new MediaFoundationEncoderProvider().Open(settings)) + { + Assert.True(session.IsHardware); + context.Log("encoder: " + session.EncoderName); + WriteFrames(session, 60, 10); + session.Complete(); + } + var report = Mp4Inspector.Inspect(settings.OutputPath); + context.Log(report.ToString()); + Assert.True(report.IsValid, string.Join("; ", report.Errors)); + Assert.Equal(VideoCodec.Hevc, report.Codec); + Assert.Equal(10, report.BitDepth); + Assert.Equal(60, report.FrameCount); + } + + [Test] + [WindowsOnly] + [Timeout(60_000)] + public void DisposeWithoutCompleteLeavesNoFile() + { + RequireEncoder(VideoCodec.H264, hardware: false, tenBit: false); + var dir = context.CreateTempDirectory(); + var settings = Settings(Path.Combine(dir, "cancelled.mp4"), VideoCodec.H264, 8) with { PreferHardware = false }; + using (var session = new MediaFoundationEncoderProvider().Open(settings)) + { + WriteFrames(session, 5, 8); + } + Assert.Empty(Directory.GetFiles(dir)); + } + + private static EncodeSettings Settings(string path, VideoCodec codec, int bitDepth) => new() + { + OutputPath = path, + Codec = codec, + Width = 1280, + Height = 720, + FrameRate = new Rational(30, 1), + BitDepth = bitDepth, + Profile = codec == VideoCodec.Hevc ? (bitDepth == 10 ? CodecProfile.HevcMain10 : CodecProfile.HevcMain) : CodecProfile.H264High, + BitRate = 4_000_000, + GopLength = 30, + }; + + /// A moving colour ramp with a bright square, distinct per frame. + private static void WriteFrames(IVideoEncoderSession session, int count, int bitDepth) + { + using var frame = new ImageBuffer(1280, 720); + for (var f = 0; f < count; f++) + { + for (var y = 0; y < 720; y++) + { + var r = frame.Row(0, y); + var g = frame.Row(1, y); + var b = frame.Row(2, y); + for (var x = 0; x < 1280; x++) + { + var square = Math.Abs(x - 100 - f * 15) < 60 && Math.Abs(y - 360) < 60; + r[x] = square ? 1f : (float)ColorSpace.SrgbToLinear((x + f * 7) % 1280 / 1280.0); + g[x] = square ? 1f : (float)ColorSpace.SrgbToLinear(y / 720.0); + b[x] = (float)ColorSpace.SrgbToLinear(bitDepth == 10 ? 0.5 + 0.4 * Math.Sin((x + y) * 0.01) : 0.3); + } + } + session.WriteFrame(frame, Frame30); + } + } + + private static void RequireMediaFoundation() + { + if (!MediaFoundationRuntime.IsAvailable) TestContext.Skip("Media Foundation is not installed (Windows N or Server without Media Foundation)"); + } + + private static void RequireEncoder(VideoCodec codec, bool hardware, bool tenBit) + { + RequireMediaFoundation(); + if (!MediaFoundationEncoders.Enumerate().Any(e => e.Codec == codec && e.IsHardware == hardware && (!tenBit || e.SupportsTenBit))) + TestContext.Skip($"No {(hardware ? "hardware" : "software")} {codec}{(tenBit ? " 10-bit" : string.Empty)} encoder on this machine"); + } +} From aba6d629b90bfb3baa65cacbf8f922ce406202d9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 10:07:59 +0000 Subject: [PATCH 9/9] fix(video): roll back rejected MP4 samples and keep inspector reports read-only Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Video/Mp4/Mp4Muxer.cs | 22 ++- src/Titano.Core/Video/Mp4Inspector.cs | 146 ++++++++++-------- .../Titano.Core.Tests/Video/Mp4MuxerTests.cs | 16 ++ 3 files changed, 109 insertions(+), 75 deletions(-) diff --git a/src/Titano.Core/Video/Mp4/Mp4Muxer.cs b/src/Titano.Core/Video/Mp4/Mp4Muxer.cs index 2e1f3e74..27bccd5d 100644 --- a/src/Titano.Core/Video/Mp4/Mp4Muxer.cs +++ b/src/Titano.Core/Video/Mp4/Mp4Muxer.cs @@ -113,6 +113,7 @@ public sealed class Mp4Muxer /// /// Appends one access unit (H.264/HEVC: Annex-B or 4-byte length-prefixed NAL units; AV1: a temporal unit of OBUs) /// in decode order. is presentation minus decode time and may be negative. + /// A unit holding only parameter sets is recorded like and adds no sample. /// public void WriteSample(ReadOnlySpan data, long duration90k, bool isSync, long compositionOffset90k = 0) { @@ -122,8 +123,14 @@ public sealed class Mp4Muxer var start = _position; if (_stream.Position != start) _stream.Position = start; - var size = _config.Codec == VideoCodec.Av1 ? WriteObus(data) : WriteNalUnits(data); - if (size == 0) throw new ArgumentException("The sample contains no picture data", nameof(data)); + var (size, picture) = _config.Codec == VideoCodec.Av1 ? WriteObus(data) : WriteNalUnits(data); + if (size == 0) return; // only parameter sets / delimiters: recorded for the sample entry, no sample + if (!picture) + { + _stream.Position = start; // the next sample overwrites what was written + throw new ArgumentException("The sample contains no picture data", nameof(data)); + } + _position += size; if (_sizes.Count == 0 && !isSync) Log.Warning(LogCategory, "First sample is not a sync sample; players may not start decoding"); if (_chunkSampleCounts.Count == 0 || _chunkTicks >= ChunkTicks || _chunkBytes >= ChunkBytes) @@ -161,11 +168,12 @@ public sealed class Mp4Muxer _stream.Write(moov); if (_config.FastStart) MoveMoovToFront(configuration, moov.Length); else _position += moov.Length; + if (_stream.Length > _position) _stream.SetLength(_position); // drop bytes of a rejected last sample _stream.Flush(); _completed = true; } - private long WriteNalUnits(ReadOnlySpan data) + private (long Size, bool Picture) WriteNalUnits(ReadOnlySpan data) { Span length = stackalloc byte[4]; long size = 0; @@ -180,11 +188,10 @@ public sealed class Mp4Muxer size += 4 + nal.Length; picture |= IsPicture(nal[0]); } - _position += size; - return picture ? size : 0; + return (size, picture); } - private long WriteObus(ReadOnlySpan data) + private (long Size, bool Picture) WriteObus(ReadOnlySpan data) { var obus = Av1ObuParser.Split(data); if (obus.Count == 0 || obus[^1].Offset + obus[^1].TotalLength != data.Length) @@ -209,8 +216,7 @@ public sealed class Mp4Muxer _stream.Write(obu.Payload(data)); size += n + obu.PayloadLength; } - _position += size; - return picture ? size : 0; + return (size, picture); } private static int WriteSizedHeader(Span header, ReadOnlySpan data, Av1Obu obu) diff --git a/src/Titano.Core/Video/Mp4Inspector.cs b/src/Titano.Core/Video/Mp4Inspector.cs index 1edfa8c7..d4cb3bf6 100644 --- a/src/Titano.Core/Video/Mp4Inspector.cs +++ b/src/Titano.Core/Video/Mp4Inspector.cs @@ -28,9 +28,16 @@ public readonly record struct Mp4SampleInfo(long Offset, int Size, long DecodeTi /// Result of : structure, stream properties and problems found. public sealed class Mp4Report { - public List Errors { get; } = []; - public List Warnings { get; } = []; - public bool IsValid => Errors.Count == 0; + private readonly List _errors = []; + private readonly List _warnings = []; + + /// Problems that make the file invalid or unplayable. + public IReadOnlyList Errors => _errors; + + /// Oddities players usually tolerate. + public IReadOnlyList Warnings => _warnings; + + public bool IsValid => _errors.Count == 0; public IReadOnlyList Boxes { get; internal set; } = []; /// Sample entry four-character code ("avc1", "hvc1", "av01"…). @@ -68,6 +75,10 @@ public sealed class Mp4Report public bool HasEditList { get; internal set; } public IReadOnlyList Samples { get; internal set; } = []; + internal void Error(string message) => _errors.Add(message); + + internal void Warning(string message) => _warnings.Add(message); + public override string ToString() { var sb = new StringBuilder(); @@ -91,6 +102,7 @@ public static class Mp4Inspector private const int MaxBoxes = 100_000; private const long MaxMoovBytes = 256L << 20; private const int CheckedSamples = 4; + private const long MaxSamples = 20_000_000; public static Mp4Report Inspect(string path) { @@ -103,7 +115,7 @@ public static class Mp4Inspector var report = new Mp4Report(); if (!stream.CanSeek || !stream.CanRead) { - report.Errors.Add("stream is not readable and seekable"); + report.Error("stream is not readable and seekable"); return report; } try @@ -112,7 +124,7 @@ public static class Mp4Inspector } catch (IOException ex) { - report.Errors.Add("read failed: " + ex.Message); + report.Error("read failed: " + ex.Message); } return report; } @@ -138,7 +150,7 @@ public static class Mp4Inspector { if (read < 16) { - report.Errors.Add($"box '{type}' at {pos} is truncated"); + report.Error($"box '{type}' at {pos} is truncated"); break; } size = (long)Math.Min(BinaryPrimitives.ReadUInt64BigEndian(header[8..]), long.MaxValue); @@ -151,12 +163,12 @@ public static class Mp4Inspector } if (size < headerSize) { - report.Errors.Add($"box '{type}' at {pos} has invalid size {size}"); + report.Error($"box '{type}' at {pos} has invalid size {size}"); break; } if (size > length - pos) { - report.Errors.Add($"box '{type}' at {pos} extends past the end of the file ({size} > {length - pos} bytes)"); + report.Error($"box '{type}' at {pos} extends past the end of the file ({size} > {length - pos} bytes)"); size = length - pos; } @@ -165,11 +177,11 @@ public static class Mp4Inspector { if (moov is not null) { - report.Errors.Add("more than one moov box"); + report.Error("more than one moov box"); } else if (size - headerSize > MaxMoovBytes) { - report.Errors.Add($"moov is implausibly large ({size} bytes)"); + report.Error($"moov is implausibly large ({size} bytes)"); } else { @@ -182,22 +194,22 @@ public static class Mp4Inspector var box = new Mp4Box(type, pos, size, headerSize, children); if (type == "moov" && moov is null && children.Count > 0) moov = box; if (type == "mdat") mdats.Add((pos + headerSize, pos + size)); - if (type == "moof") report.Warnings.Add("fragmented MP4 (moof) is not inspected"); + if (type == "moof") report.Warning("fragmented MP4 (moof) is not inspected"); top.Add(box); pos += size; } - if (pos < length && budget > 0) report.Warnings.Add($"{length - pos} trailing bytes after the last box"); + if (pos < length && budget > 0) report.Warning($"{length - pos} trailing bytes after the last box"); report.Boxes = top; if (top.Count == 0 || top[0].Type != "ftyp") { - if (top.Exists(b => b.Type == "ftyp")) report.Warnings.Add("ftyp is not the first box"); - else report.Errors.Add("ftyp box missing"); + if (top.Exists(b => b.Type == "ftyp")) report.Warning("ftyp is not the first box"); + else report.Error("ftyp box missing"); } - if (mdats.Count == 0) report.Errors.Add("mdat box missing"); + if (mdats.Count == 0) report.Error("mdat box missing"); if (moov is null) { - report.Errors.Add("moov box missing or empty"); + report.Error("moov box missing or empty"); return; } report.FastStart = mdats.Count > 0 && moov.Offset < mdats[0].Start; @@ -225,7 +237,7 @@ public static class Mp4Inspector } if (size < headerSize || size > end - pos) { - report.Errors.Add($"box '{type}' at {baseOffset + pos} overruns its parent"); + report.Error($"box '{type}' at {baseOffset + pos} overruns its parent"); return list; } var payloadStart = pos + headerSize; @@ -254,7 +266,7 @@ public static class Mp4Inspector private static void InspectMovie(Stream stream, Mp4Box moov, List<(long Start, long End)> mdats, Mp4Report report) { var mvhd = moov.Child("mvhd"); - if (mvhd is null) report.Errors.Add("mvhd missing"); + if (mvhd is null) report.Error("mvhd missing"); var movieScale = mvhd is null ? 0u : ReadTimescale(mvhd.Payload.Span, out _); Mp4Box? trak = null; @@ -268,25 +280,25 @@ public static class Mp4Inspector } if (trak is null) { - report.Errors.Add("no video track (trak with hdlr 'vide')"); + report.Error("no video track (trak with hdlr 'vide')"); return; } - if (trak.Child("tkhd") is null) report.Errors.Add("tkhd missing"); + if (trak.Child("tkhd") is null) report.Error("tkhd missing"); var mdia = trak.Child("mdia"); var mdhd = mdia?.Child("mdhd"); var minf = mdia?.Child("minf"); var stbl = minf?.Child("stbl"); - if (mdhd is null) report.Errors.Add("mdhd missing"); - if (minf?.Child("vmhd") is null) report.Errors.Add("vmhd missing"); - if (minf?.Child("dinf")?.Child("dref") is null) report.Errors.Add("dinf/dref missing"); + if (mdhd is null) report.Error("mdhd missing"); + if (minf?.Child("vmhd") is null) report.Error("vmhd missing"); + if (minf?.Child("dinf")?.Child("dref") is null) report.Error("dinf/dref missing"); if (stbl is null) { - report.Errors.Add("stbl missing"); + report.Error("stbl missing"); return; } long mdhdDuration = 0; var mediaScale = mdhd is null ? 0u : ReadTimescale(mdhd.Payload.Span, out mdhdDuration); - if (mdhd is not null && mediaScale == 0) report.Errors.Add("mdhd timescale is zero"); + if (mdhd is not null && mediaScale == 0) report.Error("mdhd timescale is zero"); report.TimeScale = mediaScale; var config = InspectSampleDescription(stbl, report); @@ -298,20 +310,20 @@ public static class Mp4Inspector long stscTotal = 0; foreach (var n in samplesPerChunk) stscTotal += n; - if (durations.Length != sizes.Length) report.Errors.Add($"stts covers {durations.Length} samples, stsz has {sizes.Length}"); - if (stscTotal != sizes.Length) report.Errors.Add($"stsc/stco map {stscTotal} samples, stsz has {sizes.Length}"); + if (durations.Length != sizes.Length) report.Error($"stts covers {durations.Length} samples, stsz has {sizes.Length}"); + if (stscTotal != sizes.Length) report.Error($"stsc/stco map {stscTotal} samples, stsz has {sizes.Length}"); var count = sizes.Length; report.FrameCount = count; if (count == 0) { - report.Errors.Add("track has no samples"); + report.Error("track has no samples"); return; } var offsets = ReadCompositionOffsets(stbl, count, report); var sync = ReadSyncSamples(stbl, count, report); report.SyncSampleCount = sync?.Count ?? count; - if (sync is not null && (sync.Count == 0 || sync[0] != 1)) report.Errors.Add("first sample is not a sync sample"); + if (sync is not null && (sync.Count == 0 || sync[0] != 1)) report.Error("first sample is not a sync sample"); var samples = new Mp4SampleInfo[count]; var index = 0; @@ -331,7 +343,7 @@ public static class Mp4Inspector decode += duration; bytes += sizes[index]; } - if (!InsideMdat(chunkStart, offset, mdats)) report.Errors.Add($"chunk {c + 1} [{chunkStart}, {offset}) lies outside mdat"); + if (!InsideMdat(chunkStart, offset, mdats)) report.Error($"chunk {c + 1} [{chunkStart}, {offset}) lies outside mdat"); } report.Samples = samples; report.MediaBytes = bytes; @@ -339,17 +351,17 @@ public static class Mp4Inspector for (var i = 0; i < durations.Length; i++) { if (durations[i] != 0) continue; - if (i == durations.Length - 1) report.Warnings.Add("last sample has zero duration"); - else report.Errors.Add($"sample {i + 1} has zero duration"); + if (i == durations.Length - 1) report.Warning("last sample has zero duration"); + else report.Error($"sample {i + 1} has zero duration"); } - if (mdhd is not null && mdhdDuration != decode) report.Warnings.Add($"mdhd duration {mdhdDuration} differs from the sample durations sum {decode}"); + if (mdhd is not null && mdhdDuration != decode) report.Warning($"mdhd duration {mdhdDuration} differs from the sample durations sum {decode}"); report.DurationTicks = decode; var elst = trak.Child("edts")?.Child("elst"); if (elst is not null && ReadEditList(elst.Payload.Span, out var segment, out var mediaTime, report)) { report.HasEditList = true; - if (mediaTime < 0 || (mediaTime >= decode && decode > 0)) report.Errors.Add($"edit list media_time {mediaTime} outside the media"); + if (mediaTime < 0 || (mediaTime >= decode && decode > 0)) report.Error($"edit list media_time {mediaTime} outside the media"); if (movieScale > 0) report.DurationTicks = (long)Math.Round(segment * (double)mediaScale / movieScale); } var seconds = mediaScale == 0 ? 0 : report.DurationTicks / (double)mediaScale; @@ -363,7 +375,7 @@ public static class Mp4Inspector var stsd = stbl.Child("stsd"); if (stsd is null || stsd.Children.Count == 0) { - report.Errors.Add("stsd missing or empty"); + report.Error("stsd missing or empty"); return null; } var entry = stsd.Children[0]; @@ -371,7 +383,7 @@ public static class Mp4Inspector var payload = entry.Payload.Span; if (payload.Length < 78) { - report.Errors.Add($"sample entry '{entry.Type}' truncated"); + report.Error($"sample entry '{entry.Type}' truncated"); return null; } report.Width = BinaryPrimitives.ReadUInt16BigEndian(payload[24..]); @@ -385,26 +397,26 @@ public static class Mp4Inspector }; if (configBox is null) { - report.Warnings.Add($"sample entry '{entry.Type}' is not validated"); + report.Warning($"sample entry '{entry.Type}' is not validated"); return null; } var box = entry.Child(configBox); if (box is null) { - report.Errors.Add($"{configBox} missing in '{entry.Type}'"); + report.Error($"{configBox} missing in '{entry.Type}'"); return null; } if (!DecoderConfiguration.TryParse(configBox, box.Payload.Span, out var config, out var error) || config is null) { - report.Errors.Add($"{configBox} unparseable: {error}"); + report.Error($"{configBox} unparseable: {error}"); return null; } - if (error is not null) report.Errors.Add(error); + if (error is not null) report.Error(error); report.Codec = config.Codec; report.CodecString = config.CodecString; report.BitDepth = config.BitDepth; if (config.Width != report.Width || config.Height != report.Height) - report.Errors.Add($"{configBox} declares {config.Width}x{config.Height}, sample entry {report.Width}x{report.Height}"); + report.Error($"{configBox} declares {config.Width}x{config.Height}, sample entry {report.Width}x{report.Height}"); return config; } @@ -423,12 +435,12 @@ public static class Mp4Inspector { var obus = Av1ObuParser.Split(data); if (obus.Count == 0 || obus[^1].Offset + obus[^1].TotalLength != data.Length) - report.Errors.Add($"sample {i + 1}: OBU sizes do not cover the sample"); + report.Error($"sample {i + 1}: OBU sizes do not cover the sample"); continue; } if (!AnnexBParser.TrySplitLengthPrefixed(data, 4, out var units)) { - report.Errors.Add($"sample {i + 1}: NAL length prefixes do not cover the sample"); + report.Error($"sample {i + 1}: NAL length prefixes do not cover the sample"); continue; } if (i != 0) continue; @@ -440,7 +452,7 @@ public static class Mp4Inspector ? AnnexBParser.H264Type(h) == H264NalUnitType.IdrSlice : AnnexBParser.HevcType(h) is >= HevcNalUnitType.BlaWLp and <= HevcNalUnitType.Cra; } - if (!randomAccess) report.Warnings.Add("first sample contains no IDR/IRAP picture"); + if (!randomAccess) report.Warning("first sample contains no IDR/IRAP picture"); } } @@ -449,7 +461,7 @@ public static class Mp4Inspector var box = stbl.Child("stts"); if (box is null) { - report.Errors.Add("stts missing"); + report.Error("stts missing"); return null; } var p = box.Payload.Span; @@ -459,9 +471,9 @@ public static class Mp4Inspector { var n = BinaryPrimitives.ReadUInt32BigEndian(p[(8 + i * 8)..]); var delta = BinaryPrimitives.ReadUInt32BigEndian(p[(12 + i * 8)..]); - if (list.Count + (long)n > 50_000_000) + if (list.Count + (long)n > MaxSamples) { - report.Errors.Add("stts sample count implausible"); + report.Error("stts sample count implausible"); return null; } for (var k = 0; k < n; k++) list.Add(delta); @@ -487,7 +499,7 @@ public static class Mp4Inspector covered += n; } report.HasCompositionOffsets = true; - if (covered != count) report.Errors.Add($"ctts covers {covered} of {count} samples"); + if (covered != count) report.Error($"ctts covers {covered} of {count} samples"); return result; } @@ -503,7 +515,7 @@ public static class Mp4Inspector var s = BinaryPrimitives.ReadUInt32BigEndian(p[(8 + i * 4)..]); if (s == 0 || s > count || (list.Count > 0 && s <= list[^1])) { - report.Errors.Add($"stss entry {s} out of order or range"); + report.Error($"stss entry {s} out of order or range"); break; } list.Add(s); @@ -516,42 +528,42 @@ public static class Mp4Inspector var box = stbl.Child("stsz") ?? stbl.Child("stz2"); if (box is null) { - report.Errors.Add("stsz missing"); + report.Error("stsz missing"); return null; } var p = box.Payload.Span; if (p.Length < 12) { - report.Errors.Add($"{box.Type} truncated"); + report.Error($"{box.Type} truncated"); return null; } var count = BinaryPrimitives.ReadUInt32BigEndian(p[8..]); - if (count > 50_000_000) + if (count > MaxSamples) { - report.Errors.Add($"{box.Type} sample count implausible"); + report.Error($"{box.Type} sample count implausible"); + return null; + } + var constant = box.Type == "stsz" ? BinaryPrimitives.ReadUInt32BigEndian(p[4..]) : 0; + if (box.Type == "stsz" && constant == 0 && p.Length < 12 + count * 4L) + { + report.Error("stsz truncated"); return null; } var sizes = new uint[count]; if (box.Type == "stsz") { - var constant = BinaryPrimitives.ReadUInt32BigEndian(p[4..]); if (constant != 0) { Array.Fill(sizes, constant); return sizes; } - if (p.Length < 12 + count * 4L) - { - report.Errors.Add("stsz truncated"); - return null; - } for (var i = 0; i < count; i++) sizes[i] = BinaryPrimitives.ReadUInt32BigEndian(p[(12 + i * 4)..]); return sizes; } var field = p[7]; if (field is not (4 or 8 or 16) || p.Length < 12 + (count * field + 7) / 8) { - report.Errors.Add("stz2 invalid"); + report.Error("stz2 invalid"); return null; } for (var i = 0; i < count; i++) @@ -571,7 +583,7 @@ public static class Mp4Inspector var box = stbl.Child("stco") ?? stbl.Child("co64"); if (box is null) { - report.Errors.Add("stco/co64 missing"); + report.Error("stco/co64 missing"); return null; } var wide = box.Type == "co64"; @@ -593,7 +605,7 @@ public static class Mp4Inspector var box = stbl.Child("stsc"); if (box is null) { - report.Errors.Add("stsc missing"); + report.Error("stsc missing"); return null; } var p = box.Payload.Span; @@ -606,10 +618,10 @@ public static class Mp4Inspector var next = i + 1 < entries ? BinaryPrimitives.ReadUInt32BigEndian(p[(8 + (i + 1) * 12)..]) : (uint)chunks + 1; if (first == 0 || next <= first || (i == 0 && first != 1)) { - report.Errors.Add("stsc first_chunk values invalid"); + report.Error("stsc first_chunk values invalid"); return null; } - if (BinaryPrimitives.ReadUInt32BigEndian(p[(16 + i * 12)..]) != 1) report.Errors.Add("stsc references a sample description other than 1"); + if (BinaryPrimitives.ReadUInt32BigEndian(p[(16 + i * 12)..]) != 1) report.Error("stsc references a sample description other than 1"); for (var c = first; c < next && c <= chunks; c++) perChunk[c - 1] = samples; } return perChunk; @@ -620,7 +632,7 @@ public static class Mp4Inspector segment = mediaTime = 0; var wide = p.Length > 0 && p[0] == 1; if (!Entries(p, wide ? 20 : 12, report, "elst", out var entries) || entries == 0) return false; - if (entries > 1) report.Warnings.Add($"edit list has {entries} entries; only the first is evaluated"); + if (entries > 1) report.Warning($"edit list has {entries} entries; only the first is evaluated"); segment = wide ? (long)Math.Min(BinaryPrimitives.ReadUInt64BigEndian(p[8..]), long.MaxValue) : BinaryPrimitives.ReadUInt32BigEndian(p[8..]); mediaTime = wide ? BinaryPrimitives.ReadInt64BigEndian(p[16..]) : BinaryPrimitives.ReadInt32BigEndian(p[12..]); return true; @@ -632,13 +644,13 @@ public static class Mp4Inspector entries = 0; if (p.Length < 8) { - report.Errors.Add($"{type} truncated"); + report.Error($"{type} truncated"); return false; } var n = BinaryPrimitives.ReadUInt32BigEndian(p[4..]); if (n > (p.Length - 8) / entrySize) { - report.Errors.Add($"{type} declares {n} entries but holds {(p.Length - 8) / entrySize}"); + report.Error($"{type} declares {n} entries but holds {(p.Length - 8) / entrySize}"); return false; } entries = (int)n; diff --git a/tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs b/tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs index d505885b..ea1fd44b 100644 --- a/tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs +++ b/tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs @@ -198,6 +198,22 @@ public sealed class Mp4MuxerTests(TestContext context) Assert.Throws(() => muxer.WriteSample([0, 0, 1, 0x65, 1], 0, true)); } + [Test] + public void ParameterSetOnlyUnitsAddNoSampleAndSeiOnlyUnitsAreRejected() + { + using var ms = new MemoryStream(); + var muxer = new Mp4Muxer(ms, new VideoTrackConfig { Codec = VideoCodec.H264, Width = 1920, Height = 1080 }); + muxer.WriteSample(AnnexBParser.ToAnnexB([SyntheticStreams.H264Sps(), SyntheticStreams.H264Pps()]), 3000, true); + Assert.Equal(0, muxer.SampleCount); + Assert.Throws(() => muxer.WriteSample(AnnexBParser.ToAnnexB([[0x06, 0x05, 0x01, 0x00, 0x80]]), 3000, false)); + muxer.WriteSample(AnnexBParser.ToAnnexB([SyntheticStreams.H264Slice(true, 64, 1)]), 3000, true); + muxer.Complete(); + var report = Mp4Inspector.Inspect(ms); + AssertClean(report); + Assert.Equal(1, report.FrameCount); + Assert.Equal(0, report.Warnings.Count); + } + [Test] public void InspectorSurvivesTruncationAndCorruption() {