From aba6d629b90bfb3baa65cacbf8f922ce406202d9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 10:07:59 +0000 Subject: [PATCH] fix(video): roll back rejected MP4 samples and keep inspector reports read-only Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Video/Mp4/Mp4Muxer.cs | 22 ++- src/Titano.Core/Video/Mp4Inspector.cs | 146 ++++++++++-------- .../Titano.Core.Tests/Video/Mp4MuxerTests.cs | 16 ++ 3 files changed, 109 insertions(+), 75 deletions(-) diff --git a/src/Titano.Core/Video/Mp4/Mp4Muxer.cs b/src/Titano.Core/Video/Mp4/Mp4Muxer.cs index 2e1f3e74..27bccd5d 100644 --- a/src/Titano.Core/Video/Mp4/Mp4Muxer.cs +++ b/src/Titano.Core/Video/Mp4/Mp4Muxer.cs @@ -113,6 +113,7 @@ public sealed class Mp4Muxer /// /// Appends one access unit (H.264/HEVC: Annex-B or 4-byte length-prefixed NAL units; AV1: a temporal unit of OBUs) /// in decode order. is presentation minus decode time and may be negative. + /// A unit holding only parameter sets is recorded like and adds no sample. /// public void WriteSample(ReadOnlySpan data, long duration90k, bool isSync, long compositionOffset90k = 0) { @@ -122,8 +123,14 @@ public sealed class Mp4Muxer var start = _position; if (_stream.Position != start) _stream.Position = start; - var size = _config.Codec == VideoCodec.Av1 ? WriteObus(data) : WriteNalUnits(data); - if (size == 0) throw new ArgumentException("The sample contains no picture data", nameof(data)); + var (size, picture) = _config.Codec == VideoCodec.Av1 ? WriteObus(data) : WriteNalUnits(data); + if (size == 0) return; // only parameter sets / delimiters: recorded for the sample entry, no sample + if (!picture) + { + _stream.Position = start; // the next sample overwrites what was written + throw new ArgumentException("The sample contains no picture data", nameof(data)); + } + _position += size; if (_sizes.Count == 0 && !isSync) Log.Warning(LogCategory, "First sample is not a sync sample; players may not start decoding"); if (_chunkSampleCounts.Count == 0 || _chunkTicks >= ChunkTicks || _chunkBytes >= ChunkBytes) @@ -161,11 +168,12 @@ public sealed class Mp4Muxer _stream.Write(moov); if (_config.FastStart) MoveMoovToFront(configuration, moov.Length); else _position += moov.Length; + if (_stream.Length > _position) _stream.SetLength(_position); // drop bytes of a rejected last sample _stream.Flush(); _completed = true; } - private long WriteNalUnits(ReadOnlySpan data) + private (long Size, bool Picture) WriteNalUnits(ReadOnlySpan data) { Span length = stackalloc byte[4]; long size = 0; @@ -180,11 +188,10 @@ public sealed class Mp4Muxer size += 4 + nal.Length; picture |= IsPicture(nal[0]); } - _position += size; - return picture ? size : 0; + return (size, picture); } - private long WriteObus(ReadOnlySpan data) + private (long Size, bool Picture) WriteObus(ReadOnlySpan data) { var obus = Av1ObuParser.Split(data); if (obus.Count == 0 || obus[^1].Offset + obus[^1].TotalLength != data.Length) @@ -209,8 +216,7 @@ public sealed class Mp4Muxer _stream.Write(obu.Payload(data)); size += n + obu.PayloadLength; } - _position += size; - return picture ? size : 0; + return (size, picture); } private static int WriteSizedHeader(Span header, ReadOnlySpan data, Av1Obu obu) diff --git a/src/Titano.Core/Video/Mp4Inspector.cs b/src/Titano.Core/Video/Mp4Inspector.cs index 1edfa8c7..d4cb3bf6 100644 --- a/src/Titano.Core/Video/Mp4Inspector.cs +++ b/src/Titano.Core/Video/Mp4Inspector.cs @@ -28,9 +28,16 @@ public readonly record struct Mp4SampleInfo(long Offset, int Size, long DecodeTi /// Result of : structure, stream properties and problems found. public sealed class Mp4Report { - public List Errors { get; } = []; - public List Warnings { get; } = []; - public bool IsValid => Errors.Count == 0; + private readonly List _errors = []; + private readonly List _warnings = []; + + /// Problems that make the file invalid or unplayable. + public IReadOnlyList Errors => _errors; + + /// Oddities players usually tolerate. + public IReadOnlyList Warnings => _warnings; + + public bool IsValid => _errors.Count == 0; public IReadOnlyList Boxes { get; internal set; } = []; /// Sample entry four-character code ("avc1", "hvc1", "av01"…). @@ -68,6 +75,10 @@ public sealed class Mp4Report public bool HasEditList { get; internal set; } public IReadOnlyList Samples { get; internal set; } = []; + internal void Error(string message) => _errors.Add(message); + + internal void Warning(string message) => _warnings.Add(message); + public override string ToString() { var sb = new StringBuilder(); @@ -91,6 +102,7 @@ public static class Mp4Inspector private const int MaxBoxes = 100_000; private const long MaxMoovBytes = 256L << 20; private const int CheckedSamples = 4; + private const long MaxSamples = 20_000_000; public static Mp4Report Inspect(string path) { @@ -103,7 +115,7 @@ public static class Mp4Inspector var report = new Mp4Report(); if (!stream.CanSeek || !stream.CanRead) { - report.Errors.Add("stream is not readable and seekable"); + report.Error("stream is not readable and seekable"); return report; } try @@ -112,7 +124,7 @@ public static class Mp4Inspector } catch (IOException ex) { - report.Errors.Add("read failed: " + ex.Message); + report.Error("read failed: " + ex.Message); } return report; } @@ -138,7 +150,7 @@ public static class Mp4Inspector { if (read < 16) { - report.Errors.Add($"box '{type}' at {pos} is truncated"); + report.Error($"box '{type}' at {pos} is truncated"); break; } size = (long)Math.Min(BinaryPrimitives.ReadUInt64BigEndian(header[8..]), long.MaxValue); @@ -151,12 +163,12 @@ public static class Mp4Inspector } if (size < headerSize) { - report.Errors.Add($"box '{type}' at {pos} has invalid size {size}"); + report.Error($"box '{type}' at {pos} has invalid size {size}"); break; } if (size > length - pos) { - report.Errors.Add($"box '{type}' at {pos} extends past the end of the file ({size} > {length - pos} bytes)"); + report.Error($"box '{type}' at {pos} extends past the end of the file ({size} > {length - pos} bytes)"); size = length - pos; } @@ -165,11 +177,11 @@ public static class Mp4Inspector { if (moov is not null) { - report.Errors.Add("more than one moov box"); + report.Error("more than one moov box"); } else if (size - headerSize > MaxMoovBytes) { - report.Errors.Add($"moov is implausibly large ({size} bytes)"); + report.Error($"moov is implausibly large ({size} bytes)"); } else { @@ -182,22 +194,22 @@ public static class Mp4Inspector var box = new Mp4Box(type, pos, size, headerSize, children); if (type == "moov" && moov is null && children.Count > 0) moov = box; if (type == "mdat") mdats.Add((pos + headerSize, pos + size)); - if (type == "moof") report.Warnings.Add("fragmented MP4 (moof) is not inspected"); + if (type == "moof") report.Warning("fragmented MP4 (moof) is not inspected"); top.Add(box); pos += size; } - if (pos < length && budget > 0) report.Warnings.Add($"{length - pos} trailing bytes after the last box"); + if (pos < length && budget > 0) report.Warning($"{length - pos} trailing bytes after the last box"); report.Boxes = top; if (top.Count == 0 || top[0].Type != "ftyp") { - if (top.Exists(b => b.Type == "ftyp")) report.Warnings.Add("ftyp is not the first box"); - else report.Errors.Add("ftyp box missing"); + if (top.Exists(b => b.Type == "ftyp")) report.Warning("ftyp is not the first box"); + else report.Error("ftyp box missing"); } - if (mdats.Count == 0) report.Errors.Add("mdat box missing"); + if (mdats.Count == 0) report.Error("mdat box missing"); if (moov is null) { - report.Errors.Add("moov box missing or empty"); + report.Error("moov box missing or empty"); return; } report.FastStart = mdats.Count > 0 && moov.Offset < mdats[0].Start; @@ -225,7 +237,7 @@ public static class Mp4Inspector } if (size < headerSize || size > end - pos) { - report.Errors.Add($"box '{type}' at {baseOffset + pos} overruns its parent"); + report.Error($"box '{type}' at {baseOffset + pos} overruns its parent"); return list; } var payloadStart = pos + headerSize; @@ -254,7 +266,7 @@ public static class Mp4Inspector private static void InspectMovie(Stream stream, Mp4Box moov, List<(long Start, long End)> mdats, Mp4Report report) { var mvhd = moov.Child("mvhd"); - if (mvhd is null) report.Errors.Add("mvhd missing"); + if (mvhd is null) report.Error("mvhd missing"); var movieScale = mvhd is null ? 0u : ReadTimescale(mvhd.Payload.Span, out _); Mp4Box? trak = null; @@ -268,25 +280,25 @@ public static class Mp4Inspector } if (trak is null) { - report.Errors.Add("no video track (trak with hdlr 'vide')"); + report.Error("no video track (trak with hdlr 'vide')"); return; } - if (trak.Child("tkhd") is null) report.Errors.Add("tkhd missing"); + if (trak.Child("tkhd") is null) report.Error("tkhd missing"); var mdia = trak.Child("mdia"); var mdhd = mdia?.Child("mdhd"); var minf = mdia?.Child("minf"); var stbl = minf?.Child("stbl"); - if (mdhd is null) report.Errors.Add("mdhd missing"); - if (minf?.Child("vmhd") is null) report.Errors.Add("vmhd missing"); - if (minf?.Child("dinf")?.Child("dref") is null) report.Errors.Add("dinf/dref missing"); + if (mdhd is null) report.Error("mdhd missing"); + if (minf?.Child("vmhd") is null) report.Error("vmhd missing"); + if (minf?.Child("dinf")?.Child("dref") is null) report.Error("dinf/dref missing"); if (stbl is null) { - report.Errors.Add("stbl missing"); + report.Error("stbl missing"); return; } long mdhdDuration = 0; var mediaScale = mdhd is null ? 0u : ReadTimescale(mdhd.Payload.Span, out mdhdDuration); - if (mdhd is not null && mediaScale == 0) report.Errors.Add("mdhd timescale is zero"); + if (mdhd is not null && mediaScale == 0) report.Error("mdhd timescale is zero"); report.TimeScale = mediaScale; var config = InspectSampleDescription(stbl, report); @@ -298,20 +310,20 @@ public static class Mp4Inspector long stscTotal = 0; foreach (var n in samplesPerChunk) stscTotal += n; - if (durations.Length != sizes.Length) report.Errors.Add($"stts covers {durations.Length} samples, stsz has {sizes.Length}"); - if (stscTotal != sizes.Length) report.Errors.Add($"stsc/stco map {stscTotal} samples, stsz has {sizes.Length}"); + if (durations.Length != sizes.Length) report.Error($"stts covers {durations.Length} samples, stsz has {sizes.Length}"); + if (stscTotal != sizes.Length) report.Error($"stsc/stco map {stscTotal} samples, stsz has {sizes.Length}"); var count = sizes.Length; report.FrameCount = count; if (count == 0) { - report.Errors.Add("track has no samples"); + report.Error("track has no samples"); return; } var offsets = ReadCompositionOffsets(stbl, count, report); var sync = ReadSyncSamples(stbl, count, report); report.SyncSampleCount = sync?.Count ?? count; - if (sync is not null && (sync.Count == 0 || sync[0] != 1)) report.Errors.Add("first sample is not a sync sample"); + if (sync is not null && (sync.Count == 0 || sync[0] != 1)) report.Error("first sample is not a sync sample"); var samples = new Mp4SampleInfo[count]; var index = 0; @@ -331,7 +343,7 @@ public static class Mp4Inspector decode += duration; bytes += sizes[index]; } - if (!InsideMdat(chunkStart, offset, mdats)) report.Errors.Add($"chunk {c + 1} [{chunkStart}, {offset}) lies outside mdat"); + if (!InsideMdat(chunkStart, offset, mdats)) report.Error($"chunk {c + 1} [{chunkStart}, {offset}) lies outside mdat"); } report.Samples = samples; report.MediaBytes = bytes; @@ -339,17 +351,17 @@ public static class Mp4Inspector for (var i = 0; i < durations.Length; i++) { if (durations[i] != 0) continue; - if (i == durations.Length - 1) report.Warnings.Add("last sample has zero duration"); - else report.Errors.Add($"sample {i + 1} has zero duration"); + if (i == durations.Length - 1) report.Warning("last sample has zero duration"); + else report.Error($"sample {i + 1} has zero duration"); } - if (mdhd is not null && mdhdDuration != decode) report.Warnings.Add($"mdhd duration {mdhdDuration} differs from the sample durations sum {decode}"); + if (mdhd is not null && mdhdDuration != decode) report.Warning($"mdhd duration {mdhdDuration} differs from the sample durations sum {decode}"); report.DurationTicks = decode; var elst = trak.Child("edts")?.Child("elst"); if (elst is not null && ReadEditList(elst.Payload.Span, out var segment, out var mediaTime, report)) { report.HasEditList = true; - if (mediaTime < 0 || (mediaTime >= decode && decode > 0)) report.Errors.Add($"edit list media_time {mediaTime} outside the media"); + if (mediaTime < 0 || (mediaTime >= decode && decode > 0)) report.Error($"edit list media_time {mediaTime} outside the media"); if (movieScale > 0) report.DurationTicks = (long)Math.Round(segment * (double)mediaScale / movieScale); } var seconds = mediaScale == 0 ? 0 : report.DurationTicks / (double)mediaScale; @@ -363,7 +375,7 @@ public static class Mp4Inspector var stsd = stbl.Child("stsd"); if (stsd is null || stsd.Children.Count == 0) { - report.Errors.Add("stsd missing or empty"); + report.Error("stsd missing or empty"); return null; } var entry = stsd.Children[0]; @@ -371,7 +383,7 @@ public static class Mp4Inspector var payload = entry.Payload.Span; if (payload.Length < 78) { - report.Errors.Add($"sample entry '{entry.Type}' truncated"); + report.Error($"sample entry '{entry.Type}' truncated"); return null; } report.Width = BinaryPrimitives.ReadUInt16BigEndian(payload[24..]); @@ -385,26 +397,26 @@ public static class Mp4Inspector }; if (configBox is null) { - report.Warnings.Add($"sample entry '{entry.Type}' is not validated"); + report.Warning($"sample entry '{entry.Type}' is not validated"); return null; } var box = entry.Child(configBox); if (box is null) { - report.Errors.Add($"{configBox} missing in '{entry.Type}'"); + report.Error($"{configBox} missing in '{entry.Type}'"); return null; } if (!DecoderConfiguration.TryParse(configBox, box.Payload.Span, out var config, out var error) || config is null) { - report.Errors.Add($"{configBox} unparseable: {error}"); + report.Error($"{configBox} unparseable: {error}"); return null; } - if (error is not null) report.Errors.Add(error); + if (error is not null) report.Error(error); report.Codec = config.Codec; report.CodecString = config.CodecString; report.BitDepth = config.BitDepth; if (config.Width != report.Width || config.Height != report.Height) - report.Errors.Add($"{configBox} declares {config.Width}x{config.Height}, sample entry {report.Width}x{report.Height}"); + report.Error($"{configBox} declares {config.Width}x{config.Height}, sample entry {report.Width}x{report.Height}"); return config; } @@ -423,12 +435,12 @@ public static class Mp4Inspector { var obus = Av1ObuParser.Split(data); if (obus.Count == 0 || obus[^1].Offset + obus[^1].TotalLength != data.Length) - report.Errors.Add($"sample {i + 1}: OBU sizes do not cover the sample"); + report.Error($"sample {i + 1}: OBU sizes do not cover the sample"); continue; } if (!AnnexBParser.TrySplitLengthPrefixed(data, 4, out var units)) { - report.Errors.Add($"sample {i + 1}: NAL length prefixes do not cover the sample"); + report.Error($"sample {i + 1}: NAL length prefixes do not cover the sample"); continue; } if (i != 0) continue; @@ -440,7 +452,7 @@ public static class Mp4Inspector ? AnnexBParser.H264Type(h) == H264NalUnitType.IdrSlice : AnnexBParser.HevcType(h) is >= HevcNalUnitType.BlaWLp and <= HevcNalUnitType.Cra; } - if (!randomAccess) report.Warnings.Add("first sample contains no IDR/IRAP picture"); + if (!randomAccess) report.Warning("first sample contains no IDR/IRAP picture"); } } @@ -449,7 +461,7 @@ public static class Mp4Inspector var box = stbl.Child("stts"); if (box is null) { - report.Errors.Add("stts missing"); + report.Error("stts missing"); return null; } var p = box.Payload.Span; @@ -459,9 +471,9 @@ public static class Mp4Inspector { var n = BinaryPrimitives.ReadUInt32BigEndian(p[(8 + i * 8)..]); var delta = BinaryPrimitives.ReadUInt32BigEndian(p[(12 + i * 8)..]); - if (list.Count + (long)n > 50_000_000) + if (list.Count + (long)n > MaxSamples) { - report.Errors.Add("stts sample count implausible"); + report.Error("stts sample count implausible"); return null; } for (var k = 0; k < n; k++) list.Add(delta); @@ -487,7 +499,7 @@ public static class Mp4Inspector covered += n; } report.HasCompositionOffsets = true; - if (covered != count) report.Errors.Add($"ctts covers {covered} of {count} samples"); + if (covered != count) report.Error($"ctts covers {covered} of {count} samples"); return result; } @@ -503,7 +515,7 @@ public static class Mp4Inspector var s = BinaryPrimitives.ReadUInt32BigEndian(p[(8 + i * 4)..]); if (s == 0 || s > count || (list.Count > 0 && s <= list[^1])) { - report.Errors.Add($"stss entry {s} out of order or range"); + report.Error($"stss entry {s} out of order or range"); break; } list.Add(s); @@ -516,42 +528,42 @@ public static class Mp4Inspector var box = stbl.Child("stsz") ?? stbl.Child("stz2"); if (box is null) { - report.Errors.Add("stsz missing"); + report.Error("stsz missing"); return null; } var p = box.Payload.Span; if (p.Length < 12) { - report.Errors.Add($"{box.Type} truncated"); + report.Error($"{box.Type} truncated"); return null; } var count = BinaryPrimitives.ReadUInt32BigEndian(p[8..]); - if (count > 50_000_000) + if (count > MaxSamples) { - report.Errors.Add($"{box.Type} sample count implausible"); + report.Error($"{box.Type} sample count implausible"); + return null; + } + var constant = box.Type == "stsz" ? BinaryPrimitives.ReadUInt32BigEndian(p[4..]) : 0; + if (box.Type == "stsz" && constant == 0 && p.Length < 12 + count * 4L) + { + report.Error("stsz truncated"); return null; } var sizes = new uint[count]; if (box.Type == "stsz") { - var constant = BinaryPrimitives.ReadUInt32BigEndian(p[4..]); if (constant != 0) { Array.Fill(sizes, constant); return sizes; } - if (p.Length < 12 + count * 4L) - { - report.Errors.Add("stsz truncated"); - return null; - } for (var i = 0; i < count; i++) sizes[i] = BinaryPrimitives.ReadUInt32BigEndian(p[(12 + i * 4)..]); return sizes; } var field = p[7]; if (field is not (4 or 8 or 16) || p.Length < 12 + (count * field + 7) / 8) { - report.Errors.Add("stz2 invalid"); + report.Error("stz2 invalid"); return null; } for (var i = 0; i < count; i++) @@ -571,7 +583,7 @@ public static class Mp4Inspector var box = stbl.Child("stco") ?? stbl.Child("co64"); if (box is null) { - report.Errors.Add("stco/co64 missing"); + report.Error("stco/co64 missing"); return null; } var wide = box.Type == "co64"; @@ -593,7 +605,7 @@ public static class Mp4Inspector var box = stbl.Child("stsc"); if (box is null) { - report.Errors.Add("stsc missing"); + report.Error("stsc missing"); return null; } var p = box.Payload.Span; @@ -606,10 +618,10 @@ public static class Mp4Inspector var next = i + 1 < entries ? BinaryPrimitives.ReadUInt32BigEndian(p[(8 + (i + 1) * 12)..]) : (uint)chunks + 1; if (first == 0 || next <= first || (i == 0 && first != 1)) { - report.Errors.Add("stsc first_chunk values invalid"); + report.Error("stsc first_chunk values invalid"); return null; } - if (BinaryPrimitives.ReadUInt32BigEndian(p[(16 + i * 12)..]) != 1) report.Errors.Add("stsc references a sample description other than 1"); + if (BinaryPrimitives.ReadUInt32BigEndian(p[(16 + i * 12)..]) != 1) report.Error("stsc references a sample description other than 1"); for (var c = first; c < next && c <= chunks; c++) perChunk[c - 1] = samples; } return perChunk; @@ -620,7 +632,7 @@ public static class Mp4Inspector segment = mediaTime = 0; var wide = p.Length > 0 && p[0] == 1; if (!Entries(p, wide ? 20 : 12, report, "elst", out var entries) || entries == 0) return false; - if (entries > 1) report.Warnings.Add($"edit list has {entries} entries; only the first is evaluated"); + if (entries > 1) report.Warning($"edit list has {entries} entries; only the first is evaluated"); segment = wide ? (long)Math.Min(BinaryPrimitives.ReadUInt64BigEndian(p[8..]), long.MaxValue) : BinaryPrimitives.ReadUInt32BigEndian(p[8..]); mediaTime = wide ? BinaryPrimitives.ReadInt64BigEndian(p[16..]) : BinaryPrimitives.ReadInt32BigEndian(p[12..]); return true; @@ -632,13 +644,13 @@ public static class Mp4Inspector entries = 0; if (p.Length < 8) { - report.Errors.Add($"{type} truncated"); + report.Error($"{type} truncated"); return false; } var n = BinaryPrimitives.ReadUInt32BigEndian(p[4..]); if (n > (p.Length - 8) / entrySize) { - report.Errors.Add($"{type} declares {n} entries but holds {(p.Length - 8) / entrySize}"); + report.Error($"{type} declares {n} entries but holds {(p.Length - 8) / entrySize}"); return false; } entries = (int)n; diff --git a/tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs b/tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs index d505885b..ea1fd44b 100644 --- a/tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs +++ b/tests/Titano.Core.Tests/Video/Mp4MuxerTests.cs @@ -198,6 +198,22 @@ public sealed class Mp4MuxerTests(TestContext context) Assert.Throws(() => muxer.WriteSample([0, 0, 1, 0x65, 1], 0, true)); } + [Test] + public void ParameterSetOnlyUnitsAddNoSampleAndSeiOnlyUnitsAreRejected() + { + using var ms = new MemoryStream(); + var muxer = new Mp4Muxer(ms, new VideoTrackConfig { Codec = VideoCodec.H264, Width = 1920, Height = 1080 }); + muxer.WriteSample(AnnexBParser.ToAnnexB([SyntheticStreams.H264Sps(), SyntheticStreams.H264Pps()]), 3000, true); + Assert.Equal(0, muxer.SampleCount); + Assert.Throws(() => muxer.WriteSample(AnnexBParser.ToAnnexB([[0x06, 0x05, 0x01, 0x00, 0x80]]), 3000, false)); + muxer.WriteSample(AnnexBParser.ToAnnexB([SyntheticStreams.H264Slice(true, 64, 1)]), 3000, true); + muxer.Complete(); + var report = Mp4Inspector.Inspect(ms); + AssertClean(report); + Assert.Equal(1, report.FrameCount); + Assert.Equal(0, report.Warnings.Count); + } + [Test] public void InspectorSurvivesTruncationAndCorruption() {