using System.Runtime.InteropServices; using System.Runtime.InteropServices.Marshalling; using Chronos.Core.Diagnostics; using Chronos.Core.Imaging; using Chronos.Core.IO; using Chronos.Core.Video; using Chronos.Core.Video.Bitstream; using Chronos.Core.Video.Mp4; using Chronos.Core.Video.Yuv; using Chronos.Core.Windows.Interop; namespace Chronos.Core.Windows.MediaFoundation; /// /// One encode through a Media Foundation encoder MFT driven directly (no SinkWriter): output type first, then NV12/P010 /// input, ICodecAPI rate control/GOP/B-frames, synchronous ProcessInput/ProcessOutput or the asynchronous event loop of /// hardware MFTs, and Annex-B/OBU output muxed by into a temporary file that /// commits. Disposing without completing deletes the partial file. /// public sealed class MediaFoundationEncoderSession : IVideoEncoderSession { private const string LogCategory = MediaFoundationEncoders.LogCategory; private readonly EncodeSettings _settings; private readonly IDisposable _runtime; private readonly IMFActivate _activate; private readonly IMFTransform _transform; private readonly IMFMediaEventGenerator? _events; private readonly bool _tenBit; private readonly string _tempPath; private readonly FileStream _file; private readonly Mp4Muxer _muxer; private readonly Dictionary _ticksByHns = []; private readonly Dictionary _durationByTicks = []; private readonly Queue _inputStarts = new(); private readonly List _ignored = []; private bool _providesSamples; private uint _outputBufferSize; private int _needInput; private bool _drained; private long _time; private Encoded? _pending; private int _framesIn; private int _framesOut; private bool _completed; private bool _released; private bool _disposed; private MediaFoundationEncoderSession(EncodeSettings settings, MediaFoundationEncoderInfo info, IMFActivate activate, IMFTransform transform, IDisposable runtime) { _settings = settings; _activate = activate; _transform = transform; _runtime = runtime; _tenBit = settings.BitDepth == 10; EncoderName = info.Name; IsHardware = info.IsHardware; if (UnlockAsync(transform)) _events = transform as IMFMediaEventGenerator ?? throw new NotSupportedException("Asynchronous MFT without an event generator"); if (transform as ICodecAPI is { } codecApi) ApplyCodecSettings(codecApi); else _ignored.AddRange(Asked()); SetOutputType(); SetInputType(); ReadOutputStreamInfo(); Message(MfNative.MessageNotifyBeginStreaming); Message(MfNative.MessageNotifyStartOfStream); _tempPath = AtomicFile.TempPathFor(settings.OutputPath); _file = new FileStream(_tempPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 1 << 20); try { _muxer = new Mp4Muxer(_file, new VideoTrackConfig { Codec = settings.Codec, Width = settings.Width, Height = settings.Height, FastStart = true }); ReadSequenceHeader(); } catch { _file.Dispose(); File.Delete(_tempPath); throw; } Log.Info(LogCategory, $"Encoding {settings.Codec} {settings.Width}×{settings.Height} {settings.BitDepth}-bit with \"{info.Name}\" ({(info.IsHardware ? "hardware" : "software")}, {(_events is null ? "sync" : "async")})"); } public string EncoderName { get; } public bool IsHardware { get; } /// Activates the MFT behind and configures it; the session then owns the activate and the runtime lease. internal static MediaFoundationEncoderSession Open(EncodeSettings settings, MediaFoundationEncoderInfo info, IMFActivate activate, IDisposable runtime) { ComObjects.Check(activate.ActivateObject(MfGuids.IidTransform, out var pointer), "IMFActivate.ActivateObject"); var transform = ComObjects.Attach(pointer); try { return new MediaFoundationEncoderSession(settings, info, activate, transform, runtime); } catch { ComObjects.Release(transform); throw; } } public void WriteFrame(ImageBuffer frame, long duration90k) { Interop.Win32Com.EnsureInitialized(); ObjectDisposedException.ThrowIf(_disposed, this); if (_completed) throw new InvalidOperationException("The session is complete"); if (frame.Width != _settings.Width || frame.Height != _settings.Height) throw new ArgumentException("Frame size differs from the encode size", nameof(frame)); ArgumentOutOfRangeException.ThrowIfNegativeOrZero(duration90k); var start = _time; _time += duration90k; var hns = ToHns(start); _ticksByHns[hns] = start; _durationByTicks[start] = duration90k; _inputStarts.Enqueue(start); var sample = CreateInputSample(frame, hns, ToHns(_time) - hns); try { if (_events is not null) { Pump(() => _needInput > 0); _needInput--; ComObjects.Check(_transform.ProcessInput(0, sample, 0), "ProcessInput"); } else { SubmitSync(sample); } } finally { ComObjects.Release(sample); } _framesIn++; } public void Complete() { Interop.Win32Com.EnsureInitialized(); ObjectDisposedException.ThrowIf(_disposed, this); if (_completed) throw new InvalidOperationException("The session is already complete"); if (_framesIn == 0) throw new InvalidOperationException("No frames were written"); Message(MfNative.MessageNotifyEndOfStream); Message(MfNative.MessageCommandDrain); if (_events is not null) Pump(() => _drained); else while (ProcessOutput()) { } if (_pending is { } last) { // The last decoded picture keeps the duration of its own presentation slot. WritePending(last, _durationByTicks.GetValueOrDefault(last.Pts, 1)); _pending = null; } Message(MfNative.MessageNotifyEndStreaming); ReleaseEncoder(); _muxer.Complete(); _file.Flush(flushToDisk: true); _file.Dispose(); // The render picked a free name: something that took it meanwhile (a restore) is never overwritten. AtomicFile.CommitNew(_tempPath, _settings.OutputPath); _completed = true; Log.Info(LogCategory, $"\"{EncoderName}\" encoded {_framesIn} frames into {_framesOut} samples: {_settings.OutputPath}"); } public void Dispose() { if (_disposed) return; _disposed = true; if (!_completed) { _file.Dispose(); try { File.Delete(_tempPath); } catch (IOException ex) { Log.Warning(LogCategory, "Could not delete the partial output " + _tempPath, ex); } } ReleaseEncoder(); _runtime.Dispose(); } private sealed record Encoded(byte[] Data, long Pts, long Dts, bool Key); private static bool UnlockAsync(IMFTransform transform) { if (transform.GetAttributes(out var pointer) < 0 || pointer == 0) return false; var attributes = ComObjects.Attach(pointer); try { if (attributes.GetUINT32(MfGuids.TransformAsync, out var isAsync) < 0 || isAsync == 0) return false; ComObjects.Check(attributes.SetUINT32(MfGuids.TransformAsyncUnlock, 1), "MF_TRANSFORM_ASYNC_UNLOCK"); return true; } finally { ComObjects.Release(attributes); } } private void ApplyCodecSettings(ICodecAPI api) { var mode = _settings.RateControl switch { RateControl.Cbr => MfNative.RateControlCbr, RateControl.Quality => MfNative.RateControlQuality, _ => MfNative.RateControlUnconstrainedVbr, }; var asked = Asked().ToHashSet(); void Set(in Guid property, ComVariant value, string name, EncoderSetting setting) { if (!SetCodecValue(api, property, value, name) && asked.Contains(setting) && !_ignored.Contains(setting)) _ignored.Add(setting); } Set(MfGuids.CodecRateControlMode, ComVariant.Create(mode), "RateControlMode", EncoderSetting.RateControl); if (_settings.RateControl == RateControl.Quality) Set(MfGuids.CodecQuality, ComVariant.Create((uint)Math.Clamp(_settings.Quality, 1, 100)), "Quality", EncoderSetting.Quality); else Set(MfGuids.CodecMeanBitRate, ComVariant.Create(BitRate), "MeanBitRate", EncoderSetting.BitRate); var gop = _settings.GopLength > 0 ? _settings.GopLength : (int)Math.Max(1, Math.Round(_settings.FrameRate.Value)); Set(MfGuids.CodecGopSize, ComVariant.Create((uint)gop), "GOPSize", EncoderSetting.KeyframeInterval); var bFrames = _settings.Profile == CodecProfile.H264Baseline ? 0 : Math.Max(0, _settings.BFrames); Set(MfGuids.CodecBPictureCount, ComVariant.Create((uint)bFrames), "BPictureCount", EncoderSetting.BFrames); SetCodecValue(api, MfGuids.CodecLowLatencyMode, ComVariant.Create(false), "LowLatencyMode"); } /// public IReadOnlyList Ignored => _ignored; /// /// The settings that name a value of their own rather than the encoder's default, whose refusal the render reports: a rate /// control other than VBR, its quality, a bit rate given, a keyframe interval given, a B-frame count other than the default. /// private IEnumerable Asked() { if (_settings.RateControl != RateControl.Vbr) yield return EncoderSetting.RateControl; if (_settings.RateControl == RateControl.Quality) yield return EncoderSetting.Quality; else if (_settings.BitRate > 0) yield return EncoderSetting.BitRate; if (_settings.GopLength > 0) yield return EncoderSetting.KeyframeInterval; if (_settings.Profile != CodecProfile.H264Baseline && _settings.BFrames != EncodeSettings.DefaultBFrames) yield return EncoderSetting.BFrames; } /// Sets an ICodecAPI property; false (logged) when the encoder does not support it or refuses the value. private static unsafe bool SetCodecValue(ICodecAPI api, in Guid property, ComVariant value, string name) { if (api.IsSupported(property) != 0) { Log.Debug(LogCategory, $"ICodecAPI {name} not supported by the encoder"); return false; } var hr = api.SetValue(property, (nint)(&value)); if (hr < 0) Log.Debug(LogCategory, $"ICodecAPI {name} rejected (0x{hr:X8})"); return hr >= 0; } /// Requested bit rate, or the preset density (§11.4: 0.10/0.07/0.05 bits per pixel per frame) when unset. private uint BitRate { get { if (_settings.BitRate > 0) return (uint)Math.Min(uint.MaxValue, _settings.BitRate); var bpp = _settings.Codec switch { VideoCodec.H264 => 0.10, VideoCodec.Hevc => 0.07, _ => 0.05 }; return (uint)Math.Min(uint.MaxValue, _settings.Width * (double)_settings.Height * _settings.FrameRate.Value * bpp); } } private void SetOutputType() { var level = CodecLimits.Check(_settings.Codec, _settings.Width, _settings.Height, _settings.FrameRate.Value, BitRate, _settings.Profile).Level; var hr = TrySetOutputType(level); if (hr < 0 && level is not null) { Log.Debug(LogCategory, $"Output type with level {level} rejected (0x{hr:X8}); letting the encoder choose"); hr = TrySetOutputType(null); } ComObjects.Check(hr, "SetOutputType"); } private int TrySetOutputType(CodecLevel? level) { var type = CreateMediaType(); try { ComObjects.Check(type.SetGUID(MfGuids.Subtype, MediaFoundationEncoders.Subtype(_settings.Codec)), "MF_MT_SUBTYPE"); ComObjects.Check(type.SetUINT32(MfGuids.AverageBitrate, BitRate), "MF_MT_AVG_BITRATE"); SetVideoFormat(type); var profile = Profile(); if (profile > 0) ComObjects.Check(type.SetUINT32(MfGuids.Mpeg2Profile, profile), "MF_MT_MPEG2_PROFILE"); if (level is { } l && _settings.Codec != VideoCodec.Av1) { var idc = _settings.Codec == VideoCodec.H264 ? l.H264LevelIdc : l.HevcLevelIdc; ComObjects.Check(type.SetUINT32(MfGuids.Mpeg2Level, (uint)idc), "MF_MT_MPEG2_LEVEL"); } return _transform.SetOutputType(0, type, 0); } finally { ComObjects.Release(type); } } /// eAVEncH264VProfile / eAVEncH265VProfile value; AV1 encoders derive the profile from the input format. private uint Profile() => _settings.Codec switch { VideoCodec.H264 => _settings.Profile switch { CodecProfile.H264Baseline => 66u, CodecProfile.H264Main => 77u, _ => 100u, }, VideoCodec.Hevc => _tenBit ? 2u : 1u, _ => 0u, }; private void SetInputType() { var subtype = _tenBit ? MfGuids.FormatP010 : MfGuids.FormatNv12; var type = CreateMediaType(); int hr; try { ComObjects.Check(type.SetGUID(MfGuids.Subtype, subtype), "MF_MT_SUBTYPE"); ComObjects.Check(type.SetUINT32(MfGuids.DefaultStride, (uint)(_settings.Width * (_tenBit ? 2 : 1))), "MF_MT_DEFAULT_STRIDE"); SetVideoFormat(type); hr = _transform.SetInputType(0, type, 0); } finally { ComObjects.Release(type); } // Some encoders only accept their own enumerated input types: complete one with our format. for (uint index = 0; hr < 0; index++) { var available = _transform.GetInputAvailableType(0, index, out var pointer); if (available < 0) break; var candidate = ComObjects.Attach(pointer); try { if (candidate.GetGUID(MfGuids.Subtype, out var s) < 0 || s != subtype) continue; SetVideoFormat(candidate); hr = _transform.SetInputType(0, candidate, 0); } finally { ComObjects.Release(candidate); } } ComObjects.Check(hr, $"SetInputType ({(_tenBit ? "P010" : "NV12")})"); } private static IMFMediaType CreateMediaType() { ComObjects.Check(MfNative.MFCreateMediaType(out var pointer), "MFCreateMediaType"); var type = ComObjects.Attach(pointer); ComObjects.Check(type.SetGUID(MfGuids.MajorType, MfGuids.MediaTypeVideo), "MF_MT_MAJOR_TYPE"); return type; } /// Frame size and rate, square pixels, progressive, BT.709 limited range. private void SetVideoFormat(IMFMediaType type) { ComObjects.Check(type.SetUINT64(MfGuids.FrameSize, Pack(_settings.Width, _settings.Height)), "MF_MT_FRAME_SIZE"); ComObjects.Check(type.SetUINT64(MfGuids.FrameRate, Pack(_settings.FrameRate.Numerator, _settings.FrameRate.Denominator)), "MF_MT_FRAME_RATE"); ComObjects.Check(type.SetUINT64(MfGuids.PixelAspectRatio, Pack(1, 1)), "MF_MT_PIXEL_ASPECT_RATIO"); ComObjects.Check(type.SetUINT32(MfGuids.InterlaceMode, MfNative.InterlaceProgressive), "MF_MT_INTERLACE_MODE"); type.SetUINT32(MfGuids.NominalRange, MfNative.NominalRange16To235); type.SetUINT32(MfGuids.VideoPrimaries, MfNative.PrimariesBt709); type.SetUINT32(MfGuids.TransferFunction, MfNative.TransferBt709); type.SetUINT32(MfGuids.YuvMatrix, MfNative.MatrixBt709); } private static ulong Pack(int high, int low) => (ulong)(uint)high << 32 | (uint)low; private void ReadOutputStreamInfo() { ComObjects.Check(_transform.GetOutputStreamInfo(0, out var info), "GetOutputStreamInfo"); _providesSamples = (info.Flags & (MfNative.OutputStreamProvidesSamples | MfNative.OutputStreamCanProvideSamples)) != 0; _outputBufferSize = Math.Max(info.Size, (uint)(_settings.Width * _settings.Height * 3 / 2)); } /// Hands MF_MT_MPEG_SEQUENCE_HEADER (parameter sets / sequence header OBU) of the current output type to the muxer. private void ReadSequenceHeader() { if (_transform.GetOutputCurrentType(0, out var pointer) < 0) return; var type = ComObjects.Attach(pointer); try { if (type.GetAllocatedBlob(MfGuids.MpegSequenceHeader, out var blob, out var size) < 0) return; try { var bytes = new byte[size]; Marshal.Copy(blob, bytes, 0, (int)size); _muxer.AddParameterSets(bytes); } finally { Marshal.FreeCoTaskMem(blob); } } finally { ComObjects.Release(type); } } private unsafe IMFSample CreateInputSample(ImageBuffer frame, long time, long duration) { var fourCc = _tenBit ? 0x30313050u : 0x3231564Eu; // 'P010' / 'NV12' ComObjects.Check(MfNative.MFCreate2DMediaBuffer((uint)_settings.Width, (uint)_settings.Height, fourCc, false, out var bufferPointer), "MFCreate2DMediaBuffer"); var buffer = ComObjects.Attach(bufferPointer); try { var buffer2D = buffer as IMF2DBuffer ?? throw new NotSupportedException("Media buffer without IMF2DBuffer"); ComObjects.Check(buffer2D.Lock2D(out var scanline0, out var pitch), "IMF2DBuffer.Lock2D"); try { if (pitch <= 0) throw new NotSupportedException("Bottom-up input buffers are not supported"); var rows = _settings.Height * 3 / 2; if (_tenBit) Nv12Converter.ToP010(frame, new Span((void*)scanline0, pitch / 2 * rows), ChromaSiting.Left, pitch / 2); else Nv12Converter.ToNv12(frame, new Span((void*)scanline0, pitch * rows), ChromaSiting.Left, pitch); } finally { buffer2D.Unlock2D(); } // 2D buffers report their contiguous size; setting it explicitly helps encoders that read the plain length. if (buffer2D.GetContiguousLength(out var length) >= 0 && buffer.SetCurrentLength(length) < 0) Log.Debug(LogCategory, "SetCurrentLength on the 2D input buffer was rejected"); ComObjects.Check(MfNative.MFCreateSample(out var samplePointer), "MFCreateSample"); var sample = ComObjects.Attach(samplePointer); try { ComObjects.Check(sample.AddBuffer(buffer), "AddBuffer"); ComObjects.Check(sample.SetSampleTime(time), "SetSampleTime"); ComObjects.Check(sample.SetSampleDuration(duration), "SetSampleDuration"); return sample; } catch { ComObjects.Release(sample); throw; } } finally { ComObjects.Release(buffer); } } private void SubmitSync(IMFSample sample) { while (true) { var hr = _transform.ProcessInput(0, sample, 0); if (hr != MfNative.NotAccepting) { ComObjects.Check(hr, "ProcessInput"); break; } // Full: collect output first; an encoder that neither accepts input nor yields output is stuck. if (!ProcessOutput()) throw new COMException("Encoder refuses input without producing output", hr); while (ProcessOutput()) { } } while (ProcessOutput()) { } } /// Runs the asynchronous MFT event loop until holds. private void Pump(Func done) { while (!done()) { ComObjects.Check(_events!.GetEvent(0, out var pointer), "IMFMediaEventGenerator.GetEvent"); var mediaEvent = ComObjects.Attach(pointer); uint type; int status; try { ComObjects.Check(mediaEvent.GetEventType(out type), "IMFMediaEvent.GetType"); ComObjects.Check(mediaEvent.GetStatus(out status), "IMFMediaEvent.GetStatus"); } finally { ComObjects.Release(mediaEvent); } if (status < 0 || type == MfNative.EventError) throw new COMException($"Encoder reported event {type} with failure", status); switch (type) { case MfNative.EventNeedInput: _needInput++; break; case MfNative.EventHaveOutput: ProcessOutput(); break; case MfNative.EventDrainComplete: _drained = true; break; } } } /// Pulls one output sample; false when the encoder needs more input. private bool ProcessOutput() { IMFSample? own = null; nint ownPointer = 0; var data = new MftOutputDataBuffer(); if (!_providesSamples) { own = CreateOutputSample(); data.Sample = ownPointer = ComObjects.GetPointer(own); } int hr; try { hr = _transform.ProcessOutput(0, 1, ref data, out _); } finally { if (data.Events != 0) Marshal.Release(data.Events); if (ownPointer != 0) Marshal.Release(ownPointer); } if (hr < 0) { ComObjects.Release(own); if (hr == MfNative.TransformNeedMoreInput) return false; if (hr == MfNative.TransformStreamChange) { Renegotiate(); return true; } ComObjects.Check(hr, "ProcessOutput"); } var output = own ?? ComObjects.AttachOrNull(data.Sample); if (output is null) return true; try { Deliver(output); } finally { ComObjects.Release(output); } return true; } private IMFSample CreateOutputSample() { ComObjects.Check(MfNative.MFCreateMemoryBuffer(_outputBufferSize, out var bufferPointer), "MFCreateMemoryBuffer"); var buffer = ComObjects.Attach(bufferPointer); try { ComObjects.Check(MfNative.MFCreateSample(out var samplePointer), "MFCreateSample"); var sample = ComObjects.Attach(samplePointer); ComObjects.Check(sample.AddBuffer(buffer), "AddBuffer"); return sample; } finally { ComObjects.Release(buffer); } } /// MF_E_TRANSFORM_STREAM_CHANGE: adopt the encoder's new output type and its sequence header. private void Renegotiate() { ComObjects.Check(_transform.GetOutputAvailableType(0, 0, out var pointer), "GetOutputAvailableType"); var type = ComObjects.Attach(pointer); try { ComObjects.Check(_transform.SetOutputType(0, type, 0), "SetOutputType (stream change)"); } finally { ComObjects.Release(type); } ReadOutputStreamInfo(); ReadSequenceHeader(); Log.Debug(LogCategory, $"\"{EncoderName}\" renegotiated its output type"); } private void Deliver(IMFSample sample) { ComObjects.Check(sample.GetSampleTime(out var pts), "GetSampleTime"); bool? cleanPoint = sample.GetUINT32(MfGuids.SampleCleanPoint, out var clean) >= 0 ? clean != 0 : null; long? dts = sample.GetUINT64(MfGuids.SampleDecodeTimestamp, out var decode) >= 0 ? (long)decode : null; ComObjects.Check(sample.ConvertToContiguousBuffer(out var bufferPointer), "ConvertToContiguousBuffer"); var buffer = ComObjects.Attach(bufferPointer); byte[] bytes; try { ComObjects.Check(buffer.Lock(out var data, out _, out var length), "IMFMediaBuffer.Lock"); try { bytes = new byte[length]; Marshal.Copy(data, bytes, 0, (int)length); } finally { buffer.Unlock(); } } finally { ComObjects.Release(buffer); } if (bytes.Length == 0) return; var key = cleanPoint ?? IsRandomAccess(bytes); var ptsTicks = _ticksByHns.Remove(pts, out var exact) ? exact : To90k(pts); // Without MFSampleExtension_DecodeTimestamp the n-th output decodes at the n-th input's time. _inputStarts.TryDequeue(out var inputStart); var dtsTicks = dts is { } d ? To90k(d) : inputStart; var encoded = new Encoded(bytes, ptsTicks, dtsTicks, key); if (_pending is { } previous) WritePending(previous, Math.Max(1, encoded.Dts - previous.Dts)); _pending = encoded; _framesOut++; } /// Sync detection for encoders that do not set MFSampleExtension_CleanPoint: IDR/IRAP NAL units or an AV1 sequence header. private bool IsRandomAccess(byte[] data) { if (_settings.Codec == VideoCodec.Av1) { foreach (var obu in Av1ObuParser.Split(data)) { if (obu.Type == Av1ObuType.SequenceHeader) return true; } return false; } foreach (var unit in AnnexBParser.SplitAccessUnit(data, out _)) { var header = data[unit.Offset]; if (_settings.Codec == VideoCodec.H264 ? AnnexBParser.H264Type(header) == H264NalUnitType.IdrSlice : AnnexBParser.HevcType(header) is >= HevcNalUnitType.BlaWLp and <= HevcNalUnitType.Cra) return true; } return false; } private void WritePending(Encoded sample, long duration) { _muxer.WriteSample(sample.Data, duration, sample.Key, sample.Pts - sample.Dts); _durationByTicks.Remove(sample.Pts); } private void Message(uint message) => ComObjects.Check(_transform.ProcessMessage(message, 0), $"ProcessMessage(0x{message:X})"); private void ReleaseEncoder() { if (_released) return; _released = true; ComObjects.Release(_transform); _activate.ShutdownObject(); ComObjects.Release(_activate); } private static long ToHns(long ticks90k) => (ticks90k * 1000 + 4) / 9; private static long To90k(long hns) => (long)Math.Round(hns * 9 / 1000.0); }