MF SinkWriter mp4文件的播放持续时间是添加音频样本时的一半，图像的播放速度也是添加音频样本的两倍

Question

我为我的c#项目创建了一个托管c++库，以将图像和音频编码到基于MSDN教程SinkWriter的mp4容器中。为了测试结果是否正确，我创建了一个提供600帧的方法。该帧表示每秒60帧的10秒视频。

我提供的图像每秒都在变化，我的音频文件中包含一个计数为10的语音。

我面临的问题是，输出的视频实际上只有5秒长。视频的元数据显示它是10秒，但不是。此外，声音几乎不到5。

如果我只写没有音频部分的图像样本，那么视频的持续时间是预期的10秒。

我在这里错过了什么？

以下是我的应用程序的一些部分

这是我用来创建600帧的c#部分，然后我也在c#部分中调用PushFrame方法。

var videoFrameCount = 10 * FPS;
SetBinaryImage();
for (int i = 0; i <= videoFrameCount; i++)
{
// New picture every second
if (i > 0 &&  i % FPS == 0)
{
SetBinaryImage();
}
PushFrame();
}

PushFrame方法将图像和音频数据复制到SinkWriter提供的指针。然后我调用SinkWriter的PushFrame方法。

private void PushFrame()
{
try
{
encodeStopwatch.Reset();
encodeStopwatch.Start();
// Video
var frameBufferHandler = GCHandle.Alloc(frameBuffer, GCHandleType.Pinned);
frameBufferPtr = frameBufferHandler.AddrOfPinnedObject();
CopyImageDataToPointer(BinaryImage, ScreenWidth, ScreenHeight, frameBufferPtr);
// Audio
var audioBufferHandler = GCHandle.Alloc(audioBuffer, GCHandleType.Pinned);
audioBufferPtr = audioBufferHandler.AddrOfPinnedObject();
var readLength = audioBuffer.Length;
if (BinaryAudio.Length - (audioOffset + audioBuffer.Length) < 0)
{
readLength = BinaryAudio.Length - audioOffset;
}
if (!EndOfFile)
{
Marshal.Copy(BinaryAudio, audioOffset, (IntPtr)audioBufferPtr, readLength);
audioOffset += audioBuffer.Length;
}
if (readLength < audioBuffer.Length && !EndOfFile)
{
EndOfFile = true;
}
unsafe
{
// Copy video data
var yuv = SinkWriter.VideoCapturerBuffer();
SinkWriter.Encode((byte*)frameBufferPtr, ScreenWidth, ScreenHeight, (int)SWPF.SWPF_RGB, yuv);
// Copy audio data
var audioDestPtr = SinkWriter.AudioCapturerBuffer();
SinkWriter.EncodeAudio((byte*)audioBufferPtr, audioDestPtr);
SinkWriter.PushFrame();
}
encodeStopwatch.Stop();
Console.WriteLine($"YUV frame generated in: {encodeStopwatch.TakeTotalMilliseconds()} ms");
}
catch (Exception ex)
{
}
}

以下是我在c++中添加到SinkWriter的一些部分。我想音频部分的MediaTypes是可以的，因为音频的播放是有效的。

rtStart和rtDuration的定义如下：

LONGLONG rtStart = 0;
UINT64 rtDuration;
MFFrameRateToAverageTimePerFrame(fps, 1, &rtDuration);

编码器的两个缓冲区的使用方式与类似

int SinkWriter::Encode(Byte * rgbBuf, int w, int h, int pxFormat, Byte * yufBuf)
{
const LONG cbWidth = 4 * VIDEO_WIDTH;
const DWORD cbBuffer = cbWidth * VIDEO_HEIGHT;
// Create a new memory buffer.
HRESULT hr = MFCreateMemoryBuffer(cbBuffer, &pFrameBuffer);
// Lock the buffer and copy the video frame to the buffer.
if (SUCCEEDED(hr))
{
hr = pFrameBuffer->Lock(&yufBuf, NULL, NULL);
}
if (SUCCEEDED(hr))
{
// Calculate the stride
DWORD bitsPerPixel = GetBitsPerPixel(pxFormat);
DWORD bytesPerPixel = bitsPerPixel / 8;
DWORD stride = w * bytesPerPixel;
// Copy image in yuv pointer
hr = MFCopyImage(
yufBuf,                      // Destination buffer.
stride,                    // Destination stride.
rgbBuf,     // First row in source image.
stride,                    // Source stride.
stride,                    // Image width in bytes.
h                // Image height in pixels.
);
}
if (pFrameBuffer)
{
pFrameBuffer->Unlock();
}
// Set the data length of the buffer.
if (SUCCEEDED(hr))
{
hr = pFrameBuffer->SetCurrentLength(cbBuffer);
}
if (SUCCEEDED(hr))
{
return 0;
}
else
{
return -1;
}
return 0;
}
int SinkWriter::EncodeAudio(Byte * src, Byte * dest)
{
DWORD samplePerSecond = AUDIO_SAMPLES_PER_SECOND * AUDIO_BITS_PER_SAMPLE * AUDIO_NUM_CHANNELS;
DWORD cbBuffer = samplePerSecond / 1000;
// Create a new memory buffer.
HRESULT hr = MFCreateMemoryBuffer(cbBuffer, &pAudioBuffer);
// Lock the buffer and copy the video frame to the buffer.
if (SUCCEEDED(hr))
{
hr = pAudioBuffer->Lock(&dest, NULL, NULL);
}
CopyMemory(dest, src, cbBuffer);
if (pAudioBuffer)
{
pAudioBuffer->Unlock();
}
// Set the data length of the buffer.
if (SUCCEEDED(hr))
{
hr = pAudioBuffer->SetCurrentLength(cbBuffer);
}
if (SUCCEEDED(hr))
{
return 0;
}
else
{
return -1;
}
return 0;
}

这是SinkWriter的PushFrame方法，它将SinkWriter、streamIndex、audioIndex、rtStart和rtDuration传递给WriteFrame方法。

int SinkWriter::PushFrame()
{
if (initialized)
{
HRESULT hr = WriteFrame(ptrSinkWriter, stream, audio, rtStart, rtDuration);
if (FAILED(hr))
{
return -1;
}
rtStart += rtDuration;
return 0;
}
return -1;
}

这里是WriteFrame方法，它结合了视频和音频样本。

HRESULT SinkWriter::WriteFrame(IMFSinkWriter *pWriter, DWORD streamIndex, DWORD audioStreamIndex, const LONGLONG& rtStart, const LONGLONG& rtDuration)
{
IMFSample *pVideoSample = NULL;
// Create a media sample and add the buffer to the sample.
HRESULT hr = MFCreateSample(&pVideoSample);
if (SUCCEEDED(hr))
{
hr = pVideoSample->AddBuffer(pFrameBuffer);
}
if (SUCCEEDED(hr))
{
pVideoSample->SetUINT32(MFSampleExtension_Discontinuity, FALSE);
}
// Set the time stamp and the duration.
if (SUCCEEDED(hr))
{
hr = pVideoSample->SetSampleTime(rtStart);
}
if (SUCCEEDED(hr))
{
hr = pVideoSample->SetSampleDuration(rtDuration);
}
// Send the sample to the Sink Writer.
if (SUCCEEDED(hr))
{
hr = pWriter->WriteSample(streamIndex, pVideoSample);
}
// Audio
IMFSample *pAudioSample = NULL;
if (SUCCEEDED(hr))
{
hr = MFCreateSample(&pAudioSample);
}
if (SUCCEEDED(hr))
{
hr = pAudioSample->AddBuffer(pAudioBuffer);
}
// Set the time stamp and the duration.
if (SUCCEEDED(hr))
{
hr = pAudioSample->SetSampleTime(rtStart);
}
if (SUCCEEDED(hr))
{
hr = pAudioSample->SetSampleDuration(rtDuration);
}
// Send the sample to the Sink Writer.
if (SUCCEEDED(hr))
{
hr = pWriter->WriteSample(audioStreamIndex, pAudioSample);
}

SafeRelease(&pVideoSample);
SafeRelease(&pFrameBuffer);
SafeRelease(&pAudioSample);
SafeRelease(&pAudioBuffer);
return hr;
}

Answer 1

问题是音频缓冲区大小的计算错误。这是正确的计算：

var avgBytesPerSecond = sampleRate * 2 * channels;
var avgBytesPerMillisecond = avgBytesPerSecond / 1000;
var bufferSize = avgBytesPerMillisecond * (1000 / 60);
audioBuffer = new byte[bufferSize];

在我的问题中，我有一毫秒的缓冲区大小。所以看起来MF框架加快了图像的速度，所以音频听起来很好。在我修复了缓冲区大小后，视频的持续时间正好符合我的预期，声音也没有错误。