Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include "FFmpegDataEncoder.h"
#include "PlatformEncoderModule.h"
#include <utility>
#include "FFmpegLog.h"
#include "libavutil/error.h"
#include "mozilla/StaticMutex.h"
#include "FFmpegUtils.h"
namespace mozilla {
template <>
AVCodecID GetFFmpegEncoderCodecId<LIBAV_VER>(CodecType aCodec) {
#if LIBAVCODEC_VERSION_MAJOR >= 58
if (aCodec == CodecType::VP8) {
return AV_CODEC_ID_VP8;
}
if (aCodec == CodecType::VP9) {
return AV_CODEC_ID_VP9;
}
if (aCodec == CodecType::H264) {
return AV_CODEC_ID_H264;
}
if (aCodec == CodecType::AV1) {
return AV_CODEC_ID_AV1;
}
if (aCodec == CodecType::Opus) {
return AV_CODEC_ID_OPUS;
}
if (aCodec == CodecType::Vorbis) {
return AV_CODEC_ID_VORBIS;
}
#endif
return AV_CODEC_ID_NONE;
}
/* static */
AVCodec* FFmpegDataEncoder<LIBAV_VER>::FindEncoderWithPreference(
const FFmpegLibWrapper* aLib, AVCodecID aCodecId) {
MOZ_ASSERT(aLib);
// Prioritize libx264 for now since it's the only h264 codec we tested. Once
// libopenh264 is supported, we can simply use `avcodec_find_encoder` and
// rename this function.
if (aCodecId == AV_CODEC_ID_H264) {
AVCodec* codec = aLib->avcodec_find_encoder_by_name("libx264");
if (codec) {
FFMPEGV_LOG("Prefer libx264 for h264 codec");
return codec;
}
FFMPEGV_LOG("Fallback to other h264 library. Fingers crossed");
}
return aLib->avcodec_find_encoder(aCodecId);
}
StaticMutex FFmpegDataEncoder<LIBAV_VER>::sMutex;
FFmpegDataEncoder<LIBAV_VER>::FFmpegDataEncoder(
const FFmpegLibWrapper* aLib, AVCodecID aCodecID,
const RefPtr<TaskQueue>& aTaskQueue, const EncoderConfig& aConfig)
: mLib(aLib),
mCodecID(aCodecID),
mTaskQueue(aTaskQueue),
mConfig(aConfig),
mCodecName(EmptyCString()),
mCodecContext(nullptr),
mFrame(nullptr),
mVideoCodec(IsVideoCodec(aCodecID)) {
MOZ_ASSERT(mLib);
MOZ_ASSERT(mTaskQueue);
#if LIBAVCODEC_VERSION_MAJOR < 58
MOZ_CRASH("FFmpegDataEncoder needs ffmpeg 58 at least.");
#endif
};
RefPtr<MediaDataEncoder::InitPromise> FFmpegDataEncoder<LIBAV_VER>::Init() {
FFMPEG_LOG("Init");
return InvokeAsync(mTaskQueue, this, __func__,
&FFmpegDataEncoder::ProcessInit);
}
RefPtr<MediaDataEncoder::EncodePromise> FFmpegDataEncoder<LIBAV_VER>::Encode(
const MediaData* aSample) {
MOZ_ASSERT(aSample != nullptr);
FFMPEG_LOG("Encode");
return InvokeAsync(mTaskQueue, __func__,
[self = RefPtr<FFmpegDataEncoder<LIBAV_VER>>(this),
sample = RefPtr<const MediaData>(aSample)]() {
return self->ProcessEncode(sample);
});
}
RefPtr<MediaDataEncoder::ReconfigurationPromise>
FFmpegDataEncoder<LIBAV_VER>::Reconfigure(
const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) {
return InvokeAsync(mTaskQueue, this, __func__,
&FFmpegDataEncoder<LIBAV_VER>::ProcessReconfigure,
aConfigurationChanges);
}
RefPtr<MediaDataEncoder::EncodePromise> FFmpegDataEncoder<LIBAV_VER>::Drain() {
FFMPEG_LOG("Drain");
return InvokeAsync(mTaskQueue, this, __func__,
&FFmpegDataEncoder::ProcessDrain);
}
RefPtr<ShutdownPromise> FFmpegDataEncoder<LIBAV_VER>::Shutdown() {
FFMPEG_LOG("Shutdown");
return InvokeAsync(mTaskQueue, this, __func__,
&FFmpegDataEncoder::ProcessShutdown);
}
RefPtr<GenericPromise> FFmpegDataEncoder<LIBAV_VER>::SetBitrate(
uint32_t aBitrate) {
FFMPEG_LOG("SetBitrate");
return GenericPromise::CreateAndReject(NS_ERROR_NOT_IMPLEMENTED, __func__);
}
RefPtr<MediaDataEncoder::InitPromise>
FFmpegDataEncoder<LIBAV_VER>::ProcessInit() {
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
FFMPEG_LOG("ProcessInit");
nsresult rv = InitSpecific();
return NS_FAILED(rv) ? InitPromise::CreateAndReject(rv, __func__)
: InitPromise::CreateAndResolve(true, __func__);
}
RefPtr<MediaDataEncoder::EncodePromise>
FFmpegDataEncoder<LIBAV_VER>::ProcessEncode(RefPtr<const MediaData> aSample) {
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
FFMPEG_LOG("ProcessEncode");
#if LIBAVCODEC_VERSION_MAJOR < 58
MOZ_CRASH("FFmpegDataEncoder needs ffmpeg 58 at least.");
return EncodePromise::CreateAndReject(NS_ERROR_NOT_IMPLEMENTED, __func__);
#else
auto rv = EncodeInputWithModernAPIs(std::move(aSample));
if (rv.isErr()) {
return EncodePromise::CreateAndReject(rv.inspectErr(), __func__);
}
return EncodePromise::CreateAndResolve(rv.unwrap(), __func__);
#endif
}
RefPtr<MediaDataEncoder::ReconfigurationPromise>
FFmpegDataEncoder<LIBAV_VER>::ProcessReconfigure(
const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) {
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
FFMPEG_LOG("ProcessReconfigure");
bool ok = false;
for (const auto& confChange : aConfigurationChanges->mChanges) {
// A reconfiguration on the fly succeeds if all changes can be applied
// successfuly. In case of failure, the encoder will be drained and
// recreated.
ok &= confChange.match(
// Not supported yet
[&](const DimensionsChange& aChange) -> bool { return false; },
[&](const DisplayDimensionsChange& aChange) -> bool { return false; },
[&](const BitrateModeChange& aChange) -> bool { return false; },
[&](const BitrateChange& aChange) -> bool {
// Verified on x264
if (!strcmp(mCodecContext->codec->name, "libx264")) {
MOZ_ASSERT(aChange.get().ref() != 0);
mConfig.mBitrate = aChange.get().ref();
mCodecContext->bit_rate =
static_cast<FFmpegBitRate>(mConfig.mBitrate);
return true;
}
return false;
},
[&](const FramerateChange& aChange) -> bool { return false; },
[&](const UsageChange& aChange) -> bool { return false; },
[&](const ContentHintChange& aChange) -> bool { return false; },
[&](const SampleRateChange& aChange) -> bool { return false; },
[&](const NumberOfChannelsChange& aChange) -> bool { return false; });
};
using P = MediaDataEncoder::ReconfigurationPromise;
if (ok) {
return P::CreateAndResolve(true, __func__);
}
return P::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
}
RefPtr<MediaDataEncoder::EncodePromise>
FFmpegDataEncoder<LIBAV_VER>::ProcessDrain() {
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
FFMPEG_LOG("ProcessDrain");
#if LIBAVCODEC_VERSION_MAJOR < 58
MOZ_CRASH("FFmpegDataEncoder needs ffmpeg 58 at least.");
return EncodePromise::CreateAndReject(NS_ERROR_NOT_IMPLEMENTED, __func__);
#else
auto rv = DrainWithModernAPIs();
if (rv.isErr()) {
return EncodePromise::CreateAndReject(rv.inspectErr(), __func__);
}
return EncodePromise::CreateAndResolve(rv.unwrap(), __func__);
#endif
}
RefPtr<ShutdownPromise> FFmpegDataEncoder<LIBAV_VER>::ProcessShutdown() {
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
FFMPEG_LOG("ProcessShutdown");
ShutdownInternal();
// Don't shut mTaskQueue down since it's owned by others.
return ShutdownPromise::CreateAndResolve(true, __func__);
}
AVCodec* FFmpegDataEncoder<LIBAV_VER>::InitCommon() {
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
FFMPEG_LOG("FFmpegDataEncoder::InitCommon");
AVCodec* codec = FindEncoderWithPreference(mLib, mCodecID);
if (!codec) {
FFMPEG_LOG("failed to find ffmpeg encoder for codec id %d", mCodecID);
return nullptr;
}
FFMPEG_LOG("found codec: %s", codec->name);
mCodecName = codec->name;
ForceEnablingFFmpegDebugLogs();
MOZ_ASSERT(!mCodecContext);
if (!(mCodecContext = mLib->avcodec_alloc_context3(codec))) {
FFMPEG_LOG("failed to allocate ffmpeg context for codec %s", codec->name);
return nullptr;
}
return codec;
}
MediaResult FFmpegDataEncoder<LIBAV_VER>::FinishInitCommon(AVCodec* aCodec) {
if (mConfig.mBitrateMode == BitrateMode::Constant) {
mCodecContext->rc_max_rate = static_cast<FFmpegBitRate>(mConfig.mBitrate);
mCodecContext->rc_min_rate = static_cast<FFmpegBitRate>(mConfig.mBitrate);
mCodecContext->bit_rate = static_cast<FFmpegBitRate>(mConfig.mBitrate);
FFMPEG_LOG("Encoding in CBR: %d", mConfig.mBitrate);
} else {
mCodecContext->rc_max_rate = static_cast<FFmpegBitRate>(mConfig.mBitrate);
mCodecContext->rc_min_rate = 0;
mCodecContext->bit_rate = static_cast<FFmpegBitRate>(mConfig.mBitrate);
FFMPEG_LOG("Encoding in VBR: [%d;%d]", (int)mCodecContext->rc_min_rate,
(int)mCodecContext->rc_max_rate);
}
#if LIBAVCODEC_VERSION_MAJOR >= 60
mCodecContext->flags |= AV_CODEC_FLAG_FRAME_DURATION;
#endif
AVDictionary* options = nullptr;
if (int ret = OpenCodecContext(aCodec, &options); ret < 0) {
FFMPEG_LOG("failed to open %s avcodec: %s", aCodec->name,
MakeErrorString(mLib, ret).get());
return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
RESULT_DETAIL("avcodec_open2 error"));
}
mLib->av_dict_free(&options);
return MediaResult(NS_OK);
}
void FFmpegDataEncoder<LIBAV_VER>::ShutdownInternal() {
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
FFMPEG_LOG("ShutdownInternal");
DestroyFrame();
if (mCodecContext) {
CloseCodecContext();
mLib->av_freep(&mCodecContext);
mCodecContext = nullptr;
}
}
int FFmpegDataEncoder<LIBAV_VER>::OpenCodecContext(const AVCodec* aCodec,
AVDictionary** aOptions) {
MOZ_ASSERT(mCodecContext);
StaticMutexAutoLock mon(sMutex);
return mLib->avcodec_open2(mCodecContext, aCodec, aOptions);
}
void FFmpegDataEncoder<LIBAV_VER>::CloseCodecContext() {
MOZ_ASSERT(mCodecContext);
StaticMutexAutoLock mon(sMutex);
mLib->avcodec_close(mCodecContext);
}
bool FFmpegDataEncoder<LIBAV_VER>::PrepareFrame() {
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
// TODO: Merge the duplicate part with FFmpegDataDecoder's PrepareFrame.
#if LIBAVCODEC_VERSION_MAJOR >= 55
if (mFrame) {
mLib->av_frame_unref(mFrame);
} else {
mFrame = mLib->av_frame_alloc();
}
#elif LIBAVCODEC_VERSION_MAJOR == 54
if (mFrame) {
mLib->avcodec_get_frame_defaults(mFrame);
} else {
mFrame = mLib->avcodec_alloc_frame();
}
#else
mLib->av_freep(&mFrame);
mFrame = mLib->avcodec_alloc_frame();
#endif
return !!mFrame;
}
void FFmpegDataEncoder<LIBAV_VER>::DestroyFrame() {
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
if (mFrame) {
#if LIBAVCODEC_VERSION_MAJOR >= 55
mLib->av_frame_unref(mFrame);
mLib->av_frame_free(&mFrame);
#elif LIBAVCODEC_VERSION_MAJOR == 54
mLib->avcodec_free_frame(&mFrame);
#else
mLib->av_freep(&mFrame);
#endif
mFrame = nullptr;
}
}
// avcodec_send_frame and avcodec_receive_packet were introduced in version 58.
#if LIBAVCODEC_VERSION_MAJOR >= 58
Result<MediaDataEncoder::EncodedData, nsresult>
FFmpegDataEncoder<LIBAV_VER>::EncodeWithModernAPIs() {
// Initialize AVPacket.
AVPacket* pkt = mLib->av_packet_alloc();
if (!pkt) {
FFMPEG_LOG("failed to allocate packet");
return Err(NS_ERROR_OUT_OF_MEMORY);
}
auto freePacket = MakeScopeExit([this, &pkt] { mLib->av_packet_free(&pkt); });
// Send frame and receive packets.
if (int ret = mLib->avcodec_send_frame(mCodecContext, mFrame); ret < 0) {
// In theory, avcodec_send_frame could sent -EAGAIN to signal its internal
// buffers is full. In practice this can't happen as we only feed one frame
// at a time, and we immediately call avcodec_receive_packet right after.
// TODO: Create a NS_ERROR_DOM_MEDIA_ENCODE_ERR in ErrorList.py?
FFMPEG_LOG("avcodec_send_frame error: %s",
MakeErrorString(mLib, ret).get());
return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR);
}
EncodedData output;
while (true) {
int ret = mLib->avcodec_receive_packet(mCodecContext, pkt);
if (ret == AVERROR(EAGAIN)) {
// The encoder is asking for more inputs.
FFMPEG_LOG("encoder is asking for more input!");
break;
}
if (ret < 0) {
// AVERROR_EOF is returned when the encoder has been fully flushed, but it
// shouldn't happen here.
FFMPEG_LOG("avcodec_receive_packet error: %s",
MakeErrorString(mLib, ret).get());
return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR);
}
RefPtr<MediaRawData> d = ToMediaRawData(pkt);
mLib->av_packet_unref(pkt);
if (!d) {
// This can happen if e.g. DTX is enabled
FFMPEG_LOG("No encoded packet output");
continue;
}
output.AppendElement(std::move(d));
}
FFMPEG_LOG("Got %zu encoded data", output.Length());
return std::move(output);
}
Result<MediaDataEncoder::EncodedData, nsresult>
FFmpegDataEncoder<LIBAV_VER>::DrainWithModernAPIs() {
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
MOZ_ASSERT(mCodecContext);
// TODO: Create a Result<EncodedData, nsresult> EncodeWithModernAPIs(AVFrame
// *aFrame) to merge the duplicate code below with EncodeWithModernAPIs above.
// Initialize AVPacket.
AVPacket* pkt = mLib->av_packet_alloc();
if (!pkt) {
FFMPEG_LOG("failed to allocate packet");
return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR);
}
auto freePacket = MakeScopeExit([this, &pkt] { mLib->av_packet_free(&pkt); });
// Enter draining mode by sending NULL to the avcodec_send_frame(). Note that
// this can leave the encoder in a permanent EOF state after draining. As a
// result, the encoder is unable to continue encoding. A new
// AVCodecContext/encoder creation is required if users need to encode after
// draining.
//
// TODO: Use `avcodec_flush_buffers` to drain the pending packets if
// AV_CODEC_CAP_ENCODER_FLUSH is set in mCodecContext->codec->capabilities.
if (int ret = mLib->avcodec_send_frame(mCodecContext, nullptr); ret < 0) {
if (ret == AVERROR_EOF) {
// The encoder has been flushed. Drain can be called multiple time.
FFMPEG_LOG("encoder has been flushed!");
return EncodedData();
}
FFMPEG_LOG("avcodec_send_frame error: %s",
MakeErrorString(mLib, ret).get());
return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR);
}
EncodedData output;
while (true) {
int ret = mLib->avcodec_receive_packet(mCodecContext, pkt);
if (ret == AVERROR_EOF) {
FFMPEG_LOG("encoder has no more output packet!");
break;
}
if (ret < 0) {
// avcodec_receive_packet should not result in a -EAGAIN once it's in
// draining mode.
FFMPEG_LOG("avcodec_receive_packet error: %s",
MakeErrorString(mLib, ret).get());
return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR);
}
RefPtr<MediaRawData> d = ToMediaRawData(pkt);
mLib->av_packet_unref(pkt);
if (!d) {
FFMPEG_LOG("failed to create a MediaRawData from the AVPacket");
return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR);
}
output.AppendElement(std::move(d));
}
FFMPEG_LOG("Encoding successful, %zu packets", output.Length());
// TODO: Only re-create AVCodecContext when avcodec_flush_buffers is
// unavailable.
ShutdownInternal();
nsresult r = InitSpecific();
return NS_FAILED(r) ? Result<MediaDataEncoder::EncodedData, nsresult>(
NS_ERROR_DOM_MEDIA_FATAL_ERR)
: Result<MediaDataEncoder::EncodedData, nsresult>(
std::move(output));
}
#endif // LIBAVCODEC_VERSION_MAJOR >= 58
RefPtr<MediaRawData> FFmpegDataEncoder<LIBAV_VER>::ToMediaRawDataCommon(
AVPacket* aPacket) {
MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
MOZ_ASSERT(aPacket);
// Copy frame data from AVPacket.
auto data = MakeRefPtr<MediaRawData>();
UniquePtr<MediaRawDataWriter> writer(data->CreateWriter());
if (!writer->Append(aPacket->data, static_cast<size_t>(aPacket->size))) {
FFMPEG_LOG("fail to allocate MediaRawData buffer");
return nullptr; // OOM
}
data->mKeyframe = (aPacket->flags & AV_PKT_FLAG_KEY) != 0;
// is recommended to be the reciprocal of the frame rate, but we set it to
// microsecond for now.
data->mTime = media::TimeUnit::FromMicroseconds(aPacket->pts);
#if LIBAVCODEC_VERSION_MAJOR >= 60
data->mDuration = media::TimeUnit::FromMicroseconds(aPacket->duration);
#else
int64_t duration;
if (mDurationMap.Find(aPacket->pts, duration)) {
data->mDuration = media::TimeUnit::FromMicroseconds(duration);
} else {
data->mDuration = media::TimeUnit::FromMicroseconds(aPacket->duration);
}
#endif
data->mTimecode = media::TimeUnit::FromMicroseconds(aPacket->dts);
if (auto r = GetExtraData(aPacket); r.isOk()) {
data->mExtraData = r.unwrap();
}
return data;
}
void FFmpegDataEncoder<LIBAV_VER>::ForceEnablingFFmpegDebugLogs() {
#if DEBUG
if (!getenv("MOZ_AV_LOG_LEVEL") &&
MOZ_LOG_TEST(sFFmpegVideoLog, LogLevel::Debug)) {
mLib->av_log_set_level(AV_LOG_DEBUG);
}
#endif // DEBUG
}
} // namespace mozilla