Source code
Revision control
Copy as Markdown
Other Tools
/*
* Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/video_codecs/libaom_av1_encoder_factory.h"
#include <cstdint>
#include <cstdio>
#include <memory>
#include <ostream>
#include <string>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
#include "absl/types/variant.h"
#include "api/array_view.h"
#include "api/scoped_refptr.h"
#include "api/units/data_rate.h"
#include "api/units/data_size.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame.h"
#include "api/video/video_frame_buffer.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder_factory_interface.h"
#include "api/video_codecs/video_encoder_interface.h"
#include "api/video_codecs/video_encoding_general.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/codecs/av1/dav1d_decoder.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/testsupport/file_utils.h"
#include "test/testsupport/frame_reader.h"
namespace webrtc {
namespace {
using ::testing::Eq;
using ::testing::Gt;
using ::testing::IsEmpty;
using ::testing::Not;
using Cbr = VideoEncoderInterface::FrameEncodeSettings::Cbr;
using Cqp = VideoEncoderInterface::FrameEncodeSettings::Cqp;
using EncodedData = VideoEncoderInterface::EncodedData;
using EncodeResult = VideoEncoderInterface::EncodeResult;
using FrameType = VideoEncoderInterface::FrameType;
std::unique_ptr<test::FrameReader> CreateFrameReader() {
return CreateY4mFrameReader(
test::ResourcePath("reference_video_640x360_30fps", "y4m"),
test::YuvFrameReaderImpl::RepeatMode::kPingPong);
}
std::string OutPath() {
std::string res = test::OutputPath();
res += "frame_dump/";
RTC_CHECK(test::DirExists(res) || test::CreateDir(res));
return res;
}
class Av1Decoder : public DecodedImageCallback {
public:
Av1Decoder() : Av1Decoder("") {}
explicit Av1Decoder(const std::string& name)
: decoder_(CreateDav1dDecoder()), file_name_(name) {
decoder_->Configure({});
decoder_->RegisterDecodeCompleteCallback(this);
if (!file_name_.empty()) {
std::string out = OutPath();
out += file_name_;
out += "_raw.av1";
RTC_CHECK(raw_out_file_ = fopen(out.c_str(), "wb"));
RTC_LOG(LS_INFO) << "Recording bitstream to " << out;
}
}
~Av1Decoder() {
if (raw_out_file_) {
fclose(raw_out_file_);
}
}
// DecodedImageCallback
int32_t Decoded(VideoFrame& frame) override {
decode_result_ = std::make_unique<VideoFrame>(std::move(frame));
return 0;
}
VideoFrame Decode(rtc::ArrayView<uint8_t> bitstream_data) {
EncodedImage img;
img.SetEncodedData(EncodedImageBuffer::Create(bitstream_data.data(),
bitstream_data.size()));
if (raw_out_file_) {
fwrite(bitstream_data.data(), 1, bitstream_data.size(), raw_out_file_);
}
decoder_->Decode(img, /*dont_care=*/0);
VideoFrame res(std::move(*decode_result_));
return res;
}
private:
std::unique_ptr<VideoDecoder> decoder_;
std::unique_ptr<VideoFrame> decode_result_;
std::string file_name_;
FILE* raw_out_file_ = nullptr;
};
struct EncOut {
std::vector<uint8_t> bitstream;
EncodeResult res;
};
class FrameEncoderSettingsBuilder {
public:
FrameEncoderSettingsBuilder() {
class IgnoredOutput : public VideoEncoderInterface::FrameOutput {
public:
rtc::ArrayView<uint8_t> GetBitstreamOutputBuffer(DataSize size) override {
unread_.resize(size.bytes());
return unread_;
}
void EncodeComplete(const EncodeResult& encode_result) override {}
private:
std::vector<uint8_t> unread_;
};
frame_encode_settings_.frame_output = std::make_unique<IgnoredOutput>();
}
FrameEncoderSettingsBuilder& Key() {
frame_encode_settings_.frame_type = FrameType::kKeyframe;
return *this;
}
FrameEncoderSettingsBuilder& Start() {
frame_encode_settings_.frame_type = FrameType::kStartFrame;
return *this;
}
FrameEncoderSettingsBuilder& Delta() {
frame_encode_settings_.frame_type = FrameType::kStartFrame;
return *this;
}
FrameEncoderSettingsBuilder& Rate(
const absl::variant<Cqp, Cbr>& rate_options) {
frame_encode_settings_.rate_options = rate_options;
return *this;
}
FrameEncoderSettingsBuilder& T(int id) {
frame_encode_settings_.temporal_id = id;
return *this;
}
FrameEncoderSettingsBuilder& S(int id) {
frame_encode_settings_.spatial_id = id;
return *this;
}
FrameEncoderSettingsBuilder& Res(int width, int height) {
frame_encode_settings_.resolution = {width, height};
return *this;
}
FrameEncoderSettingsBuilder& Ref(const std::vector<int>& ref) {
frame_encode_settings_.reference_buffers = ref;
return *this;
}
FrameEncoderSettingsBuilder& Upd(int upd) {
frame_encode_settings_.update_buffer = upd;
return *this;
}
FrameEncoderSettingsBuilder& Effort(int effort_level) {
frame_encode_settings_.effort_level = effort_level;
return *this;
}
FrameEncoderSettingsBuilder& Out(EncOut& out) {
frame_encode_settings_.frame_output = std::make_unique<FrameOut>(out);
return *this;
}
operator VideoEncoderInterface::FrameEncodeSettings&&() {
return std::move(frame_encode_settings_);
}
private:
struct FrameOut : public VideoEncoderInterface::FrameOutput {
explicit FrameOut(EncOut& e) : eo(e) {}
rtc::ArrayView<uint8_t> GetBitstreamOutputBuffer(DataSize size) override {
eo.bitstream.resize(size.bytes());
return rtc::ArrayView<uint8_t>(eo.bitstream);
}
void EncodeComplete(const EncodeResult& encode_result) override {
eo.res = encode_result;
}
EncOut& eo;
};
VideoEncoderInterface::FrameEncodeSettings frame_encode_settings_;
};
using Fb = FrameEncoderSettingsBuilder;
// Since FrameEncodeSettings is move only, initalizer-list initialization won't
// work, so instead a C-style array can be used to do aggregate initialization.
template <int N>
std::vector<VideoEncoderInterface::FrameEncodeSettings> ToVec(
VideoEncoderInterface::FrameEncodeSettings (&&settings)[N]) {
return std::vector<VideoEncoderInterface::FrameEncodeSettings>(
std::make_move_iterator(std::begin(settings)),
std::make_move_iterator(std::end(settings)));
}
// For reasonable debug printout when an EXPECT fail.
struct Resolution {
explicit Resolution(const VideoFrame& frame)
: width(frame.width()), height(frame.height()) {}
friend void PrintTo(const Resolution& res, std::ostream* os) {
*os << "(width: " << res.width << " height: " << res.height << ")";
}
int width;
int height;
};
MATCHER_P2(ResolutionIs, width, height, "") {
return arg.width == width && arg.height == height;
}
MATCHER_P(QpIs, qp, "") {
if (auto ed = absl::get_if<EncodedData>(&arg.res)) {
return ed->encoded_qp == qp;
}
return false;
}
MATCHER(HasBitstreamAndMetaData, "") {
return !arg.bitstream.empty() &&
absl::holds_alternative<EncodedData>(arg.res);
}
double Psnr(const rtc::scoped_refptr<I420BufferInterface>& ref_buffer,
const VideoFrame& decoded_frame) {
return I420PSNR(*ref_buffer, *decoded_frame.video_frame_buffer()->ToI420());
}
static constexpr VideoEncoderFactoryInterface::StaticEncoderSettings
kCbrEncoderSettings{
.max_encode_dimensions = {.width = 1920, .height = 1080},
.encoding_format = {.sub_sampling = EncodingFormat::SubSampling::k420,
.bit_depth = 8},
.rc_mode =
VideoEncoderFactoryInterface::StaticEncoderSettings::Cbr{
.max_buffer_size = TimeDelta::Millis(1000),
.target_buffer_size = TimeDelta::Millis(600)},
.max_number_of_threads = 1,
};
static constexpr VideoEncoderFactoryInterface::StaticEncoderSettings
kCqpEncoderSettings{
.max_encode_dimensions = {.width = 1920, .height = 1080},
.encoding_format = {.sub_sampling = EncodingFormat::SubSampling::k420,
.bit_depth = 8},
.rc_mode = VideoEncoderFactoryInterface::StaticEncoderSettings::Cqp(),
.max_number_of_threads = 1,
};
static constexpr Cbr kCbr{.duration = TimeDelta::Millis(100),
.target_bitrate = DataRate::KilobitsPerSec(1000)};
TEST(LibaomAv1EncoderFactory, CodecName) {
EXPECT_THAT(LibaomAv1EncoderFactory().CodecName(), Eq("AV1"));
}
TEST(LibaomAv1EncoderFactory, CodecSpecifics) {
EXPECT_THAT(LibaomAv1EncoderFactory().CodecSpecifics(), IsEmpty());
}
TEST(LibaomAv1EncoderFactory, QpRange) {
const std::pair<int, int> kMinMaxQp = {0, 63};
EXPECT_THAT(
LibaomAv1EncoderFactory().GetEncoderCapabilities().rate_control.qp_range,
Eq(kMinMaxQp));
}
TEST(LibaomAv1Encoder, KeyframeUpdatesSpecifiedBuffer) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
Av1Decoder dec;
auto raw_key = frame_reader->PullFrame();
auto raw_delta = frame_reader->PullFrame();
EncOut key;
enc->Encode(raw_key, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(5).Key().Out(key)}));
ASSERT_THAT(key.bitstream, Not(IsEmpty()));
VideoFrame decoded_key = dec.Decode(key.bitstream);
EXPECT_THAT(Resolution(decoded_key), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(raw_key, decoded_key), Gt(40));
EncOut delta;
enc->Encode(raw_delta, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Out(delta)}));
EXPECT_THAT(delta, Not(HasBitstreamAndMetaData()));
}
TEST(LibaomAv1Encoder, MidTemporalUnitKeyframeResetsBuffers) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncOut tu0_s2;
enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key(),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Out(tu0_s2)}));
EXPECT_THAT(tu0_s2, HasBitstreamAndMetaData());
EncOut tu1_s0;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Ref({0}).Out(tu1_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Upd(1).Key(),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0})}));
EXPECT_THAT(tu1_s0, Not(HasBitstreamAndMetaData()));
}
TEST(LibaomAv1Encoder, ResolutionSwitching) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame();
EncOut tu0;
enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(320, 180).Upd(0).Key().Out(tu0)}));
rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame();
EncOut tu1;
enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Out(tu1)}));
rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame();
EncOut tu2;
enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu2)}));
Av1Decoder dec;
VideoFrame f0 = dec.Decode(tu0.bitstream);
EXPECT_THAT(Resolution(f0), ResolutionIs(320, 180));
// TD:
// EXPECT_THAT(Psnr(in0, f0), Gt(40));
VideoFrame f1 = dec.Decode(tu1.bitstream);
EXPECT_THAT(Resolution(f1), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(in1, f1), Gt(40));
VideoFrame f2 = dec.Decode(tu2.bitstream);
EXPECT_THAT(Resolution(f2), ResolutionIs(160, 90));
// TD:
// EXPECT_THAT(Psnr(in2, f2), Gt(40));
}
TEST(LibaomAv1Encoder, InputResolutionSwitching) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame();
EncOut tu0;
enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Upd(0).Key().Out(tu0)}));
rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame(
/*frame_num=*/nullptr,
/*resolution=*/{320, 180},
/*framerate_scale=*/{1, 1});
EncOut tu1;
enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu1)}));
rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame(
/*frame_num=*/nullptr,
/*resolution=*/{160, 90},
/*framerate_scale=*/{1, 1});
EncOut tu2;
enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu2)}));
Av1Decoder dec;
VideoFrame f0 = dec.Decode(tu0.bitstream);
EXPECT_THAT(Resolution(f0), ResolutionIs(160, 90));
// TD:
// EXPECT_THAT(Psnr(in0, f0), Gt(40));
VideoFrame f1 = dec.Decode(tu1.bitstream);
EXPECT_THAT(Resolution(f1), ResolutionIs(160, 90));
// TD:
// EXPECT_THAT(Psnr(in1, f1), Gt(40));
VideoFrame f2 = dec.Decode(tu2.bitstream);
EXPECT_THAT(Resolution(f2), ResolutionIs(160, 90));
EXPECT_THAT(Psnr(in2, f2), Gt(40));
}
TEST(LibaomAv1Encoder, TempoSpatial) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
const Cbr k10Fps{.duration = TimeDelta::Millis(100),
.target_bitrate = DataRate::KilobitsPerSec(500)};
const Cbr k20Fps{.duration = TimeDelta::Millis(50),
.target_bitrate = DataRate::KilobitsPerSec(500)};
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec(
{Fb().Rate(k10Fps).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EncOut tu1_s2;
enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(50)},
ToVec({Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({2}).Upd(2).Out(
tu1_s2)}));
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode(
frame, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec(
{Fb().Rate(k10Fps).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Out(tu2_s1),
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Out(
tu2_s2)}));
Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu1_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f = dec.Decode(tu2_s2.bitstream);
EXPECT_THAT(Resolution(f), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(frame, f), Gt(40));
}
TEST(DISABLED_LibaomAv1Encoder, InvertedTempoSpatial) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncOut tu0_s0;
EncOut tu0_s1;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(640, 360).S(1).Ref({0}).Upd(1).Out(tu0_s1)}));
EncOut tu1_s0;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Out(tu1_s0)}));
EncOut tu2_s0;
EncOut tu2_s1;
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
enc->Encode(
frame, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec(
{Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(640, 360).S(1).Ref({1, 0}).Upd(1).Out(tu2_s1)}));
Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(640, 360));
}
TEST(LibaomAv1Encoder, SkipMidLayer) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EncOut tu1_s0;
EncOut tu1_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)}));
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
enc->Encode(
frame, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec(
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Out(tu2_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Out(tu2_s2)}));
Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu1_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f = dec.Decode(tu2_s2.bitstream);
EXPECT_THAT(Resolution(f), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(frame, f), Gt(40));
}
TEST(LibaomAv1Encoder, L3T1) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
Av1Decoder dec;
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360));
auto tu1_frame = frame_reader->PullFrame();
EncOut tu1_s0;
EncOut tu1_s1;
EncOut tu1_s2;
enc->Encode(
tu1_frame, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec(
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Out(tu1_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Out(tu1_s2)}));
EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu1_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f_tu1_s2 = dec.Decode(tu1_s2.bitstream);
EXPECT_THAT(Resolution(f_tu1_s2), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(tu1_frame, f_tu1_s2), Gt(40));
auto tu2_frame = frame_reader->PullFrame();
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode(
tu2_frame, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec(
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Out(tu2_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Out(tu2_s2)}));
EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f_tu2 = dec.Decode(tu2_s2.bitstream);
EXPECT_THAT(Resolution(f_tu2), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(tu2_frame, f_tu2), Gt(40));
}
TEST(LibaomAv1Encoder, L3T1_KEY) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
Av1Decoder dec_s0;
Av1Decoder dec_s1;
Av1Decoder dec_s2;
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(tu0_s0.bitstream)),
ResolutionIs(160, 90));
dec_s1.Decode(tu0_s0.bitstream);
EXPECT_THAT(Resolution(dec_s1.Decode(tu0_s1.bitstream)),
ResolutionIs(320, 180));
dec_s2.Decode(tu0_s0.bitstream);
dec_s2.Decode(tu0_s1.bitstream);
EXPECT_THAT(Resolution(dec_s2.Decode(tu0_s2.bitstream)),
ResolutionIs(640, 360));
EncOut tu1_s0;
EncOut tu1_s1;
EncOut tu1_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu1_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(tu1_s0.bitstream)),
ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(tu1_s1.bitstream)),
ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(tu1_s2.bitstream)),
ResolutionIs(640, 360));
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu2_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu2_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(tu2_s0.bitstream)),
ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(tu2_s1.bitstream)),
ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(tu2_s2.bitstream)),
ResolutionIs(640, 360));
}
TEST(LibaomAv1Encoder, S3T1) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
Av1Decoder dec_s0;
Av1Decoder dec_s1;
Av1Decoder dec_s2;
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Start().Upd(0).Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Start().Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Start().Upd(2).Out(tu0_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(tu0_s0.bitstream)),
ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(tu0_s1.bitstream)),
ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(tu0_s2.bitstream)),
ResolutionIs(640, 360));
EncOut tu1_s0;
EncOut tu1_s1;
EncOut tu1_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu1_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(tu1_s0.bitstream)),
ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(tu1_s1.bitstream)),
ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(tu1_s2.bitstream)),
ResolutionIs(640, 360));
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu2_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu2_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(tu2_s0.bitstream)),
ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(tu2_s1.bitstream)),
ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(tu2_s2.bitstream)),
ResolutionIs(640, 360));
}
TEST(LibaomAv1Encoder, HigherEffortLevelYieldsHigherQualityFrames) {
auto frame_in = CreateFrameReader()->PullFrame();
std::pair<int, int> effort_range = LibaomAv1EncoderFactory()
.GetEncoderCapabilities()
.performance.min_max_effort_level;
// Cbr rc{.duration = TimeDelta::Millis(100),
// .target_bitrate = DataRate::KilobitsPerSec(100)};
absl::optional<double> psnr_last;
Av1Decoder dec;
for (int i = effort_range.first; i <= effort_range.second; ++i) {
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncOut tu0;
enc->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(0).Key().Effort(i).Out(tu0)}));
double psnr = Psnr(frame_in, dec.Decode(tu0.bitstream));
EXPECT_THAT(psnr, Gt(psnr_last));
psnr_last = psnr;
}
}
TEST(LibaomAv1Encoder, KeyframeAndStartrameAreApproximatelyEqual) {
int max_spatial_layers = LibaomAv1EncoderFactory()
.GetEncoderCapabilities()
.prediction_constraints.max_spatial_layers;
const Cbr kRate{.duration = TimeDelta::Millis(100),
.target_bitrate = DataRate::KilobitsPerSec(500)};
for (int sid = 0; sid < max_spatial_layers; ++sid) {
std::string key_name = "cbr_key_sl_";
key_name += std::to_string(sid);
Av1Decoder dec_key(key_name);
std::string start_name = "cbr_start_sl_";
start_name += std::to_string(sid);
Av1Decoder dec_start(start_name);
auto frame_reader = CreateFrameReader();
auto enc_key =
LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
auto enc_start =
LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
DataSize total_size_key = DataSize::Zero();
DataSize total_size_start = DataSize::Zero();
TimeDelta total_duration = TimeDelta::Zero();
auto frame_in = frame_reader->PullFrame();
EncOut key;
EncOut start;
enc_key->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Out(key)}));
enc_start->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec(
{Fb().Rate(kRate).Res(640, 360).S(sid).Start().Upd(0).Out(start)}));
total_size_key += DataSize::Bytes(key.bitstream.size());
total_size_start += DataSize::Bytes(start.bitstream.size());
total_duration += kRate.duration;
dec_key.Decode(key.bitstream);
dec_start.Decode(start.bitstream);
EXPECT_NEAR(total_size_key.bytes(), total_size_start.bytes(),
0.1 * total_size_key.bytes());
for (int f = 1; f < 10; ++f) {
frame_in = frame_reader->PullFrame();
enc_key->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out(
key)}));
enc_start->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out(
start)}));
total_size_key += DataSize::Bytes(key.bitstream.size());
total_size_start += DataSize::Bytes(start.bitstream.size());
total_duration += kRate.duration;
dec_key.Decode(key.bitstream);
dec_start.Decode(start.bitstream);
}
double key_encode_kbps = (total_size_key / total_duration).kbps();
double start_encode_kbps = (total_size_start / total_duration).kbps();
EXPECT_NEAR(key_encode_kbps, start_encode_kbps, start_encode_kbps * 0.05);
}
}
TEST(LibaomAv1Encoder, BitrateConsistentAcrossSpatialLayers) {
int max_spatial_layers = LibaomAv1EncoderFactory()
.GetEncoderCapabilities()
.prediction_constraints.max_spatial_layers;
const Cbr kRate{.duration = TimeDelta::Millis(100),
.target_bitrate = DataRate::KilobitsPerSec(500)};
for (int sid = 0; sid < max_spatial_layers; ++sid) {
std::string out_name = "cbr_sl_";
out_name += std::to_string(sid);
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
DataSize total_size = DataSize::Zero();
TimeDelta total_duration = TimeDelta::Zero();
EncOut out;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Out(out)}));
total_size += DataSize::Bytes(out.bitstream.size());
total_duration += kRate.duration;
for (int f = 1; f < 30; ++f) {
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(f * 100)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out(
out)}));
total_size += DataSize::Bytes(out.bitstream.size());
total_duration += kRate.duration;
}
double encode_kbps = (total_size / total_duration).kbps();
double target_kbps = kRate.target_bitrate.kbps();
EXPECT_NEAR(encode_kbps, target_kbps, target_kbps * 0.1);
}
}
TEST(LibaomAv1Encoder, ConstantQp) {
int max_spatial_layers = LibaomAv1EncoderFactory()
.GetEncoderCapabilities()
.prediction_constraints.max_spatial_layers;
constexpr int kQp = 30;
for (int sid = 0; sid < max_spatial_layers; ++sid) {
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCqpEncoderSettings, {});
std::string out_name = "cqp_sl_";
out_name += std::to_string(sid);
auto frame_reader = CreateFrameReader();
EncOut out;
enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(Cqp{.target_qp = kQp})
.Res(640, 360)
.S(sid)
.Upd(0)
.Key()
.Out(out)}));
EXPECT_THAT(out, QpIs(kQp));
for (int f = 1; f < 10; ++f) {
enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(f * 100)},
ToVec({Fb().Rate(Cqp{.target_qp = kQp - f})
.Res(640, 360)
.S(sid)
.Ref({0})
.Upd(0)
.Out(out)}));
EXPECT_THAT(out, QpIs(kQp - f));
}
}
}
} // namespace
} // namespace webrtc