Source code
Revision control
Copy as Markdown
Other Tools
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/video_quality_observer2.h"
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <string>
#include "rtc_base/logging.h"
#include "rtc_base/strings/string_builder.h"
#include "system_wrappers/include/metrics.h"
#include "video/video_receive_stream2.h"
namespace webrtc {
namespace internal {
const uint32_t VideoQualityObserver::kMinFrameSamplesToDetectFreeze = 5;
const uint32_t VideoQualityObserver::kMinIncreaseForFreezeMs = 150;
const uint32_t VideoQualityObserver::kAvgInterframeDelaysWindowSizeFrames = 30;
namespace {
constexpr int kMinVideoDurationMs = 3000;
constexpr int kMinRequiredSamples = 1;
constexpr int kPixelsInHighResolution =
960 * 540; // CPU-adapted HD still counts.
constexpr int kPixelsInMediumResolution = 640 * 360;
constexpr int kBlockyQpThresholdVp8 = 70;
constexpr int kBlockyQpThresholdVp9 = 180;
constexpr int kMaxNumCachedBlockyFrames = 100;
// TODO(ilnik): Add H264/HEVC thresholds.
} // namespace
VideoQualityObserver::VideoQualityObserver()
: last_frame_rendered_ms_(-1),
num_frames_rendered_(0),
first_frame_rendered_ms_(-1),
last_frame_pixels_(0),
is_last_frame_blocky_(false),
last_unfreeze_time_ms_(0),
render_interframe_delays_(kAvgInterframeDelaysWindowSizeFrames),
sum_squared_interframe_delays_secs_(0.0),
time_in_resolution_ms_(3, 0),
current_resolution_(Resolution::Low),
num_resolution_downgrades_(0),
time_in_blocky_video_ms_(0),
is_paused_(false) {}
void VideoQualityObserver::UpdateHistograms(bool screenshare) {
// TODO(bugs.webrtc.org/11489): Called on the decoder thread - which _might_
// be the same as the construction thread.
// Don't report anything on an empty video stream.
if (num_frames_rendered_ == 0) {
return;
}
char log_stream_buf[2 * 1024];
rtc::SimpleStringBuilder log_stream(log_stream_buf);
if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
smooth_playback_durations_.Add(last_frame_rendered_ms_ -
last_unfreeze_time_ms_);
}
std::string uma_prefix =
screenshare ? "WebRTC.Video.Screenshare" : "WebRTC.Video";
auto mean_time_between_freezes =
smooth_playback_durations_.Avg(kMinRequiredSamples);
if (mean_time_between_freezes) {
RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanTimeBetweenFreezesMs",
*mean_time_between_freezes);
log_stream << uma_prefix << ".MeanTimeBetweenFreezesMs "
<< *mean_time_between_freezes << "\n";
}
auto avg_freeze_length = freezes_durations_.Avg(kMinRequiredSamples);
if (avg_freeze_length) {
RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanFreezeDurationMs",
*avg_freeze_length);
log_stream << uma_prefix << ".MeanFreezeDurationMs " << *avg_freeze_length
<< "\n";
}
int64_t video_duration_ms =
last_frame_rendered_ms_ - first_frame_rendered_ms_;
if (video_duration_ms >= kMinVideoDurationMs) {
int time_spent_in_hd_percentage = static_cast<int>(
time_in_resolution_ms_[Resolution::High] * 100 / video_duration_ms);
RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInHdPercentage",
time_spent_in_hd_percentage);
log_stream << uma_prefix << ".TimeInHdPercentage "
<< time_spent_in_hd_percentage << "\n";
int time_with_blocky_video_percentage =
static_cast<int>(time_in_blocky_video_ms_ * 100 / video_duration_ms);
RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInBlockyVideoPercentage",
time_with_blocky_video_percentage);
log_stream << uma_prefix << ".TimeInBlockyVideoPercentage "
<< time_with_blocky_video_percentage << "\n";
int num_resolution_downgrades_per_minute =
num_resolution_downgrades_ * 60000 / video_duration_ms;
if (!screenshare) {
RTC_HISTOGRAM_COUNTS_SPARSE_100(
uma_prefix + ".NumberResolutionDownswitchesPerMinute",
num_resolution_downgrades_per_minute);
log_stream << uma_prefix << ".NumberResolutionDownswitchesPerMinute "
<< num_resolution_downgrades_per_minute << "\n";
}
int num_freezes_per_minute =
freezes_durations_.NumSamples() * 60000 / video_duration_ms;
RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".NumberFreezesPerMinute",
num_freezes_per_minute);
log_stream << uma_prefix << ".NumberFreezesPerMinute "
<< num_freezes_per_minute << "\n";
if (sum_squared_interframe_delays_secs_ > 0.0) {
int harmonic_framerate_fps = std::round(
video_duration_ms / (1000 * sum_squared_interframe_delays_secs_));
RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".HarmonicFrameRate",
harmonic_framerate_fps);
log_stream << uma_prefix << ".HarmonicFrameRate "
<< harmonic_framerate_fps << "\n";
}
}
RTC_LOG(LS_INFO) << log_stream.str();
}
void VideoQualityObserver::OnRenderedFrame(
const VideoFrameMetaData& frame_meta) {
RTC_DCHECK_LE(last_frame_rendered_ms_, frame_meta.decode_timestamp.ms());
RTC_DCHECK_LE(last_unfreeze_time_ms_, frame_meta.decode_timestamp.ms());
if (num_frames_rendered_ == 0) {
first_frame_rendered_ms_ = last_unfreeze_time_ms_ =
frame_meta.decode_timestamp.ms();
}
auto blocky_frame_it = blocky_frames_.find(frame_meta.rtp_timestamp);
if (num_frames_rendered_ > 0) {
// Process inter-frame delay.
const int64_t interframe_delay_ms =
frame_meta.decode_timestamp.ms() - last_frame_rendered_ms_;
const double interframe_delays_secs = interframe_delay_ms / 1000.0;
// Sum of squared inter frame intervals is used to calculate the harmonic
// frame rate metric. The metric aims to reflect overall experience related
// to smoothness of video playback and includes both freezes and pauses.
sum_squared_interframe_delays_secs_ +=
interframe_delays_secs * interframe_delays_secs;
if (!is_paused_) {
render_interframe_delays_.AddSample(interframe_delay_ms);
bool was_freeze = false;
if (render_interframe_delays_.Size() >= kMinFrameSamplesToDetectFreeze) {
const absl::optional<int64_t> avg_interframe_delay =
render_interframe_delays_.GetAverageRoundedDown();
RTC_DCHECK(avg_interframe_delay);
was_freeze = interframe_delay_ms >=
std::max(3 * *avg_interframe_delay,
*avg_interframe_delay + kMinIncreaseForFreezeMs);
}
if (was_freeze) {
freezes_durations_.Add(interframe_delay_ms);
smooth_playback_durations_.Add(last_frame_rendered_ms_ -
last_unfreeze_time_ms_);
last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms();
} else {
// Count spatial metrics if there were no freeze.
time_in_resolution_ms_[current_resolution_] += interframe_delay_ms;
if (is_last_frame_blocky_) {
time_in_blocky_video_ms_ += interframe_delay_ms;
}
}
}
}
if (is_paused_) {
// If the stream was paused since the previous frame, do not count the
// pause toward smooth playback. Explicitly count the part before it and
// start the new smooth playback interval from this frame.
is_paused_ = false;
if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
smooth_playback_durations_.Add(last_frame_rendered_ms_ -
last_unfreeze_time_ms_);
}
last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms();
if (num_frames_rendered_ > 0) {
pauses_durations_.Add(frame_meta.decode_timestamp.ms() -
last_frame_rendered_ms_);
}
}
int64_t pixels = frame_meta.width * frame_meta.height;
if (pixels >= kPixelsInHighResolution) {
current_resolution_ = Resolution::High;
} else if (pixels >= kPixelsInMediumResolution) {
current_resolution_ = Resolution::Medium;
} else {
current_resolution_ = Resolution::Low;
}
if (pixels < last_frame_pixels_) {
++num_resolution_downgrades_;
}
last_frame_pixels_ = pixels;
last_frame_rendered_ms_ = frame_meta.decode_timestamp.ms();
is_last_frame_blocky_ = blocky_frame_it != blocky_frames_.end();
if (is_last_frame_blocky_) {
blocky_frames_.erase(blocky_frames_.begin(), ++blocky_frame_it);
}
++num_frames_rendered_;
}
void VideoQualityObserver::OnDecodedFrame(uint32_t rtp_frame_timestamp,
absl::optional<uint8_t> qp,
VideoCodecType codec) {
if (!qp)
return;
absl::optional<int> qp_blocky_threshold;
// TODO(ilnik): add other codec types when we have QP for them.
switch (codec) {
case kVideoCodecVP8:
qp_blocky_threshold = kBlockyQpThresholdVp8;
break;
case kVideoCodecVP9:
qp_blocky_threshold = kBlockyQpThresholdVp9;
break;
default:
qp_blocky_threshold = absl::nullopt;
}
RTC_DCHECK(blocky_frames_.find(rtp_frame_timestamp) == blocky_frames_.end());
if (qp_blocky_threshold && *qp > *qp_blocky_threshold) {
// Cache blocky frame. Its duration will be calculated in render callback.
if (blocky_frames_.size() > kMaxNumCachedBlockyFrames) {
RTC_LOG(LS_WARNING) << "Overflow of blocky frames cache.";
blocky_frames_.erase(
blocky_frames_.begin(),
std::next(blocky_frames_.begin(), kMaxNumCachedBlockyFrames / 2));
}
blocky_frames_.insert(rtp_frame_timestamp);
}
}
void VideoQualityObserver::OnStreamInactive() {
is_paused_ = true;
}
uint32_t VideoQualityObserver::NumFreezes() const {
return freezes_durations_.NumSamples();
}
uint32_t VideoQualityObserver::NumPauses() const {
return pauses_durations_.NumSamples();
}
uint32_t VideoQualityObserver::TotalFreezesDurationMs() const {
return freezes_durations_.Sum(kMinRequiredSamples).value_or(0);
}
uint32_t VideoQualityObserver::TotalPausesDurationMs() const {
return pauses_durations_.Sum(kMinRequiredSamples).value_or(0);
}
uint32_t VideoQualityObserver::TotalFramesDurationMs() const {
return last_frame_rendered_ms_ - first_frame_rendered_ms_;
}
double VideoQualityObserver::SumSquaredFrameDurationsSec() const {
return sum_squared_interframe_delays_secs_;
}
} // namespace internal
} // namespace webrtc