Source code
Revision control
Copy as Markdown
Other Tools
/*
* Copyright 2024 The WebRTC project authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/corruption_detection/halton_frame_sampler.h"
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <vector>
#include "api/scoped_refptr.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame_buffer.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "video/corruption_detection/halton_sequence.h"
namespace webrtc {
namespace {
constexpr int kMaxFramesBetweenSamples = 33;
// Corresponds to 1 second for RTP timestamps (which are 90kHz).
constexpr uint32_t kMaxDurationBetweenSamples = 90'000;
// The second *time* is always later than the first. If the second *timestamp*
// is smaller than the first, we interpret that as if one wraparound has
// occurred.
uint32_t EnoughTimeHasPassed(uint32_t from, uint32_t to) {
return (to - from) >= kMaxDurationBetweenSamples;
}
} // namespace
HaltonFrameSampler::HaltonFrameSampler()
: coordinate_sampler_prng_(HaltonSequence(2)) {}
std::vector<HaltonFrameSampler::Coordinates>
HaltonFrameSampler::GetSampleCoordinatesForFrameIfFrameShouldBeSampled(
bool is_key_frame,
uint32_t rtp_timestamp,
int num_samples) {
if (num_samples < 1) {
return {};
}
if (rtp_timestamp_last_frame_sampled_.has_value()) {
RTC_CHECK_NE(*rtp_timestamp_last_frame_sampled_, rtp_timestamp);
}
if (is_key_frame || frames_until_next_sample_ <= 0 ||
!rtp_timestamp_last_frame_sampled_.has_value() ||
EnoughTimeHasPassed(*rtp_timestamp_last_frame_sampled_, rtp_timestamp)) {
frames_until_next_sample_ =
(kMaxFramesBetweenSamples - 1) - (frames_sampled_ % 8);
++frames_sampled_;
rtp_timestamp_last_frame_sampled_ = rtp_timestamp;
return GetSampleCoordinatesForFrame(num_samples);
}
--frames_until_next_sample_;
return {};
}
std::vector<HaltonFrameSampler::Coordinates>
HaltonFrameSampler::GetSampleCoordinatesForFrame(int num_samples) {
RTC_CHECK_GE(num_samples, 1);
std::vector<Coordinates> coordinates;
coordinates.reserve(num_samples);
for (int i = 0; i < num_samples; ++i) {
coordinates.push_back(GetNextSampleCoordinates());
}
return coordinates;
}
HaltonFrameSampler::Coordinates HaltonFrameSampler::GetNextSampleCoordinates() {
std::vector<double> point = coordinate_sampler_prng_.GetNext();
return {.row = point[0], .column = point[1]};
}
void HaltonFrameSampler::Restart() {
coordinate_sampler_prng_.Reset();
}
int HaltonFrameSampler::GetCurrentIndex() const {
return coordinate_sampler_prng_.GetCurrentIndex();
}
void HaltonFrameSampler::SetCurrentIndex(int index) {
coordinate_sampler_prng_.SetCurrentIndex(index);
}
// Apply Gaussian filtering to the data.
double GetFilteredElement(int width,
int height,
int stride,
const uint8_t* data,
int row,
int column,
double stddev) {
RTC_CHECK_GE(row, 0);
RTC_CHECK_LT(row, height);
RTC_CHECK_GE(column, 0);
RTC_CHECK_LT(column, width);
RTC_CHECK_GE(stride, width);
RTC_CHECK_GE(stddev, 0.0);
if (stddev == 0.0) {
return data[row * stride + column];
}
const double kCutoff = 0.2;
const int kMaxDistance =
std::ceil(sqrt(-2.0 * std::log(kCutoff) * std::pow(stddev, 2.0))) - 1;
RTC_CHECK_GE(kMaxDistance, 0);
if (kMaxDistance == 0) {
return data[row * stride + column];
}
double element_sum = 0.0;
double total_weight = 0.0;
for (int r = std::max(row - kMaxDistance, 0);
r < std::min(row + kMaxDistance + 1, height); ++r) {
for (int c = std::max(column - kMaxDistance, 0);
c < std::min(column + kMaxDistance + 1, width); ++c) {
double weight =
std::exp(-1.0 * (std::pow(row - r, 2) + std::pow(column - c, 2)) /
(2.0 * std::pow(stddev, 2)));
element_sum += data[r * stride + c] * weight;
total_weight += weight;
}
}
return element_sum / total_weight;
}
std::vector<FilteredSample> GetSampleValuesForFrame(
const scoped_refptr<I420BufferInterface> i420_frame_buffer,
std::vector<HaltonFrameSampler::Coordinates> sample_coordinates,
int scaled_width,
int scaled_height,
double stddev_gaussian_blur) {
// Validate input.
if (i420_frame_buffer == nullptr) {
RTC_LOG(LS_WARNING) << "The framebuffer must not be nullptr";
return {};
}
if (sample_coordinates.empty()) {
RTC_LOG(LS_WARNING) << "There must be at least one coordinate provided";
return {};
}
for (HaltonFrameSampler::Coordinates coordinate : sample_coordinates) {
if (coordinate.column < 0.0 || coordinate.column >= 1.0 ||
coordinate.row < 0.0 || coordinate.row >= 1.0) {
RTC_LOG(LS_WARNING) << "The coordinates must be in [0,1): column="
<< coordinate.column << ", row=" << coordinate.row
<< ".\n";
return {};
}
}
if (scaled_width <= 0 || scaled_height <= 0) {
RTC_LOG(LS_WARNING)
<< "The width and height to scale to must be positive: width="
<< scaled_width << ", height=" << scaled_height << ".\n";
return {};
}
if (stddev_gaussian_blur <= 0.0) {
RTC_LOG(LS_WARNING) << "The standard deviation for the Gaussian blur must "
"be larger than 0: "
<< stddev_gaussian_blur << ".\n";
return {};
}
// Scale the frame to the desired resolution:
// 1. Create a new buffer with the desired resolution.
// 2. Scale the old buffer to the size of the new buffer.
scoped_refptr<I420Buffer> scaled_i420_buffer =
I420Buffer::Create(scaled_width, scaled_height);
scaled_i420_buffer->ScaleFrom(*i420_frame_buffer);
// Treat the planes as if they would have the following 2-dimensional layout:
// +------+---+
// | | U |
// | Y +---+
// | | V |
// +------+---+
// where width:=(Y.width+U.width) and height:=Y.height.
// When interpreting the 2D sample coordinates, we simply treat them
// as if they were taken from the above layout. We then need to translate the
// coordinates back to the corresponding plane's corresponding 2D coordinates.
// Then we find the filtered value that corresponds to those coordinates.
int width_merged_planes =
scaled_i420_buffer->width() + scaled_i420_buffer->ChromaWidth();
int height_merged_planes = scaled_i420_buffer->height();
// Fetch the sample value for all of the requested coordinates.
std::vector<FilteredSample> filtered_samples;
filtered_samples.reserve(sample_coordinates.size());
for (HaltonFrameSampler::Coordinates coordinate : sample_coordinates) {
// Scale the coordinates from [0,1) to [0,`width_merged_planes`) and
// [0,`height_merged_planes`). Truncation is intentional.
int column = coordinate.column * width_merged_planes;
int row = coordinate.row * height_merged_planes;
// Map to plane coordinates and fetch the value.
double value_for_coordinate;
if (column < scaled_i420_buffer->width()) {
// Y plane.
value_for_coordinate = GetFilteredElement(
scaled_i420_buffer->width(), scaled_i420_buffer->height(),
scaled_i420_buffer->StrideY(), scaled_i420_buffer->DataY(), row,
column, stddev_gaussian_blur);
filtered_samples.push_back(
{.value = value_for_coordinate, .plane = ImagePlane::kLuma});
} else if (row < scaled_i420_buffer->ChromaHeight()) {
// U plane.
column -= scaled_i420_buffer->width();
value_for_coordinate = GetFilteredElement(
scaled_i420_buffer->ChromaWidth(), scaled_i420_buffer->ChromaHeight(),
scaled_i420_buffer->StrideU(), scaled_i420_buffer->DataU(), row,
column, stddev_gaussian_blur);
filtered_samples.push_back(
{.value = value_for_coordinate, .plane = ImagePlane::kChroma});
} else {
// V plane.
column -= scaled_i420_buffer->width();
row -= scaled_i420_buffer->ChromaHeight();
value_for_coordinate = GetFilteredElement(
scaled_i420_buffer->ChromaWidth(), scaled_i420_buffer->ChromaHeight(),
scaled_i420_buffer->StrideV(), scaled_i420_buffer->DataV(), row,
column, stddev_gaussian_blur);
filtered_samples.push_back(
{.value = value_for_coordinate, .plane = ImagePlane::kChroma});
}
}
return filtered_samples;
}
} // namespace webrtc