Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#ifdef XP_WIN
// Include Windows headers required for enabling high precision timers.
# include <windows.h>
# include <mmsystem.h>
#endif
#include "VideoSink.h"
#include "AudioDeviceInfo.h"
#include "MediaQueue.h"
#include "VideoUtils.h"
#include "mozilla/IntegerPrintfMacros.h"
#include "mozilla/ProfilerLabels.h"
#include "mozilla/ProfilerMarkerTypes.h"
#include "mozilla/StaticPrefs_browser.h"
#include "mozilla/StaticPrefs_media.h"
namespace mozilla {
extern LazyLogModule gMediaDecoderLog;
}
#undef FMT
#define FMT(x, ...) "VideoSink=%p " x, this, ##__VA_ARGS__
#define VSINK_LOG(x, ...) \
MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (FMT(x, ##__VA_ARGS__)))
#define VSINK_LOG_V(x, ...) \
MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, (FMT(x, ##__VA_ARGS__)))
namespace mozilla {
using namespace mozilla::layers;
// Minimum update frequency is 1/120th of a second, i.e. half the
// duration of a 60-fps frame.
static const int64_t MIN_UPDATE_INTERVAL_US = 1000000 / (60 * 2);
static void SetImageToGreenPixel(PlanarYCbCrImage* aImage) {
static uint8_t greenPixel[] = {0x00, 0x00, 0x00};
PlanarYCbCrData data;
data.mYChannel = greenPixel;
data.mCbChannel = greenPixel + 1;
data.mCrChannel = greenPixel + 2;
data.mYStride = data.mCbCrStride = 1;
data.mPictureRect = gfx::IntRect(0, 0, 1, 1);
data.mYUVColorSpace = gfx::YUVColorSpace::BT601;
aImage->CopyData(data);
}
VideoSink::VideoSink(AbstractThread* aThread, MediaSink* aAudioSink,
MediaQueue<VideoData>& aVideoQueue,
VideoFrameContainer* aContainer,
FrameStatistics& aFrameStats,
uint32_t aVQueueSentToCompositerSize)
: mOwnerThread(aThread),
mAudioSink(aAudioSink),
mVideoQueue(aVideoQueue),
mContainer(aContainer),
mProducerID(ImageContainer::AllocateProducerID()),
mFrameStats(aFrameStats),
mOldCompositorDroppedCount(mContainer ? mContainer->GetDroppedImageCount()
: 0),
mPendingDroppedCount(0),
mHasVideo(false),
mUpdateScheduler(aThread),
mVideoQueueSendToCompositorSize(aVQueueSentToCompositerSize)
#ifdef XP_WIN
,
mHiResTimersRequested(false)
#endif
{
MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
if (StaticPrefs::browser_measurement_render_anims_and_video_solid() &&
mContainer) {
InitializeBlankImage();
MOZ_ASSERT(mBlankImage, "Blank image should exist.");
}
}
VideoSink::~VideoSink() {
#ifdef XP_WIN
MOZ_ASSERT(!mHiResTimersRequested);
#endif
}
RefPtr<VideoSink::EndedPromise> VideoSink::OnEnded(TrackType aType) {
AssertOwnerThread();
MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
if (aType == TrackInfo::kAudioTrack) {
return mAudioSink->OnEnded(aType);
} else if (aType == TrackInfo::kVideoTrack) {
return mEndPromise;
}
return nullptr;
}
media::TimeUnit VideoSink::GetEndTime(TrackType aType) const {
AssertOwnerThread();
MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
if (aType == TrackInfo::kVideoTrack) {
return mVideoFrameEndTime;
} else if (aType == TrackInfo::kAudioTrack) {
return mAudioSink->GetEndTime(aType);
}
return media::TimeUnit::Zero();
}
media::TimeUnit VideoSink::GetPosition(TimeStamp* aTimeStamp) {
AssertOwnerThread();
return mAudioSink->GetPosition(aTimeStamp);
}
bool VideoSink::HasUnplayedFrames(TrackType aType) const {
AssertOwnerThread();
MOZ_ASSERT(aType == TrackInfo::kAudioTrack,
"Not implemented for non audio tracks.");
return mAudioSink->HasUnplayedFrames(aType);
}
media::TimeUnit VideoSink::UnplayedDuration(TrackType aType) const {
AssertOwnerThread();
MOZ_ASSERT(aType == TrackInfo::kAudioTrack,
"Not implemented for non audio tracks.");
return mAudioSink->UnplayedDuration(aType);
}
void VideoSink::SetPlaybackRate(double aPlaybackRate) {
AssertOwnerThread();
mAudioSink->SetPlaybackRate(aPlaybackRate);
}
void VideoSink::SetVolume(double aVolume) {
AssertOwnerThread();
mAudioSink->SetVolume(aVolume);
}
void VideoSink::SetStreamName(const nsAString& aStreamName) {
AssertOwnerThread();
mAudioSink->SetStreamName(aStreamName);
}
void VideoSink::SetPreservesPitch(bool aPreservesPitch) {
AssertOwnerThread();
mAudioSink->SetPreservesPitch(aPreservesPitch);
}
RefPtr<GenericPromise> VideoSink::SetAudioDevice(
RefPtr<AudioDeviceInfo> aDevice) {
return mAudioSink->SetAudioDevice(std::move(aDevice));
}
double VideoSink::PlaybackRate() const {
AssertOwnerThread();
return mAudioSink->PlaybackRate();
}
void VideoSink::EnsureHighResTimersOnOnlyIfPlaying() {
#ifdef XP_WIN
const bool needed = IsPlaying();
if (needed == mHiResTimersRequested) {
return;
}
if (needed) {
// Ensure high precision timers are enabled on Windows, otherwise the
// VideoSink isn't woken up at reliable intervals to set the next frame, and
// we drop frames while painting. Note that each call must be matched by a
// corresponding timeEndPeriod() call. Enabling high precision timers causes
// the CPU to wake up more frequently on Windows 7 and earlier, which causes
// more CPU load and battery use. So we only enable high precision timers
// when we're actually playing.
timeBeginPeriod(1);
} else {
timeEndPeriod(1);
}
mHiResTimersRequested = needed;
#endif
}
void VideoSink::SetPlaying(bool aPlaying) {
AssertOwnerThread();
VSINK_LOG_V(" playing (%d) -> (%d)", mAudioSink->IsPlaying(), aPlaying);
if (!aPlaying) {
// Reset any update timer if paused.
mUpdateScheduler.Reset();
// Since playback is paused, tell compositor to render only current frame.
TimeStamp nowTime;
const auto clockTime = mAudioSink->GetPosition(&nowTime);
RefPtr<VideoData> currentFrame = VideoQueue().PeekFront();
if (currentFrame) {
RenderVideoFrames(Span(¤tFrame, 1), clockTime.ToMicroseconds(),
nowTime);
}
if (mContainer) {
mContainer->ClearCachedResources();
}
if (mSecondaryContainer) {
mSecondaryContainer->ClearCachedResources();
}
}
mAudioSink->SetPlaying(aPlaying);
if (mHasVideo && aPlaying) {
// There's no thread in VideoSink for pulling video frames, need to trigger
// rendering while becoming playing status. because the VideoQueue may be
// full already.
TryUpdateRenderedVideoFrames();
}
EnsureHighResTimersOnOnlyIfPlaying();
}
nsresult VideoSink::Start(const media::TimeUnit& aStartTime,
const MediaInfo& aInfo) {
AssertOwnerThread();
VSINK_LOG("[%s]", __func__);
nsresult rv = mAudioSink->Start(aStartTime, aInfo);
mHasVideo = aInfo.HasVideo();
if (mHasVideo) {
mEndPromise = mEndPromiseHolder.Ensure(__func__);
// If the underlying MediaSink has an end promise for the video track (which
// happens when mAudioSink refers to a DecodedStream), we must wait for it
// to complete before resolving our own end promise. Otherwise, MDSM might
// stop playback before DecodedStream plays to the end and cause
// test_streams_element_capture.html to time out.
RefPtr<EndedPromise> p = mAudioSink->OnEnded(TrackInfo::kVideoTrack);
if (p) {
RefPtr<VideoSink> self = this;
p->Then(
mOwnerThread, __func__,
[self]() {
self->mVideoSinkEndRequest.Complete();
self->TryUpdateRenderedVideoFrames();
// It is possible the video queue size is 0 and we have no
// frames to render. However, we need to call
// MaybeResolveEndPromise() to ensure mEndPromiseHolder is
// resolved.
self->MaybeResolveEndPromise();
},
[self]() {
self->mVideoSinkEndRequest.Complete();
self->TryUpdateRenderedVideoFrames();
self->MaybeResolveEndPromise();
})
->Track(mVideoSinkEndRequest);
}
ConnectListener();
// Run the render loop at least once so we can resolve the end promise
// when video duration is 0.
UpdateRenderedVideoFrames();
}
return rv;
}
void VideoSink::Stop() {
AssertOwnerThread();
MOZ_ASSERT(mAudioSink->IsStarted(), "playback not started.");
VSINK_LOG("[%s]", __func__);
mAudioSink->Stop();
mUpdateScheduler.Reset();
if (mHasVideo) {
DisconnectListener();
mVideoSinkEndRequest.DisconnectIfExists();
mEndPromiseHolder.ResolveIfExists(true, __func__);
mEndPromise = nullptr;
}
mVideoFrameEndTime = media::TimeUnit::Zero();
EnsureHighResTimersOnOnlyIfPlaying();
}
bool VideoSink::IsStarted() const {
AssertOwnerThread();
return mAudioSink->IsStarted();
}
bool VideoSink::IsPlaying() const {
AssertOwnerThread();
return mAudioSink->IsPlaying();
}
void VideoSink::Shutdown() {
AssertOwnerThread();
MOZ_ASSERT(!mAudioSink->IsStarted(), "must be called after playback stops.");
VSINK_LOG("[%s]", __func__);
mAudioSink->Shutdown();
}
void VideoSink::OnVideoQueuePushed(RefPtr<VideoData>&& aSample) {
AssertOwnerThread();
// Listen to push event, VideoSink should try rendering ASAP if first frame
// arrives but update scheduler is not triggered yet.
if (!aSample->IsSentToCompositor()) {
// Since we push rendered frames back to the queue, we will receive
// push events for them. We only need to trigger render loop
// when this frame is not rendered yet.
TryUpdateRenderedVideoFrames();
}
}
void VideoSink::OnVideoQueueFinished() {
AssertOwnerThread();
// Run render loop if the end promise is not resolved yet.
if (!mUpdateScheduler.IsScheduled() && mAudioSink->IsPlaying() &&
!mEndPromiseHolder.IsEmpty()) {
UpdateRenderedVideoFrames();
}
}
void VideoSink::Redraw(const VideoInfo& aInfo) {
AUTO_PROFILER_LABEL("VideoSink::Redraw", MEDIA_PLAYBACK);
AssertOwnerThread();
// No video track, nothing to draw.
if (!aInfo.IsValid() || !mContainer) {
return;
}
auto now = TimeStamp::Now();
RefPtr<VideoData> video = VideoQueue().PeekFront();
if (video) {
if (mBlankImage) {
video->mImage = mBlankImage;
}
video->MarkSentToCompositor();
mContainer->SetCurrentFrame(video->mDisplay, video->mImage, now,
media::TimeUnit::Invalid(), video->mTime);
if (mSecondaryContainer) {
mSecondaryContainer->SetCurrentFrame(video->mDisplay, video->mImage, now,
media::TimeUnit::Invalid(),
video->mTime);
}
return;
}
// When we reach here, it means there are no frames in this video track.
// Draw a blank frame to ensure there is something in the image container
// to fire 'loadeddata'.
RefPtr<Image> blank =
mContainer->GetImageContainer()->CreatePlanarYCbCrImage();
mContainer->SetCurrentFrame(aInfo.mDisplay, blank, now,
media::TimeUnit::Invalid(),
media::TimeUnit::Invalid());
if (mSecondaryContainer) {
mSecondaryContainer->SetCurrentFrame(aInfo.mDisplay, blank, now,
media::TimeUnit::Invalid(),
media::TimeUnit::Invalid());
}
}
void VideoSink::TryUpdateRenderedVideoFrames() {
AUTO_PROFILER_LABEL("VideoSink::TryUpdateRenderedVideoFrames",
MEDIA_PLAYBACK);
AssertOwnerThread();
if (mUpdateScheduler.IsScheduled() || !mAudioSink->IsPlaying()) {
return;
}
RefPtr<VideoData> v = VideoQueue().PeekFront();
if (!v) {
// No frames to render.
return;
}
TimeStamp nowTime;
const media::TimeUnit clockTime = mAudioSink->GetPosition(&nowTime);
if (clockTime >= v->mTime) {
// Time to render this frame.
UpdateRenderedVideoFrames();
return;
}
// If we send this future frame to the compositor now, it will be rendered
// immediately and break A/V sync. Instead, we schedule a timer to send it
// later.
int64_t delta =
(v->mTime - clockTime).ToMicroseconds() / mAudioSink->PlaybackRate();
TimeStamp target = nowTime + TimeDuration::FromMicroseconds(delta);
RefPtr<VideoSink> self = this;
mUpdateScheduler.Ensure(
target, [self]() { self->UpdateRenderedVideoFramesByTimer(); },
[self]() { self->UpdateRenderedVideoFramesByTimer(); });
}
void VideoSink::UpdateRenderedVideoFramesByTimer() {
AssertOwnerThread();
mUpdateScheduler.CompleteRequest();
UpdateRenderedVideoFrames();
}
void VideoSink::ConnectListener() {
AssertOwnerThread();
mPushListener = VideoQueue().PushEvent().Connect(
mOwnerThread, this, &VideoSink::OnVideoQueuePushed);
mFinishListener = VideoQueue().FinishEvent().Connect(
mOwnerThread, this, &VideoSink::OnVideoQueueFinished);
}
void VideoSink::DisconnectListener() {
AssertOwnerThread();
mPushListener.Disconnect();
mFinishListener.Disconnect();
}
void VideoSink::RenderVideoFrames(Span<const RefPtr<VideoData>> aFrames,
int64_t aClockTime,
const TimeStamp& aClockTimeStamp) {
AUTO_PROFILER_LABEL("VideoSink::RenderVideoFrames", MEDIA_PLAYBACK);
AssertOwnerThread();
if (aFrames.IsEmpty() || !mContainer) {
return;
}
PROFILER_MARKER("VideoSink::RenderVideoFrames", MEDIA_PLAYBACK, {},
VideoSinkRenderMarker, aClockTime);
AutoTArray<ImageContainer::NonOwningImage, 16> images;
TimeStamp lastFrameTime;
double playbackRate = mAudioSink->PlaybackRate();
for (uint32_t i = 0; i < aFrames.Length(); ++i) {
VideoData* frame = aFrames[i];
bool wasSent = frame->IsSentToCompositor();
frame->MarkSentToCompositor();
if (!frame->mImage || !frame->mImage->IsValid() ||
!frame->mImage->GetSize().width || !frame->mImage->GetSize().height) {
continue;
}
if (frame->mTime.IsNegative()) {
// Frame times before the start time are invalid; drop such frames
continue;
}
MOZ_ASSERT(!aClockTimeStamp.IsNull());
int64_t delta = frame->mTime.ToMicroseconds() - aClockTime;
TimeStamp t =
aClockTimeStamp + TimeDuration::FromMicroseconds(delta / playbackRate);
if (!lastFrameTime.IsNull() && t <= lastFrameTime) {
// Timestamps out of order; drop the new frame. In theory we should
// probably replace the previous frame with the new frame if the
// timestamps are equal, but this is a corrupt video file already so
// never mind.
continue;
}
MOZ_ASSERT(!t.IsNull());
lastFrameTime = t;
ImageContainer::NonOwningImage* img = images.AppendElement();
img->mTimeStamp = t;
img->mImage = frame->mImage;
if (mBlankImage) {
img->mImage = mBlankImage;
}
img->mFrameID = frame->mFrameID;
img->mProducerID = mProducerID;
img->mMediaTime = frame->mTime;
VSINK_LOG_V("playing video frame %" PRId64
" (id=%x, vq-queued=%zu, clock=%" PRId64 ")",
frame->mTime.ToMicroseconds(), frame->mFrameID,
VideoQueue().GetSize(), aClockTime);
if (!wasSent) {
PROFILER_MARKER("PlayVideo", MEDIA_PLAYBACK, {}, MediaSampleMarker,
frame->mTime.ToMicroseconds(),
frame->GetEndTime().ToMicroseconds(),
VideoQueue().GetSize());
}
}
if (images.Length() > 0) {
mContainer->SetCurrentFrames(aFrames[0]->mDisplay, images);
if (mSecondaryContainer) {
mSecondaryContainer->SetCurrentFrames(aFrames[0]->mDisplay, images);
}
}
}
void VideoSink::UpdateRenderedVideoFrames() {
AUTO_PROFILER_LABEL("VideoSink::UpdateRenderedVideoFrames", MEDIA_PLAYBACK);
AssertOwnerThread();
MOZ_ASSERT(mAudioSink->IsPlaying(), "should be called while playing.");
// Get the current playback position.
TimeStamp nowTime;
const auto clockTime = mAudioSink->GetPosition(&nowTime);
MOZ_ASSERT(!clockTime.IsNegative(), "Should have positive clock time.");
uint32_t sentToCompositorCount = 0;
uint32_t droppedInSink = 0;
// Skip frames up to the playback position.
// At least the last frame is retained, even when out of date, because it
// will be used if no more frames are received before the queue finishes or
// the video is paused.
RefPtr<VideoData> lastExpiredFrameInCompositor;
while (VideoQueue().GetSize() > 1 &&
clockTime >= VideoQueue().PeekFront()->GetEndTime()) {
RefPtr<VideoData> frame = VideoQueue().PopFront();
if (frame->IsSentToCompositor()) {
lastExpiredFrameInCompositor = frame;
sentToCompositorCount++;
} else {
droppedInSink++;
mDroppedInSinkSequenceDuration += frame->mDuration;
VSINK_LOG_V("discarding video frame mTime=%" PRId64
" clock_time=%" PRId64,
frame->mTime.ToMicroseconds(), clockTime.ToMicroseconds());
struct VideoSinkDroppedFrameMarker {
static constexpr Span<const char> MarkerTypeName() {
return MakeStringSpan("VideoSinkDroppedFrame");
}
static void StreamJSONMarkerData(
baseprofiler::SpliceableJSONWriter& aWriter,
int64_t aSampleStartTimeUs, int64_t aSampleEndTimeUs,
int64_t aClockTimeUs) {
aWriter.IntProperty("sampleStartTimeUs", aSampleStartTimeUs);
aWriter.IntProperty("sampleEndTimeUs", aSampleEndTimeUs);
aWriter.IntProperty("clockTimeUs", aClockTimeUs);
}
static MarkerSchema MarkerTypeDisplay() {
using MS = MarkerSchema;
MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable};
schema.AddKeyLabelFormat("sampleStartTimeUs", "Sample start time",
MS::Format::Microseconds);
schema.AddKeyLabelFormat("sampleEndTimeUs", "Sample end time",
MS::Format::Microseconds);
schema.AddKeyLabelFormat("clockTimeUs", "Audio clock time",
MS::Format::Microseconds);
return schema;
}
};
profiler_add_marker(
"VideoSinkDroppedFrame", geckoprofiler::category::MEDIA_PLAYBACK, {},
VideoSinkDroppedFrameMarker{}, frame->mTime.ToMicroseconds(),
frame->GetEndTime().ToMicroseconds(), clockTime.ToMicroseconds());
}
}
if (droppedInSink || sentToCompositorCount) {
uint32_t totalCompositorDroppedCount = mContainer->GetDroppedImageCount();
uint32_t droppedInCompositor =
totalCompositorDroppedCount - mOldCompositorDroppedCount;
if (droppedInCompositor > 0) {
mOldCompositorDroppedCount = totalCompositorDroppedCount;
VSINK_LOG_V("%u video frame previously discarded by compositor",
droppedInCompositor);
}
mPendingDroppedCount += droppedInCompositor;
uint32_t droppedReported = mPendingDroppedCount > sentToCompositorCount
? sentToCompositorCount
: mPendingDroppedCount;
mPendingDroppedCount -= droppedReported;
mFrameStats.Accumulate({0, 0, sentToCompositorCount - droppedReported, 0,
droppedInSink, droppedInCompositor});
}
AutoTArray<RefPtr<VideoData>, 16> frames;
RefPtr<VideoData> currentFrame = VideoQueue().PeekFront();
if (currentFrame) {
// The presentation end time of the last video frame consumed is the end
// time of the current frame.
mVideoFrameEndTime =
std::max(mVideoFrameEndTime, currentFrame->GetEndTime());
// Gecko doesn't support VideoPlaybackQuality.totalFrameDelay
// would not be reported to content. If frames are late, then throttle
// the number of frames sent to the compositor, so that the
// droppedVideoFrames are reported. Perhaps the reduced number of frames
// composited might free up some resources for decode.
if ( // currentFrame is on time, or almost so, or
currentFrame->GetEndTime() >= clockTime ||
// there is only one frame in the VideoQueue() because the current
// frame would have otherwise been removed above. Send this frame if
// it has already been sent to the compositor because it has not been
// dropped and sending it again now, without any preceding frames, will
// drop references to any preceding frames and update the intrinsic
// size on the VideoFrameContainer.
currentFrame->IsSentToCompositor() ||
// Send this frame if its lateness is less than the duration that has
// been skipped for throttling, or
clockTime - currentFrame->GetEndTime() <
mDroppedInSinkSequenceDuration ||
// in a talos test for the compositor, which requires that the most
// recently decoded frame is passed to the compositor so that the
// compositor has something to composite during the talos test when the
// decode is stressed.
StaticPrefs::media_ruin_av_sync_enabled()) {
mDroppedInSinkSequenceDuration = media::TimeUnit::Zero();
VideoQueue().GetFirstElements(
std::max(2u, mVideoQueueSendToCompositorSize), &frames);
} else if (lastExpiredFrameInCompositor) {
// Release references to all but the last frame passed to the
// compositor. Passing this frame to RenderVideoFrames() as the first
// in frames also updates the intrinsic size on the VideoFrameContainer
// to that of this frame.
frames.AppendElement(lastExpiredFrameInCompositor);
}
RenderVideoFrames(Span(frames.Elements(),
std::min<size_t>(frames.Length(),
mVideoQueueSendToCompositorSize)),
clockTime.ToMicroseconds(), nowTime);
}
MaybeResolveEndPromise();
// Get the timestamp of the next frame. Schedule the next update at
// the start time of the next frame. If we don't have a next frame,
// we will run render loops again upon incoming frames.
if (frames.Length() < 2) {
return;
}
int64_t nextFrameTime = frames[1]->mTime.ToMicroseconds();
int64_t delta = std::max(nextFrameTime - clockTime.ToMicroseconds(),
MIN_UPDATE_INTERVAL_US);
TimeStamp target = nowTime + TimeDuration::FromMicroseconds(
delta / mAudioSink->PlaybackRate());
RefPtr<VideoSink> self = this;
mUpdateScheduler.Ensure(
target, [self]() { self->UpdateRenderedVideoFramesByTimer(); },
[self]() { self->UpdateRenderedVideoFramesByTimer(); });
}
void VideoSink::MaybeResolveEndPromise() {
AssertOwnerThread();
// All frames are rendered, Let's resolve the promise.
if (VideoQueue().IsFinished() && VideoQueue().GetSize() <= 1 &&
!mVideoSinkEndRequest.Exists()) {
TimeStamp nowTime;
const auto clockTime = mAudioSink->GetPosition(&nowTime);
if (VideoQueue().GetSize() == 1) {
// The last frame is no longer required in the VideoQueue().
RefPtr<VideoData> frame = VideoQueue().PopFront();
// Ensure that the last frame and its dimensions have been set on the
// VideoFrameContainer, even if the frame was decoded late. This also
// removes references to any other frames currently held by the
// VideoFrameContainer.
RenderVideoFrames(Span(&frame, 1), clockTime.ToMicroseconds(), nowTime);
if (mPendingDroppedCount > 0) {
mFrameStats.Accumulate({0, 0, 0, 0, 0, 1});
mPendingDroppedCount--;
} else {
mFrameStats.NotifyPresentedFrame();
}
}
if (clockTime < mVideoFrameEndTime) {
VSINK_LOG_V(
"Not reach video end time yet, reschedule timer to resolve "
"end promise. clockTime=%" PRId64 ", endTime=%" PRId64,
clockTime.ToMicroseconds(), mVideoFrameEndTime.ToMicroseconds());
int64_t delta = (mVideoFrameEndTime - clockTime).ToMicroseconds() /
mAudioSink->PlaybackRate();
TimeStamp target = nowTime + TimeDuration::FromMicroseconds(delta);
auto resolveEndPromise = [self = RefPtr<VideoSink>(this)]() {
self->mEndPromiseHolder.ResolveIfExists(true, __func__);
self->mUpdateScheduler.CompleteRequest();
};
mUpdateScheduler.Ensure(target, std::move(resolveEndPromise),
std::move(resolveEndPromise));
} else {
mEndPromiseHolder.ResolveIfExists(true, __func__);
}
}
}
void VideoSink::SetSecondaryVideoContainer(VideoFrameContainer* aSecondary) {
AssertOwnerThread();
// Clear all images of secondary ImageContainer, when it is removed from
// VideoSink.
if (mSecondaryContainer && aSecondary != mSecondaryContainer) {
ImageContainer* secondaryImageContainer =
mSecondaryContainer->GetImageContainer();
secondaryImageContainer->ClearAllImages();
}
mSecondaryContainer = aSecondary;
if (!IsPlaying() && mSecondaryContainer) {
ImageContainer* mainImageContainer = mContainer->GetImageContainer();
ImageContainer* secondaryImageContainer =
mSecondaryContainer->GetImageContainer();
MOZ_DIAGNOSTIC_ASSERT(mainImageContainer);
MOZ_DIAGNOSTIC_ASSERT(secondaryImageContainer);
// If the video isn't currently playing, get the current frame and display
// that in the secondary container as well.
AutoLockImage lockImage(mainImageContainer);
TimeStamp now = TimeStamp::Now();
if (const auto* owningImage = lockImage.GetOwningImage(now)) {
AutoTArray<ImageContainer::NonOwningImage, 1> currentFrame;
currentFrame.AppendElement(ImageContainer::NonOwningImage(
owningImage->mImage, now, /* frameID */ 1,
/* producerId */ ImageContainer::AllocateProducerID(),
owningImage->mProcessingDuration, owningImage->mMediaTime,
owningImage->mWebrtcCaptureTime, owningImage->mWebrtcReceiveTime,
owningImage->mRtpTimestamp));
secondaryImageContainer->SetCurrentImages(currentFrame);
}
}
}
void VideoSink::GetDebugInfo(dom::MediaSinkDebugInfo& aInfo) {
AssertOwnerThread();
aInfo.mVideoSink.mIsStarted = IsStarted();
aInfo.mVideoSink.mIsPlaying = IsPlaying();
aInfo.mVideoSink.mFinished = VideoQueue().IsFinished();
aInfo.mVideoSink.mSize = VideoQueue().GetSize();
aInfo.mVideoSink.mVideoFrameEndTime = mVideoFrameEndTime.ToMicroseconds();
aInfo.mVideoSink.mHasVideo = mHasVideo;
aInfo.mVideoSink.mVideoSinkEndRequestExists = mVideoSinkEndRequest.Exists();
aInfo.mVideoSink.mEndPromiseHolderIsEmpty = mEndPromiseHolder.IsEmpty();
mAudioSink->GetDebugInfo(aInfo);
}
bool VideoSink::InitializeBlankImage() {
mBlankImage = mContainer->GetImageContainer()->CreatePlanarYCbCrImage();
if (mBlankImage == nullptr) {
return false;
}
SetImageToGreenPixel(mBlankImage->AsPlanarYCbCrImage());
return true;
}
} // namespace mozilla