Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
#ifndef VideoOutput_h
#define VideoOutput_h
#include "MediaTrackListener.h"
#include "VideoFrameContainer.h"
namespace mozilla {
static bool SetImageToBlackPixel(layers::PlanarYCbCrImage* aImage) {
uint8_t blackPixel[] = {0x10, 0x80, 0x80};
layers::PlanarYCbCrData data;
data.mYChannel = blackPixel;
data.mCbChannel = blackPixel + 1;
data.mCrChannel = blackPixel + 2;
data.mYStride = data.mCbCrStride = 1;
data.mPictureRect = gfx::IntRect(0, 0, 1, 1);
data.mYUVColorSpace = gfx::YUVColorSpace::BT601;
// 0x00, 0x80, 0x80
data.mColorRange = gfx::ColorRange::LIMITED;
return NS_SUCCEEDED(aImage->CopyData(data));
}
class VideoOutput : public DirectMediaTrackListener {
protected:
typedef layers::Image Image;
typedef layers::ImageContainer ImageContainer;
typedef layers::ImageContainer::FrameID FrameID;
typedef layers::ImageContainer::ProducerID ProducerID;
virtual ~VideoOutput() = default;
void DropPastFrames() {
TimeStamp now = TimeStamp::Now();
size_t nrChunksInPast = 0;
for (const auto& idChunkPair : mFrames) {
const VideoChunk& chunk = idChunkPair.second;
if (chunk.mTimeStamp > now) {
break;
}
++nrChunksInPast;
}
if (nrChunksInPast > 1) {
// We need to keep one frame that starts in the past, because it only ends
// when the next frame starts (which also needs to be in the past for it
// to drop).
mFrames.RemoveElementsAt(0, nrChunksInPast - 1);
}
}
void SendFramesEnsureLocked() {
mMutex.AssertCurrentThreadOwns();
SendFrames();
}
void SendFrames() {
DropPastFrames();
if (mFrames.IsEmpty()) {
return;
}
if (!mEnabled && mDisabledBlackImageSent) {
return;
}
// Collect any new frames produced in this iteration.
AutoTArray<ImageContainer::NonOwningImage, 16> images;
PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
for (const auto& idChunkPair : mFrames) {
ImageContainer::FrameID frameId = idChunkPair.first;
const VideoChunk& chunk = idChunkPair.second;
const VideoFrame& frame = chunk.mFrame;
Image* image = frame.GetImage();
if (frame.GetForceBlack() || !mEnabled) {
if (!mBlackImage) {
RefPtr<Image> blackImage = mVideoFrameContainer->GetImageContainer()
->CreatePlanarYCbCrImage();
if (blackImage) {
// Sets the image to a single black pixel, which will be scaled to
// fill the rendered size.
if (SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage())) {
mBlackImage = blackImage;
}
}
}
if (mBlackImage) {
image = mBlackImage;
}
}
if (!image) {
// We ignore null images.
continue;
}
ImageContainer::NonOwningImage nonOwningImage(
image, chunk.mTimeStamp, frameId, mProducerID,
chunk.mProcessingDuration, chunk.mMediaTime, chunk.mWebrtcCaptureTime,
chunk.mWebrtcReceiveTime, chunk.mRtpTimestamp);
images.AppendElement(std::move(nonOwningImage));
lastPrincipalHandle = chunk.GetPrincipalHandle();
if (!mEnabled && mBlackImage) {
MOZ_ASSERT(images.Length() == 1);
mDisabledBlackImageSent = true;
break;
}
}
if (images.IsEmpty()) {
// This could happen if the only images in mFrames are null. We leave the
// container at the current frame in this case.
mVideoFrameContainer->ClearFutureFrames();
return;
}
bool principalHandleChanged =
lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
lastPrincipalHandle != mVideoFrameContainer->GetLastPrincipalHandle();
if (principalHandleChanged) {
mVideoFrameContainer->UpdatePrincipalHandleForFrameID(
lastPrincipalHandle, images.LastElement().mFrameID);
}
mVideoFrameContainer->SetCurrentFrames(
mFrames[0].second.mFrame.GetIntrinsicSize(), images);
mMainThread->Dispatch(NewRunnableMethod("VideoFrameContainer::Invalidate",
mVideoFrameContainer,
&VideoFrameContainer::Invalidate));
}
FrameID NewFrameID() {
mMutex.AssertCurrentThreadOwns();
return ++mFrameID;
}
public:
VideoOutput(VideoFrameContainer* aContainer, AbstractThread* aMainThread)
: mMutex("VideoOutput::mMutex"),
mVideoFrameContainer(aContainer),
mMainThread(aMainThread) {}
void NotifyRealtimeTrackData(MediaTrackGraph* aGraph, TrackTime aTrackOffset,
const MediaSegment& aMedia) override {
MOZ_ASSERT(aMedia.GetType() == MediaSegment::VIDEO);
const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
MutexAutoLock lock(mMutex);
for (VideoSegment::ConstChunkIterator i(video); !i.IsEnded(); i.Next()) {
if (!mLastFrameTime.IsNull() && i->mTimeStamp < mLastFrameTime) {
// Time can go backwards if the source is a captured MediaDecoder and
// it seeks, as the previously buffered frames would stretch into the
// future. If this happens, we clear the buffered frames and start over.
mFrames.ClearAndRetainStorage();
}
mFrames.AppendElement(std::make_pair(NewFrameID(), *i));
mLastFrameTime = i->mTimeStamp;
}
SendFramesEnsureLocked();
}
void NotifyRemoved(MediaTrackGraph* aGraph) override {
// Doesn't need locking by mMutex, since the direct listener is removed from
// the track before we get notified.
if (mFrames.Length() <= 1) {
// The compositor has already received the last frame.
mFrames.ClearAndRetainStorage();
mVideoFrameContainer->ClearFutureFrames();
return;
}
// The compositor has multiple frames. ClearFutureFrames() would only retain
// the first as that's normally the current one. We however stop doing
// SetCurrentFrames() once we've received the last frame in a track, so
// there might be old frames lingering. We'll find the current one and
// re-send that.
DropPastFrames();
mFrames.RemoveLastElements(mFrames.Length() - 1);
SendFrames();
mFrames.ClearAndRetainStorage();
}
void NotifyEnded(MediaTrackGraph* aGraph) override {
// Doesn't need locking by mMutex, since for the track to end, it must have
// been ended by the source, meaning that the source won't append more data.
if (mFrames.IsEmpty()) {
return;
}
// Re-send only the last one to the compositor.
mFrames.RemoveElementsAt(0, mFrames.Length() - 1);
SendFrames();
mFrames.ClearAndRetainStorage();
}
void NotifyEnabledStateChanged(MediaTrackGraph* aGraph,
bool aEnabled) override {
MutexAutoLock lock(mMutex);
mEnabled = aEnabled;
DropPastFrames();
if (mEnabled) {
mDisabledBlackImageSent = false;
}
if (!mEnabled || mFrames.Length() > 1) {
// Re-send frames when disabling, as new frames may not arrive. When
// enabling we keep them black until new frames arrive, or re-send if we
// already have frames in the future. If we're disabling and there are no
// frames available yet, we invent one. Unfortunately with a hardcoded
// size.
//
// Since mEnabled will affect whether
// frames are real, or black, we assign new FrameIDs whenever we re-send
// frames after an mEnabled change.
for (auto& idChunkPair : mFrames) {
idChunkPair.first = NewFrameID();
}
if (mFrames.IsEmpty()) {
VideoSegment v;
v.AppendFrame(nullptr, gfx::IntSize(640, 480), PRINCIPAL_HANDLE_NONE,
true, TimeStamp::Now());
mFrames.AppendElement(std::make_pair(NewFrameID(), *v.GetLastChunk()));
}
SendFramesEnsureLocked();
}
}
Mutex mMutex MOZ_UNANNOTATED;
TimeStamp mLastFrameTime;
// Once the frame is forced to black, we initialize mBlackImage for use in any
// following forced-black frames.
RefPtr<Image> mBlackImage;
// True once mBlackImage has been sent due to mEnabled being false.
bool mDisabledBlackImageSent = false;
bool mEnabled = true;
// This array is accessed from both the direct video thread, and the graph
// thread. Protected by mMutex.
nsTArray<std::pair<ImageContainer::FrameID, VideoChunk>> mFrames;
// Accessed from both the direct video thread, and the graph thread. Protected
// by mMutex.
FrameID mFrameID = 0;
const RefPtr<VideoFrameContainer> mVideoFrameContainer;
const RefPtr<AbstractThread> mMainThread;
const ProducerID mProducerID = ImageContainer::AllocateProducerID();
};
/**
* This listener observes the first video frame to arrive with a non-empty size,
* and renders it to its VideoFrameContainer.
*/
class FirstFrameVideoOutput : public VideoOutput {
public:
FirstFrameVideoOutput(VideoFrameContainer* aContainer,
AbstractThread* aMainThread)
: VideoOutput(aContainer, aMainThread) {
MOZ_ASSERT(NS_IsMainThread());
}
// NB that this overrides VideoOutput::NotifyRealtimeTrackData, so we can
// filter out all frames but the first one with a real size. This allows us to
// later re-use the logic in VideoOutput for rendering that frame.
void NotifyRealtimeTrackData(MediaTrackGraph* aGraph, TrackTime aTrackOffset,
const MediaSegment& aMedia) override {
MOZ_ASSERT(aMedia.GetType() == MediaSegment::VIDEO);
if (mInitialSizeFound) {
return;
}
const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0, 0)) {
mInitialSizeFound = true;
mMainThread->Dispatch(NS_NewRunnableFunction(
"FirstFrameVideoOutput::FirstFrameRenderedSetter",
[self = RefPtr<FirstFrameVideoOutput>(this)] {
self->mFirstFrameRendered = true;
}));
// Pick the first frame and run it through the rendering code.
VideoSegment segment;
segment.AppendFrame(*c);
VideoOutput::NotifyRealtimeTrackData(aGraph, aTrackOffset, segment);
return;
}
}
}
// Main thread only.
Watchable<bool> mFirstFrameRendered = {
false, "FirstFrameVideoOutput::mFirstFrameRendered"};
private:
// Whether a frame with a concrete size has been received. May only be
// accessed on the MTG's appending thread. (this is a direct listener so we
// get called by whoever is producing this track's data)
bool mInitialSizeFound = false;
};
} // namespace mozilla
#endif // VideoOutput_h