Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
#ifndef MEDIASTREAMTRACK_H_
#define MEDIASTREAMTRACK_H_
#include "MediaTrackConstraints.h"
#include "PrincipalChangeObserver.h"
#include "PrincipalHandle.h"
#include "mozilla/DOMEventTargetHelper.h"
#include "mozilla/dom/MediaStreamTrackBinding.h"
#include "mozilla/dom/MediaTrackCapabilitiesBinding.h"
#include "mozilla/dom/MediaTrackSettingsBinding.h"
#include "mozilla/media/MediaUtils.h"
#include "mozilla/WeakPtr.h"
#include "nsError.h"
#include "nsID.h"
#include "nsIPrincipal.h"
#include "PerformanceRecorder.h"
namespace mozilla {
class DOMMediaStream;
class MediaEnginePhotoCallback;
class MediaInputPort;
class MediaTrack;
class MediaTrackGraph;
class MediaTrackGraphImpl;
class MediaTrackListener;
class DirectMediaTrackListener;
class PeerConnectionImpl;
class PeerIdentity;
class ProcessedMediaTrack;
class RemoteSourceStreamInfo;
class SourceStreamInfo;
class MediaMgrError;
namespace dom {
class AudioStreamTrack;
class VideoStreamTrack;
class RTCStatsTimestampMaker;
enum class CallerType : uint32_t;
/**
* Common interface through which a MediaStreamTrack can communicate with its
* producer on the main thread.
*
* Kept alive by a strong ref in all MediaStreamTracks (original and clones)
* sharing this source.
*/
class MediaStreamTrackSource : public nsISupports {
NS_DECL_CYCLE_COLLECTING_ISUPPORTS
NS_DECL_CYCLE_COLLECTION_CLASS(MediaStreamTrackSource)
public:
class Sink : public SupportsWeakPtr {
public:
/**
* Must be constant throughout the Sink's lifetime.
*
* Return true to keep the MediaStreamTrackSource where this sink is
* registered alive.
* Return false to allow the source to stop.
*
* Typically MediaStreamTrack::Sink returns true and other Sinks
* (like HTMLMediaElement::StreamCaptureTrackSource) return false.
*/
virtual bool KeepsSourceAlive() const = 0;
/**
* Return true to ensure that the MediaStreamTrackSource where this Sink is
* registered is kept turned on and active.
* Return false to allow the source to pause, and any underlying devices to
* temporarily stop.
*
* When the underlying enabled state of the sink changes,
* call MediaStreamTrackSource::SinkEnabledStateChanged().
*
* Typically MediaStreamTrack returns the track's enabled state and other
* Sinks (like HTMLMediaElement::StreamCaptureTrackSource) return false so
* control over device state remains with tracks and their enabled state.
*/
virtual bool Enabled() const = 0;
/**
* Called when the principal of the MediaStreamTrackSource where this sink
* is registered has changed.
*/
virtual void PrincipalChanged() = 0;
/**
* Called when the muted state of the MediaStreamTrackSource where this sink
* is registered has changed.
*/
virtual void MutedChanged(bool aNewState) = 0;
/**
* Called when the MediaStreamTrackSource where this sink is registered has
* stopped producing data for good, i.e., it has ended.
*/
virtual void OverrideEnded() = 0;
protected:
virtual ~Sink() = default;
};
MediaStreamTrackSource(nsIPrincipal* aPrincipal, const nsString& aLabel,
TrackingId aTrackingId)
: mPrincipal(aPrincipal),
mLabel(aLabel),
mTrackingId(std::move(aTrackingId)),
mStopped(false) {}
/**
* Use to clean up any resources that have to be cleaned before the
* destructor is called. It is often too late in the destructor because
* of garbage collection having removed the members already.
*/
virtual void Destroy() {}
/**
* Gets the source's MediaSourceEnum for usage by PeerConnections.
*/
virtual MediaSourceEnum GetMediaSource() const = 0;
/**
* Get this TrackSource's principal.
*/
nsIPrincipal* GetPrincipal() const { return mPrincipal; }
/**
* This is used in WebRTC. A peerIdentity constrained MediaStreamTrack cannot
* be sent across the network to anything other than a peer with the provided
* identity. If this is set, then GetPrincipal() should return an instance of
* NullPrincipal.
*
* A track's PeerIdentity is immutable and will not change during the track's
* lifetime.
*/
virtual const PeerIdentity* GetPeerIdentity() const { return nullptr; }
/**
* This is used in WebRTC. The timestampMaker can convert between different
* timestamp types used during the session.
*/
virtual const RTCStatsTimestampMaker* GetTimestampMaker() const {
return nullptr;
}
/**
* MediaStreamTrack::GetLabel (see spec) calls through to here.
*/
void GetLabel(nsAString& aLabel) { aLabel.Assign(mLabel); }
/**
* Whether this TrackSource provides video frames with an alpha channel. Only
* applies to video sources. Used by HTMLVideoElement.
*/
virtual bool HasAlpha() const { return false; }
/**
* Forwards a photo request to backends that support it. Other backends return
* NS_ERROR_NOT_IMPLEMENTED to indicate that a MediaTrackGraph-based fallback
* should be used.
*/
virtual nsresult TakePhoto(MediaEnginePhotoCallback*) const {
return NS_ERROR_NOT_IMPLEMENTED;
}
typedef MozPromise<bool /* aIgnored */, RefPtr<MediaMgrError>, true>
ApplyConstraintsPromise;
/**
* We provide a fallback solution to ApplyConstraints() here.
* Sources that support ApplyConstraints() will have to override it.
*/
virtual RefPtr<ApplyConstraintsPromise> ApplyConstraints(
const dom::MediaTrackConstraints& aConstraints, CallerType aCallerType);
/**
* Same for GetSettings (no-op).
*/
virtual void GetSettings(dom::MediaTrackSettings& aResult) {};
virtual void GetCapabilities(dom::MediaTrackCapabilities& aResult) {};
/**
* Called by the source interface when all registered sinks with
* KeepsSourceAlive() == true have unregistered.
*/
virtual void Stop() = 0;
/**
* Called by the source interface when all registered sinks with
* KeepsSourceAlive() == true become disabled.
*/
virtual void Disable() = 0;
/**
* Called by the source interface when at least one registered sink with
* KeepsSourceAlive() == true become enabled.
*/
virtual void Enable() = 0;
/**
* Called when a Sink's Enabled() state changed. Will iterate through all
* sinks and notify the source of the aggregated enabled state.
*
* Note that a Sink with KeepsSourceAlive() == false counts as disabled.
*/
void SinkEnabledStateChanged() {
if (IsEnabled()) {
Enable();
} else {
Disable();
}
}
/**
* Called by each MediaStreamTrack clone on initialization.
*/
void RegisterSink(Sink* aSink) {
MOZ_ASSERT(NS_IsMainThread());
if (mStopped) {
return;
}
mSinks.AppendElement(aSink);
while (mSinks.RemoveElement(nullptr)) {
MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
}
}
/**
* Called by each MediaStreamTrack clone on Stop() if supported by the
* source (us) or destruction.
*/
void UnregisterSink(Sink* aSink) {
MOZ_ASSERT(NS_IsMainThread());
while (mSinks.RemoveElement(nullptr)) {
MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
}
if (mSinks.RemoveElement(aSink) && !IsActive()) {
MOZ_ASSERT(!aSink->KeepsSourceAlive() || !mStopped,
"When the last sink keeping the source alive is removed, "
"we should still be live");
Stop();
mStopped = true;
}
if (!mStopped) {
SinkEnabledStateChanged();
}
}
protected:
virtual ~MediaStreamTrackSource() = default;
bool IsActive() {
for (const WeakPtr<Sink>& sink : mSinks) {
if (sink && sink->KeepsSourceAlive()) {
return true;
}
}
return false;
}
bool IsEnabled() {
for (const WeakPtr<Sink>& sink : mSinks) {
if (sink && sink->KeepsSourceAlive() && sink->Enabled()) {
return true;
}
}
return false;
}
/**
* Called by a sub class when the principal has changed.
* Notifies all sinks.
*/
void PrincipalChanged() {
MOZ_ASSERT(NS_IsMainThread());
for (auto& sink : mSinks.Clone()) {
if (!sink) {
DebugOnly<bool> removed = mSinks.RemoveElement(sink);
MOZ_ASSERT(!removed, "Sink was not explicitly removed");
continue;
}
sink->PrincipalChanged();
}
}
/**
* Called by a sub class when the source's muted state has changed. Note that
* the source is responsible for making the content black/silent during mute.
* Notifies all sinks.
*/
void MutedChanged(bool aNewState) {
MOZ_ASSERT(NS_IsMainThread());
for (auto& sink : mSinks.Clone()) {
if (!sink) {
DebugOnly<bool> removed = mSinks.RemoveElement(sink);
MOZ_ASSERT(!removed, "Sink was not explicitly removed");
continue;
}
sink->MutedChanged(aNewState);
}
}
/**
* Called by a sub class when the source has stopped producing data for good,
* i.e., it has ended. Notifies all sinks.
*/
void OverrideEnded() {
MOZ_ASSERT(NS_IsMainThread());
for (auto& sink : mSinks.Clone()) {
if (!sink) {
DebugOnly<bool> removed = mSinks.RemoveElement(sink);
MOZ_ASSERT(!removed, "Sink was not explicitly removed");
continue;
}
sink->OverrideEnded();
}
}
// Principal identifying who may access the contents of this source.
RefPtr<nsIPrincipal> mPrincipal;
// Currently registered sinks.
nsTArray<WeakPtr<Sink>> mSinks;
public:
// The label of the track we are the source of per the MediaStreamTrack spec.
const nsString mLabel;
// Set for all video sources; an id for tracking the source of the video
// frames for this track.
const TrackingId mTrackingId;
protected:
// True if all MediaStreamTrack users have unregistered from this source and
// Stop() has been called.
bool mStopped;
};
/**
* Base class that consumers of a MediaStreamTrack can use to get notifications
* about state changes in the track.
*/
class MediaStreamTrackConsumer : public SupportsWeakPtr {
public:
/**
* Called when the track's readyState transitions to "ended".
* Unlike the "ended" event exposed to script this is called for any reason,
* including MediaStreamTrack::Stop().
*/
virtual void NotifyEnded(MediaStreamTrack* aTrack) {};
/**
* Called when the track's enabled state changes.
*/
virtual void NotifyEnabledChanged(MediaStreamTrack* aTrack, bool aEnabled) {};
};
// clang-format off
/**
* DOM wrapper for MediaTrackGraph-MediaTracks.
*
* To account for cloning, a MediaStreamTrack wraps two internal (and chained)
* MediaTracks:
* 1. mInputTrack
* - Controlled by the producer of the data in the track. The producer
* decides on lifetime of the MediaTrack and the track inside it.
* - It can be any type of MediaTrack.
* - Contains one track only.
* 2. mTrack
* - A ForwardedInputTrack representing this MediaStreamTrack.
* - Its data is piped from mInputTrack through mPort.
* - Contains one track only.
* - When this MediaStreamTrack is enabled/disabled this is reflected in
* the chunks in the track in mTrack.
* - When this MediaStreamTrack has ended, mTrack gets destroyed.
* Note that mInputTrack is unaffected, such that any clones of mTrack
* can live on. When all clones are ended, this is signaled to the
* producer via our MediaStreamTrackSource. It is then likely to destroy
* mInputTrack.
*
* A graphical representation of how tracks are connected when cloned follows:
*
* MediaStreamTrack A
* mInputTrack mTrack
* t1 ---------> t1
* \
* -----
* MediaStreamTrack B \ (clone of A)
* mInputTrack \ mTrack
* * -> t1
*
* (*) is a copy of A's mInputTrack
*/
// clang-format on
class MediaStreamTrack : public DOMEventTargetHelper, public SupportsWeakPtr {
// PeerConnection and friends need to know our owning DOMStream and track id.
friend class mozilla::PeerConnectionImpl;
friend class mozilla::SourceStreamInfo;
friend class mozilla::RemoteSourceStreamInfo;
class MTGListener;
class TrackSink;
public:
MediaStreamTrack(
nsPIDOMWindowInner* aWindow, mozilla::MediaTrack* aInputTrack,
MediaStreamTrackSource* aSource,
MediaStreamTrackState aReadyState = MediaStreamTrackState::Live,
bool aMuted = false,
const MediaTrackConstraints& aConstraints = MediaTrackConstraints());
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaStreamTrack,
DOMEventTargetHelper)
nsPIDOMWindowInner* GetParentObject() const { return mWindow; }
JSObject* WrapObject(JSContext* aCx,
JS::Handle<JSObject*> aGivenProto) override;
virtual AudioStreamTrack* AsAudioStreamTrack() { return nullptr; }
virtual VideoStreamTrack* AsVideoStreamTrack() { return nullptr; }
virtual const AudioStreamTrack* AsAudioStreamTrack() const { return nullptr; }
virtual const VideoStreamTrack* AsVideoStreamTrack() const { return nullptr; }
// WebIDL
virtual void GetKind(nsAString& aKind) = 0;
void GetId(nsAString& aID) const;
virtual void GetLabel(nsAString& aLabel, CallerType /* aCallerType */) {
GetSource().GetLabel(aLabel);
}
bool Enabled() const { return mEnabled; }
void SetEnabled(bool aEnabled);
bool Muted() { return mMuted; }
void Stop();
void GetCapabilities(MediaTrackCapabilities& aResult, CallerType aCallerType);
void GetConstraints(dom::MediaTrackConstraints& aResult);
void GetSettings(dom::MediaTrackSettings& aResult, CallerType aCallerType);
already_AddRefed<Promise> ApplyConstraints(
const dom::MediaTrackConstraints& aConstraints, CallerType aCallerType,
ErrorResult& aRv);
already_AddRefed<MediaStreamTrack> Clone();
MediaStreamTrackState ReadyState() { return mReadyState; }
IMPL_EVENT_HANDLER(mute)
IMPL_EVENT_HANDLER(unmute)
IMPL_EVENT_HANDLER(ended)
/**
* Convenience (and legacy) method for when ready state is "ended".
*/
bool Ended() const { return mReadyState == MediaStreamTrackState::Ended; }
/**
* Get this track's principal.
*/
nsIPrincipal* GetPrincipal() const { return mPrincipal; }
/**
* Get this track's PeerIdentity.
*/
const PeerIdentity* GetPeerIdentity() const {
return GetSource().GetPeerIdentity();
}
/**
* Get this track's RTCStatsTimestampMaker.
*/
const RTCStatsTimestampMaker* GetTimestampMaker() const {
return GetSource().GetTimestampMaker();
}
ProcessedMediaTrack* GetTrack() const;
MediaTrackGraph* Graph() const;
MediaTrackGraphImpl* GraphImpl() const;
MediaStreamTrackSource& GetSource() const {
MOZ_RELEASE_ASSERT(mSource,
"The track source is only removed on destruction");
return *mSource;
}
// Webrtc allows the remote side to name tracks whatever it wants, and we
// need to surface this to content.
void AssignId(const nsAString& aID) { mID = aID; }
/**
* Add a PrincipalChangeObserver to this track.
*
* Returns true if it was successfully added.
*
* Ownership of the PrincipalChangeObserver remains with the caller, and it's
* the caller's responsibility to remove the observer before it dies.
*/
bool AddPrincipalChangeObserver(
PrincipalChangeObserver<MediaStreamTrack>* aObserver);
/**
* Remove an added PrincipalChangeObserver from this track.
*
* Returns true if it was successfully removed.
*/
bool RemovePrincipalChangeObserver(
PrincipalChangeObserver<MediaStreamTrack>* aObserver);
/**
* Add a MediaStreamTrackConsumer to this track.
*
* Adding the same consumer multiple times is prohibited.
*/
void AddConsumer(MediaStreamTrackConsumer* aConsumer);
/**
* Remove an added MediaStreamTrackConsumer from this track.
*/
void RemoveConsumer(MediaStreamTrackConsumer* aConsumer);
/**
* Adds a MediaTrackListener to the MediaTrackGraph representation of
* this track.
*/
virtual void AddListener(MediaTrackListener* aListener);
/**
* Removes a MediaTrackListener from the MediaTrackGraph representation
* of this track.
*/
void RemoveListener(MediaTrackListener* aListener);
/**
* Attempts to add a direct track listener to this track.
* Callers must listen to the NotifyInstalled event to know if installing
* the listener succeeded (tracks originating from SourceMediaTracks) or
* failed (e.g., WebAudio originated tracks).
*/
virtual void AddDirectListener(DirectMediaTrackListener* aListener);
void RemoveDirectListener(DirectMediaTrackListener* aListener);
/**
* Sets up a MediaInputPort from the underlying track that this
* MediaStreamTrack represents, to aTrack, and returns it.
*/
already_AddRefed<MediaInputPort> ForwardTrackContentsTo(
ProcessedMediaTrack* aTrack);
protected:
virtual ~MediaStreamTrack();
/**
* Forces the ready state to a particular value, for instance when we're
* cloning an already ended track.
*/
void SetReadyState(MediaStreamTrackState aState);
/**
* Notified by the MediaTrackGraph, through our owning MediaStream on the
* main thread.
*
* Note that this sets the track to ended and raises the "ended" event
* synchronously.
*/
void OverrideEnded();
/**
* Called by the MTGListener when this track's PrincipalHandle changes on
* the MediaTrackGraph thread. When the PrincipalHandle matches the pending
* principal we know that the principal change has propagated to consumers.
*/
void NotifyPrincipalHandleChanged(const PrincipalHandle& aNewPrincipalHandle);
/**
* Called when this track's readyState transitions to "ended".
* Notifies all MediaStreamTrackConsumers that this track ended.
*/
void NotifyEnded();
/**
* Called when this track's enabled state has changed.
* Notifies all MediaStreamTrackConsumers.
*/
void NotifyEnabledChanged();
/**
* Called when mSource's principal has changed.
*/
void PrincipalChanged();
/**
* Called when mSource's muted state has changed.
*/
void MutedChanged(bool aNewState);
/**
* Sets this track's muted state without raising any events.
* Only really set by cloning. See MutedChanged for runtime changes.
*/
void SetMuted(bool aMuted) { mMuted = aMuted; }
virtual void Destroy();
/**
* Sets the principal and notifies PrincipalChangeObservers if it changes.
*/
void SetPrincipal(nsIPrincipal* aPrincipal);
/**
* Creates a new MediaStreamTrack with the same kind, input track, input
* track ID and source as this MediaStreamTrack.
*/
virtual already_AddRefed<MediaStreamTrack> CloneInternal() = 0;
nsTArray<PrincipalChangeObserver<MediaStreamTrack>*>
mPrincipalChangeObservers;
nsTArray<WeakPtr<MediaStreamTrackConsumer>> mConsumers;
// We need this to track our parent object.
nsCOMPtr<nsPIDOMWindowInner> mWindow;
// The input MediaTrack assigned us by the data producer.
// Owned by the producer.
const RefPtr<mozilla::MediaTrack> mInputTrack;
// The MediaTrack representing this MediaStreamTrack in the MediaTrackGraph.
// Set on construction if we're live. Valid until we end. Owned by us.
RefPtr<ProcessedMediaTrack> mTrack;
// The MediaInputPort connecting mInputTrack to mTrack. Set on construction
// if mInputTrack is non-destroyed and we're live. Valid until we end. Owned
// by us.
RefPtr<MediaInputPort> mPort;
RefPtr<MediaStreamTrackSource> mSource;
const UniquePtr<TrackSink> mSink;
nsCOMPtr<nsIPrincipal> mPrincipal;
nsCOMPtr<nsIPrincipal> mPendingPrincipal;
RefPtr<MTGListener> mMTGListener;
// Keep tracking MediaTrackListener and DirectMediaTrackListener,
// so we can remove them in |Destory|.
nsTArray<RefPtr<MediaTrackListener>> mTrackListeners;
nsTArray<RefPtr<DirectMediaTrackListener>> mDirectTrackListeners;
nsString mID;
MediaStreamTrackState mReadyState;
bool mEnabled;
bool mMuted;
dom::MediaTrackConstraints mConstraints;
};
} // namespace dom
} // namespace mozilla
#endif /* MEDIASTREAMTRACK_H_ */