Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
#ifndef TrackEncoder_h_
#define TrackEncoder_h_
#include "AudioSegment.h"
#include "EncodedFrame.h"
#include "MediaQueue.h"
#include "MediaTrackGraph.h"
#include "TrackMetadataBase.h"
#include "VideoSegment.h"
namespace mozilla {
class AbstractThread;
class DriftCompensator;
class TrackEncoder;
class TrackEncoderListener {
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackEncoderListener)
/**
* Called when the TrackEncoder has received its first real data.
*/
virtual void Started(TrackEncoder* aEncoder) = 0;
/**
* Called when the TrackEncoder's underlying encoder has been successfully
* initialized and there's non-null data ready to be encoded.
*/
virtual void Initialized(TrackEncoder* aEncoder) = 0;
/**
* Called after the TrackEncoder hit an unexpected error, causing it to
* abort operation.
*/
virtual void Error(TrackEncoder* aEncoder) = 0;
protected:
virtual ~TrackEncoderListener() = default;
};
/**
* Base class of AudioTrackEncoder and VideoTrackEncoder. Lifetime managed by
* MediaEncoder. All methods are to be called only on the worker thread.
*
* The control APIs are all called by MediaEncoder on its dedicated thread. Data
* is encoded as soon as it has been appended (and time has advanced past its
* end in case of video) and pushed to mEncodedDataQueue.
*/
class TrackEncoder {
public:
TrackEncoder(TrackRate aTrackRate,
MediaQueue<EncodedFrame>& aEncodedDataQueue);
/**
* Called by MediaEncoder to cancel the encoding.
*/
virtual void Cancel() = 0;
/**
* Notifies us that we have reached the end of the stream and no more data
* will be appended.
*/
virtual void NotifyEndOfStream() = 0;
/**
* Creates and sets up meta data for a specific codec, called on the worker
* thread.
*/
virtual already_AddRefed<TrackMetadataBase> GetMetadata() = 0;
/**
* MediaQueue containing encoded data, that is pushed as soon as it's ready.
*/
MediaQueue<EncodedFrame>& EncodedDataQueue() { return mEncodedDataQueue; }
/**
* Returns true once this TrackEncoder is initialized.
*/
bool IsInitialized();
/**
* Returns true once this TrackEncoder has received some data.
*/
bool IsStarted();
/**
* True if the track encoder has encoded all source segments coming from
* MediaTrackGraph. Call on the worker thread.
*/
bool IsEncodingComplete() const;
/**
* Registers a listener to events from this TrackEncoder.
* We hold a strong reference to the listener.
*/
void RegisterListener(TrackEncoderListener* aListener);
/**
* Unregisters a listener from events from this TrackEncoder.
* The listener will stop receiving events synchronously.
*/
bool UnregisterListener(TrackEncoderListener* aListener);
virtual void SetBitrate(const uint32_t aBitrate) = 0;
/**
* It's optional to set the worker thread, but if you do we'll assert that
* we are in the worker thread in every method that gets called.
*/
void SetWorkerThread(AbstractThread* aWorkerThread);
/**
* Measure size of internal buffers.
*/
virtual size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) = 0;
protected:
virtual ~TrackEncoder() { MOZ_ASSERT(mListeners.IsEmpty()); }
/**
* If this TrackEncoder was not already initialized, it is set to initialized
* and listeners are notified.
*/
void SetInitialized();
/**
* If this TrackEncoder was not already marked started, its started state is
* set and listeners are notified.
*/
void SetStarted();
/**
* Called after an error. Cancels the encoding and notifies listeners.
*/
void OnError();
/**
* True if the track encoder has been initialized successfully.
*/
bool mInitialized;
/**
* True if the track encoder has received data.
*/
bool mStarted;
/**
* True once all data until the end of the input track has been received.
*/
bool mEndOfStream;
/**
* True once this encoding has been cancelled.
*/
bool mCanceled;
// How many times we have tried to initialize the encoder.
uint32_t mInitCounter;
/**
* True if this TrackEncoder is currently suspended.
*/
bool mSuspended;
/**
* The track rate of source media.
*/
const TrackRate mTrackRate;
/**
* If set we assert that all methods are called on this thread.
*/
RefPtr<AbstractThread> mWorkerThread;
/**
* MediaQueue where encoded data ends up. Note that metadata goes out of band.
*/
MediaQueue<EncodedFrame>& mEncodedDataQueue;
nsTArray<RefPtr<TrackEncoderListener>> mListeners;
};
class AudioTrackEncoder : public TrackEncoder {
public:
AudioTrackEncoder(TrackRate aTrackRate,
MediaQueue<EncodedFrame>& aEncodedDataQueue)
: TrackEncoder(aTrackRate, aEncodedDataQueue),
mChannels(0),
mNotInitDuration(0),
mAudioBitrate(0) {}
/**
* Suspends encoding from now, i.e., all future audio data received through
* AppendAudioSegment() until the next Resume() will be dropped.
*/
void Suspend();
/**
* Resumes encoding starting now, i.e., data from the next
* AppendAudioSegment() will get encoded.
*/
void Resume();
/**
* Appends and consumes track data from aSegment.
*/
void AppendAudioSegment(AudioSegment&& aSegment);
template <typename T>
static void InterleaveTrackData(nsTArray<const T*>& aInput, int32_t aDuration,
uint32_t aOutputChannels,
AudioDataValue* aOutput, float aVolume) {
if (aInput.Length() < aOutputChannels) {
// Up-mix. This might make the mChannelData have more than aChannels.
AudioChannelsUpMix(&aInput, aOutputChannels,
SilentChannel::ZeroChannel<T>());
}
if (aInput.Length() > aOutputChannels) {
DownmixAndInterleave<T>(aInput, aDuration, aVolume, aOutputChannels,
aOutput);
} else {
InterleaveAndConvertBuffer(aInput.Elements(), aDuration, aVolume,
aOutputChannels, aOutput);
}
}
/**
* Interleaves the track data and stores the result into aOutput. Might need
* to up-mix or down-mix the channel data if the channels number of this chunk
* is different from aOutputChannels. The channel data from aChunk might be
* modified by up-mixing.
*/
static void InterleaveTrackData(AudioChunk& aChunk, int32_t aDuration,
uint32_t aOutputChannels,
AudioDataValue* aOutput);
/**
* De-interleaves the aInput data and stores the result into aOutput.
* No up-mix or down-mix operations inside.
*/
static void DeInterleaveTrackData(AudioDataValue* aInput, int32_t aDuration,
int32_t aChannels, AudioDataValue* aOutput);
/**
* Measure size of internal buffers.
*/
size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;
void SetBitrate(const uint32_t aBitrate) override {
mAudioBitrate = aBitrate;
}
/**
* Tries to initiate the AudioEncoder based on data in aSegment.
* This can be re-called often, as it will exit early should we already be
* initiated. mInitiated will only be set if there was enough data in
* aSegment to infer metadata. If mInitiated gets set, listeners are notified.
*
* Not having enough data in aSegment to initiate the encoder for an
* accumulated aDuration of one second will make us initiate with a default
* number of channels.
*
* If we attempt to initiate the underlying encoder but fail, we Cancel() and
* notify listeners.
*/
void TryInit(const AudioSegment& aSegment, TrackTime aDuration);
void Cancel() override;
/**
* Dispatched from MediaTrackGraph when we have finished feeding data to
* mOutgoingBuffer.
*/
void NotifyEndOfStream() override;
protected:
/**
* Number of samples per channel in a pcm buffer. This is also the value of
* frame size required by audio encoder, and listeners will be notified when
* at least this much data has been added to mOutgoingBuffer.
*/
virtual int NumInputFramesPerPacket() const { return 0; }
/**
* Initializes the audio encoder. The call of this method is delayed until we
* have received the first valid track from MediaTrackGraph.
*/
virtual nsresult Init(int aChannels) = 0;
/**
* Encodes buffered data and pushes it to mEncodedDataQueue.
*/
virtual nsresult Encode(AudioSegment* aSegment) = 0;
/**
* The number of channels are used for processing PCM data in the audio
* encoder. This value comes from the first valid audio chunk. If encoder
* can't support the channels in the chunk, downmix PCM stream can be
* performed. This value also be used to initialize the audio encoder.
*/
int mChannels;
/**
* A segment queue of outgoing audio track data to the encoder.
* The contents of mOutgoingBuffer will always be what has been appended on
* the encoder thread but not yet consumed by the encoder sub class.
*/
AudioSegment mOutgoingBuffer;
TrackTime mNotInitDuration;
uint32_t mAudioBitrate;
};
enum class FrameDroppingMode {
ALLOW, // Allowed to drop frames to keep up under load
DISALLOW, // Must not drop any frames, even if it means we will OOM
};
class VideoTrackEncoder : public TrackEncoder {
public:
VideoTrackEncoder(RefPtr<DriftCompensator> aDriftCompensator,
TrackRate aTrackRate,
MediaQueue<EncodedFrame>& aEncodedDataQueue,
FrameDroppingMode aFrameDroppingMode);
/**
* Suspends encoding from aTime, i.e., all video frame with a timestamp
* between aTime and the timestamp of the next Resume() will be dropped.
*/
void Suspend(const TimeStamp& aTime);
/**
* Resumes encoding starting at aTime.
*/
void Resume(const TimeStamp& aTime);
/**
* Makes the video black from aTime.
*/
void Disable(const TimeStamp& aTime);
/**
* Makes the video non-black from aTime.
*
* NB that it could still be forced black for other reasons, like principals.
*/
void Enable(const TimeStamp& aTime);
/**
* Appends source video frames to mIncomingBuffer. We only append the source
* chunk if the image is different from mLastChunk's image. Called on the
* MediaTrackGraph thread.
*/
void AppendVideoSegment(VideoSegment&& aSegment);
/**
* Measure size of internal buffers.
*/
size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;
void SetBitrate(const uint32_t aBitrate) override {
mVideoBitrate = aBitrate;
}
/**
* Tries to initiate the VideoEncoder based on data in aSegment.
* This can be re-called often, as it will exit early should we already be
* initiated. mInitiated will only be set if there was enough data in
* aSegment to infer metadata. If mInitiated gets set, listeners are notified.
* The amount of chunks needed can be controlled by
* aFrameRateDetectionMinChunks which denotes the minimum number of chunks
* needed to infer the framerate.
*
* Failing to initiate the encoder for an accumulated aDuration of 30 seconds
* is seen as an error and will cancel the current encoding.
*/
void Init(const VideoSegment& aSegment, const TimeStamp& aTime,
size_t aFrameRateDetectionMinChunks);
TrackTime SecondsToMediaTime(double aS) const {
NS_ASSERTION(0 <= aS && aS <= TRACK_TICKS_MAX / TRACK_RATE_MAX,
"Bad seconds");
return mTrackRate * aS;
}
/**
* MediaTrackGraph notifies us about the time of the track's start.
* This gets called on the MediaEncoder thread after a dispatch.
*/
void SetStartOffset(const TimeStamp& aStartOffset);
void Cancel() override;
/**
* Notifies us that we have reached the end of the stream and no more data
* will be appended to mIncomingBuffer.
*/
void NotifyEndOfStream() override;
/**
* Dispatched from MediaTrackGraph when it has run an iteration so we can
* hand more data to the encoder.
*/
void AdvanceCurrentTime(const TimeStamp& aTime);
protected:
/**
* Initialize the video encoder. In order to collect the value of width and
* height of source frames, this initialization is delayed until we have
* received the first valid video frame from MediaTrackGraph.
* Listeners will be notified after it has been successfully initialized.
*/
virtual nsresult Init(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
int32_t aDisplayHeight, float aEstimatedFrameRate) = 0;
/**
* Encodes data in the outgoing buffer and pushes it to mEncodedDataQueue.
*/
virtual nsresult Encode(VideoSegment* aSegment) = 0;
/**
* Drift compensator for re-clocking incoming video frame wall-clock
* timestamps to audio time.
*/
const RefPtr<DriftCompensator> mDriftCompensator;
/**
* The last unique frame and duration so far handled by
* NotifyAdvanceCurrentTime. When a new frame is detected, mLastChunk is added
* to mOutgoingBuffer.
*/
VideoChunk mLastChunk;
/**
* A segment queue of incoming video track data, from listeners.
* The duration of mIncomingBuffer is irrelevant as we only look at TimeStamps
* of frames. Consumed data is replaced by null data.
*/
VideoSegment mIncomingBuffer;
/**
* A segment queue of outgoing video track data to the encoder.
* The contents of mOutgoingBuffer will always be what has been consumed from
* mIncomingBuffer (up to mCurrentTime) but not yet consumed by the encoder
* sub class. There won't be any null data at the beginning of mOutgoingBuffer
* unless explicitly pushed by the producer.
*/
VideoSegment mOutgoingBuffer;
/**
* The number of mTrackRate ticks we have passed to mOutgoingBuffer.
*/
TrackTime mEncodedTicks;
/**
* The time up to which we have forwarded data from mIncomingBuffer to
* mOutgoingBuffer.
*/
TimeStamp mCurrentTime;
/**
* The time the video track started, so the start of the video track can be
* synced to the start of the audio track.
*
* Note that this time will progress during suspension, to make sure the
* incoming frames stay in sync with the output.
*/
TimeStamp mStartTime;
/**
* The time Suspend was called on the MediaRecorder, so we can calculate the
* duration on the next Resume().
*/
TimeStamp mSuspendTime;
uint32_t mVideoBitrate;
/**
* ALLOW to drop frames under load.
* DISALLOW to encode all frames, mainly for testing.
*/
FrameDroppingMode mFrameDroppingMode;
/**
* True if the video MediaTrackTrack this VideoTrackEncoder is attached to is
* currently enabled. While false, we encode all frames as black.
*/
bool mEnabled;
};
} // namespace mozilla
#endif