nsAVIFDecoder.h - mozsearch

mozilla-central/image/decoders/nsAVIFDecoder.h (file symbol)

Enable keyboard shortcuts

Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-

 * This Source Code Form is subject to the terms of the Mozilla Public

 * License, v. 2.0. If a copy of the MPL was not distributed with this

 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef mozilla_image_decoders_nsAVIFDecoder_h

#define mozilla_image_decoders_nsAVIFDecoder_h

#include "Decoder.h"

#include "mozilla/gfx/Types.h"

#include "MP4Metadata.h"

#include "mp4parse.h"

#include "SampleIterator.h"

#include "SurfacePipe.h"

#include <aom/aom_decoder.h>

#include "dav1d/dav1d.h"

#include "mozilla/Telemetry.h"

namespace mozilla {

namespace image {

class RasterImage;

class AVIFDecoderStream;

class AVIFParser;

class AVIFDecoderInterface;

class nsAVIFDecoder final : public Decoder {

 public:

  virtual ~nsAVIFDecoder();

  DecoderType GetType() const override { return DecoderType::AVIF; }

 protected:

  LexerResult DoDecode(SourceBufferIterator& aIterator,

                       IResumable* aOnResume) override;

  Maybe<Telemetry::HistogramID> SpeedHistogram() const override;

 private:

  friend class DecoderFactory;

  friend class AVIFDecoderInterface;

  friend class AVIFParser;

  // Decoders should only be instantiated via DecoderFactory.

  explicit nsAVIFDecoder(RasterImage* aImage);

  static intptr_t ReadSource(uint8_t* aDestBuf, uintptr_t aDestBufSize,

                             void* aUserData);

  typedef int Dav1dResult;

  enum class NonAOMCodecError { NoFrame, SizeOverflow };

  typedef Variant<aom_codec_err_t, NonAOMCodecError> AOMResult;

  enum class NonDecoderResult {

    NeedMoreData,

    OutputAvailable,

    Complete,

    SizeOverflow,

    OutOfMemory,

    PipeInitError,

    WriteBufferError,

    AlphaYSizeMismatch,

    AlphaYColorDepthMismatch,

    MetadataImageSizeMismatch,

    RenderSizeMismatch,

    FrameSizeChanged,

    InvalidCICP,

    NoSamples,

    ConvertYCbCrFailure,

};

  using DecodeResult =

      Variant<Mp4parseStatus, NonDecoderResult, Dav1dResult, AOMResult>;

  Mp4parseStatus CreateParser();

  DecodeResult CreateDecoder();

  DecodeResult DoDecodeInternal(SourceBufferIterator& aIterator,

                                IResumable* aOnResume);

  static bool IsDecodeSuccess(const DecodeResult& aResult);

  void RecordDecodeResultTelemetry(const DecodeResult& aResult);

  Vector<uint8_t> mBufferedData;

  RefPtr<AVIFDecoderStream> mBufferStream;

  /// Pointer to the next place to read from mBufferedData

  const uint8_t* mReadCursor = nullptr;

  UniquePtr<AVIFParser> mParser = nullptr;

  UniquePtr<AVIFDecoderInterface> mDecoder = nullptr;

  bool mIsAnimated = false;

  bool mHasAlpha = false;

  bool mUsePipeTransform = true;

};

class AVIFDecoderStream : public ByteStream {

 public:

  explicit AVIFDecoderStream(Vector<uint8_t>* aBuffer) { mBuffer = aBuffer; }

  virtual bool ReadAt(int64_t offset, void* data, size_t size,

                      size_t* bytes_read) override;

  virtual bool CachedReadAt(int64_t offset, void* data, size_t size,

                            size_t* bytes_read) override {

    return ReadAt(offset, data, size, bytes_read);

};

  virtual bool Length(int64_t* size) override;

  virtual const uint8_t* GetContiguousAccess(int64_t aOffset,

                                             size_t aSize) override;

 private:

  Vector<uint8_t>* mBuffer;

};

struct AVIFImage {

  uint32_t mFrameNum = 0;

  FrameTimeout mDuration = FrameTimeout::Zero();

  RefPtr<MediaRawData> mColorImage = nullptr;

  RefPtr<MediaRawData> mAlphaImage = nullptr;

};

class AVIFParser {

 public:

  static Mp4parseStatus Create(const Mp4parseIo* aIo, ByteStream* aBuffer,

                               UniquePtr<AVIFParser>& aParserOut,

                               bool aAllowSequences, bool aAnimateAVIFMajor);

  ~AVIFParser();

  const Mp4parseAvifInfo& GetInfo() const { return mInfo; }

  uint32_t GetFrameCount();

  nsAVIFDecoder::DecodeResult GetImage(AVIFImage& aImage);

  bool IsAnimated() const;

 private:

  explicit AVIFParser(const Mp4parseIo* aIo);

  Mp4parseStatus Init(ByteStream* aBuffer, bool aAllowSequences,

                      bool aAnimateAVIFMajor);

  struct FreeAvifParser {

    void operator()(Mp4parseAvifParser* aPtr) { mp4parse_avif_free(aPtr); }

};

  const Mp4parseIo* mIo;

  UniquePtr<Mp4parseAvifParser, FreeAvifParser> mParser = nullptr;

  Mp4parseAvifInfo mInfo = {};

  UniquePtr<SampleIterator> mColorSampleIter = nullptr;

  UniquePtr<SampleIterator> mAlphaSampleIter = nullptr;

  uint32_t mFrameNum = 0;

};

struct Dav1dPictureUnref {

  void operator()(Dav1dPicture* aPtr) {

    dav1d_picture_unref(aPtr);

    delete aPtr;

};

using OwnedDav1dPicture = UniquePtr<Dav1dPicture, Dav1dPictureUnref>;

class OwnedAOMImage {

 public:

  ~OwnedAOMImage();

  static OwnedAOMImage* CopyFrom(aom_image_t* aImage, bool aIsAlpha);

  aom_image_t* GetImage() { return mImage.isSome() ? mImage.ptr() : nullptr; }

 private:

  OwnedAOMImage();

  bool CloneFrom(aom_image_t* aImage, bool aIsAlpha);

  // The mImage's planes are referenced to mBuffer

  Maybe<aom_image_t> mImage;

  UniquePtr<uint8_t[]> mBuffer;

};

struct AVIFDecodedData : layers::PlanarYCbCrData {

 public:

  Maybe<OrientedIntSize> mRenderSize = Nothing();

  gfx::CICP::ColourPrimaries mColourPrimaries = gfx::CICP::CP_UNSPECIFIED;

  gfx::CICP::TransferCharacteristics mTransferCharacteristics =

      gfx::CICP::TC_UNSPECIFIED;

  gfx::CICP::MatrixCoefficients mMatrixCoefficients = gfx::CICP::MC_UNSPECIFIED;

  OwnedDav1dPicture mColorDav1d;

  OwnedDav1dPicture mAlphaDav1d;

  UniquePtr<OwnedAOMImage> mColorAOM;

  UniquePtr<OwnedAOMImage> mAlphaAOM;

  // CICP values (either from the BMFF container or the AV1 sequence header) are

  // used to create the colorspace transform. CICP::MatrixCoefficients is only

  // stored for the sake of telemetry, since the relevant information for YUV ->

  // RGB conversion is stored in mYUVColorSpace.

//

  // There are three potential sources of color information for an AVIF:

  // 1. ICC profile via a ColourInformationBox (colr) defined in [ISOBMFF]

  //    § 12.1.5 "Colour information" and [MIAF] § 7.3.6.4 "Colour information

  //    property"

  // 2. NCLX (AKA CICP see [ITU-T H.273]) values in the same

  // ColourInformationBox

  //    which can have an ICC profile or NCLX values, not both).

  // 3. NCLX values in the AV1 bitstream

//

  // The 'colr' box is optional, but there are always CICP values in the AV1

  // bitstream, so it is possible to have both. Per ISOBMFF § 12.1.5.1

  // > If colour information is supplied in both this box, and also in the

  // > video bitstream, this box takes precedence, and over-rides the

  // > information in the bitstream.

//

  // If present, the ICC profile takes precedence over CICP values, but only

  // specifies the color space, not the matrix coefficients necessary to convert

  // YCbCr data (as most AVIF are encoded) to RGB. The matrix coefficients are

  // always derived from the CICP values for matrix_coefficients (and

  // potentially colour_primaries, but in that case only the CICP values for

  // colour_primaries will be used, not anything harvested from the ICC

  // profile).

//

  // If there is no ICC profile, the color space transform will be based on the

  // CICP values either from the 'colr' box, or if absent/unspecified, the

  // decoded AV1 sequence header.

//

  // For values that are 2 (meaning unspecified) after trying both, the

  // fallback values are:

  // - CP:  1 (BT.709/sRGB)

  // - TC: 13 (sRGB)

  // - MC:  6 (BT.601)

  // - Range: Full

//

  // Additional details here:

  // <https://github.com/AOMediaCodec/libavif/wiki/CICP#unspecified>. Note

  // that this contradicts the current version of [MIAF] § 7.3.6.4 which

  // specifies MC=1 (BT.709). This is revised in [MIAF DAMD2] and confirmed by

  // <https://github.com/AOMediaCodec/av1-avif/issues/77#issuecomment-676526097>

//

  // The precedence for applying the various values and defaults in the event

  // no valid values are found are managed by the following functions.

//

  // References:

  // [ISOBMFF]: ISO/IEC 14496-12:2020 <https://www.iso.org/standard/74428.html>

  // [MIAF]: ISO/IEC 23000-22:2019 <https://www.iso.org/standard/74417.html>

  // [MIAF DAMD2]: ISO/IEC 23000-22:2019/FDAmd 2

  // <https://www.iso.org/standard/81634.html>

  // [ITU-T H.273]: Rec. ITU-T H.273 (12/2016)

  //     <https://www.itu.int/rec/T-REC-H.273-201612-I/en>

  void SetCicpValues(

      const Mp4parseNclxColourInformation* aNclx,

      const gfx::CICP::ColourPrimaries aAv1ColourPrimaries,

      const gfx::CICP::TransferCharacteristics aAv1TransferCharacteristics,

      const gfx::CICP::MatrixCoefficients aAv1MatrixCoefficients);

};

// An interface to do decode and get the decoded data

class AVIFDecoderInterface {

 public:

  using Dav1dResult = nsAVIFDecoder::Dav1dResult;

  using NonAOMCodecError = nsAVIFDecoder::NonAOMCodecError;

  using AOMResult = nsAVIFDecoder::AOMResult;

  using NonDecoderResult = nsAVIFDecoder::NonDecoderResult;

  using DecodeResult = nsAVIFDecoder::DecodeResult;

  virtual ~AVIFDecoderInterface() = default;

  // Set the mDecodedData if Decode() succeeds

  virtual DecodeResult Decode(bool aShouldSendTelemetry,

                              const Mp4parseAvifInfo& aAVIFInfo,

                              const AVIFImage& aSamples) = 0;

  // Must be called only once after Decode() succeeds

  UniquePtr<AVIFDecodedData> GetDecodedData() {

    MOZ_ASSERT(mDecodedData);

    return std::move(mDecodedData);

 protected:

  explicit AVIFDecoderInterface() = default;

  inline static bool IsDecodeSuccess(const DecodeResult& aResult) {

    return nsAVIFDecoder::IsDecodeSuccess(aResult);

  // The mDecodedData is valid after Decode() succeeds

  UniquePtr<AVIFDecodedData> mDecodedData;

};

}  // namespace image

}  // namespace mozilla

#endif  // mozilla_image_decoders_nsAVIFDecoder_h