/* * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #ifndef API_VIDEO_ENCODED_IMAGE_H_ #define API_VIDEO_ENCODED_IMAGE_H_ #include #include #include #include #include #include "api/ref_count.h" #include "api/rtp_packet_infos.h" #include "api/scoped_refptr.h" #include "api/units/timestamp.h" #include "api/video/color_space.h" #include "api/video/corruption_detection_filter_settings.h" #include "api/video/video_codec_constants.h" #include "api/video/video_content_type.h" #include "api/video/video_frame_type.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" #include "rtc_base/buffer.h" #include "rtc_base/checks.h" #include "rtc_base/system/rtc_export.h" namespace webrtc { // Abstract interface for buffer storage. Intended to support buffers owned by // external encoders with special release requirements, e.g, java encoders with // releaseOutputBuffer. class EncodedImageBufferInterface : public RefCountInterface { public: using value_type = uint8_t; virtual const uint8_t* data() const = 0; // TODO(bugs.webrtc.org/9378): Make interface essentially read-only, delete // this non-const data method. virtual uint8_t* data() = 0; virtual size_t size() const = 0; const uint8_t* begin() const { return data(); } const uint8_t* end() const { return data() + size(); } }; // Basic implementation of EncodedImageBufferInterface. class RTC_EXPORT EncodedImageBuffer : public EncodedImageBufferInterface { public: static scoped_refptr Create() { return Create(0); } static scoped_refptr Create(size_t size); static scoped_refptr Create(const uint8_t* data, size_t size); static scoped_refptr Create(rtc::Buffer buffer); const uint8_t* data() const override; uint8_t* data() override; size_t size() const override; void Realloc(size_t t); protected: explicit EncodedImageBuffer(size_t size); EncodedImageBuffer(const uint8_t* data, size_t size); explicit EncodedImageBuffer(rtc::Buffer buffer); rtc::Buffer buffer_; }; // TODO(bug.webrtc.org/9378): This is a legacy api class, which is slowly being // cleaned up. Direct use of its members is strongly discouraged. class RTC_EXPORT EncodedImage { public: EncodedImage(); EncodedImage(EncodedImage&&); EncodedImage(const EncodedImage&); ~EncodedImage(); EncodedImage& operator=(EncodedImage&&); EncodedImage& operator=(const EncodedImage&); // Frame capture time in RTP timestamp representation (90kHz). void SetRtpTimestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; } uint32_t RtpTimestamp() const { return timestamp_rtp_; } void SetEncodeTime(int64_t encode_start_ms, int64_t encode_finish_ms); // Frame capture time in local time. Timestamp CaptureTime() const; // Frame capture time in ntp epoch time, i.e. time since 1st Jan 1900 int64_t NtpTimeMs() const { return ntp_time_ms_; } // Every simulcast layer (= encoding) has its own encoder and RTP stream. // There can be no dependencies between different simulcast layers. std::optional SimulcastIndex() const { return simulcast_index_; } void SetSimulcastIndex(std::optional simulcast_index) { RTC_DCHECK_GE(simulcast_index.value_or(0), 0); RTC_DCHECK_LT(simulcast_index.value_or(0), kMaxSimulcastStreams); simulcast_index_ = simulcast_index; } const std::optional& PresentationTimestamp() const { return presentation_timestamp_; } void SetPresentationTimestamp( const std::optional& presentation_timestamp) { presentation_timestamp_ = presentation_timestamp; } // Encoded images can have dependencies between spatial and/or temporal // layers, depending on the scalability mode used by the encoder. See diagrams // at https://w3c.github.io/webrtc-svc/#dependencydiagrams*. std::optional SpatialIndex() const { return spatial_index_; } void SetSpatialIndex(std::optional spatial_index) { RTC_DCHECK_GE(spatial_index.value_or(0), 0); RTC_DCHECK_LT(spatial_index.value_or(0), kMaxSpatialLayers); spatial_index_ = spatial_index; } std::optional TemporalIndex() const { return temporal_index_; } void SetTemporalIndex(std::optional temporal_index) { RTC_DCHECK_GE(temporal_index_.value_or(0), 0); RTC_DCHECK_LT(temporal_index_.value_or(0), kMaxTemporalStreams); temporal_index_ = temporal_index; } // These methods can be used to set/get size of subframe with spatial index // `spatial_index` on encoded frames that consist of multiple spatial layers. std::optional SpatialLayerFrameSize(int spatial_index) const; void SetSpatialLayerFrameSize(int spatial_index, size_t size_bytes); const webrtc::ColorSpace* ColorSpace() const { return color_space_ ? &*color_space_ : nullptr; } void SetColorSpace(const std::optional& color_space) { color_space_ = color_space; } std::optional PlayoutDelay() const { return playout_delay_; } void SetPlayoutDelay(std::optional playout_delay) { playout_delay_ = playout_delay; } // These methods along with the private member video_frame_tracking_id_ are // meant for media quality testing purpose only. std::optional VideoFrameTrackingId() const { return video_frame_tracking_id_; } void SetVideoFrameTrackingId(std::optional tracking_id) { video_frame_tracking_id_ = tracking_id; } const RtpPacketInfos& PacketInfos() const { return packet_infos_; } void SetPacketInfos(RtpPacketInfos packet_infos) { packet_infos_ = std::move(packet_infos); } bool RetransmissionAllowed() const { return retransmission_allowed_; } void SetRetransmissionAllowed(bool retransmission_allowed) { retransmission_allowed_ = retransmission_allowed; } size_t size() const { return size_; } void set_size(size_t new_size) { // Allow set_size(0) even if we have no buffer. RTC_DCHECK_LE(new_size, new_size == 0 ? 0 : capacity()); size_ = new_size; } void SetEncodedData( rtc::scoped_refptr encoded_data) { encoded_data_ = encoded_data; size_ = encoded_data->size(); } void ClearEncodedData() { encoded_data_ = nullptr; size_ = 0; } rtc::scoped_refptr GetEncodedData() const { return encoded_data_; } const uint8_t* data() const { return encoded_data_ ? encoded_data_->data() : nullptr; } const uint8_t* begin() const { return data(); } const uint8_t* end() const { return data() + size(); } // Returns whether the encoded image can be considered to be of target // quality. [[deprecated]] bool IsAtTargetQuality() const { return at_target_quality_; } // Sets that the encoded image can be considered to be of target quality to // true or false. [[deprecated]] void SetAtTargetQuality(bool at_target_quality) { at_target_quality_ = at_target_quality; } // Returns whether the frame that was encoded is a steady-state refresh frame // intended to improve the visual quality. bool IsSteadyStateRefreshFrame() const { return is_steady_state_refresh_frame_; } void SetIsSteadyStateRefreshFrame(bool refresh_frame) { is_steady_state_refresh_frame_ = refresh_frame; } webrtc::VideoFrameType FrameType() const { return _frameType; } void SetFrameType(webrtc::VideoFrameType frame_type) { _frameType = frame_type; } VideoContentType contentType() const { return content_type_; } VideoRotation rotation() const { return rotation_; } std::optional corruption_detection_filter_settings() const { return corruption_detection_filter_settings_; } void set_corruption_detection_filter_settings( const CorruptionDetectionFilterSettings& settings) { corruption_detection_filter_settings_ = settings; } uint32_t _encodedWidth = 0; uint32_t _encodedHeight = 0; // NTP time of the capture time in local timebase in milliseconds. // TODO(minyue): make this member private. int64_t ntp_time_ms_ = 0; int64_t capture_time_ms_ = 0; VideoFrameType _frameType = VideoFrameType::kVideoFrameDelta; VideoRotation rotation_ = kVideoRotation_0; VideoContentType content_type_ = VideoContentType::UNSPECIFIED; int qp_ = -1; // Quantizer value. struct Timing { uint8_t flags = VideoSendTiming::kInvalid; int64_t encode_start_ms = 0; int64_t encode_finish_ms = 0; int64_t packetization_finish_ms = 0; int64_t pacer_exit_ms = 0; int64_t network_timestamp_ms = 0; int64_t network2_timestamp_ms = 0; int64_t receive_start_ms = 0; int64_t receive_finish_ms = 0; } timing_; EncodedImage::Timing video_timing() const { return timing_; } EncodedImage::Timing* video_timing_mutable() { return &timing_; } private: size_t capacity() const { return encoded_data_ ? encoded_data_->size() : 0; } // When set, indicates that all future frames will be constrained with those // limits until the application indicates a change again. std::optional playout_delay_; rtc::scoped_refptr encoded_data_; size_t size_ = 0; // Size of encoded frame data. uint32_t timestamp_rtp_ = 0; std::optional simulcast_index_; std::optional presentation_timestamp_; std::optional spatial_index_; std::optional temporal_index_; std::map spatial_layer_frame_size_bytes_; std::optional color_space_; // This field is meant for media quality testing purpose only. When enabled it // carries the webrtc::VideoFrame id field from the sender to the receiver. std::optional video_frame_tracking_id_; // Information about packets used to assemble this video frame. This is needed // by `SourceTracker` when the frame is delivered to the RTCRtpReceiver's // MediaStreamTrack, in order to implement getContributingSources(). See: // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources RtpPacketInfos packet_infos_; bool retransmission_allowed_ = true; // True if the encoded image can be considered to be of target quality. bool at_target_quality_ = false; // True if the frame that was encoded is a steady-state refresh frame intended // to improve the visual quality. bool is_steady_state_refresh_frame_ = false; // Filter settings for corruption detection suggested by the encoder // implementation, if any. Otherwise generic per-codec-type settings will be // used. std::optional corruption_detection_filter_settings_; }; } // namespace webrtc #endif // API_VIDEO_ENCODED_IMAGE_H_