diff options
Diffstat (limited to 'third_party/libwebrtc/webrtc/api/video')
12 files changed, 1117 insertions, 0 deletions
diff --git a/third_party/libwebrtc/webrtc/api/video/OWNERS b/third_party/libwebrtc/webrtc/api/video/OWNERS new file mode 100644 index 0000000000..8327124e23 --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/OWNERS @@ -0,0 +1,3 @@ +magjed@webrtc.org + +per-file video_timing.h=ilnik@webrtc.org diff --git a/third_party/libwebrtc/webrtc/api/video/i420_buffer.cc b/third_party/libwebrtc/webrtc/api/video/i420_buffer.cc new file mode 100644 index 0000000000..20af2c2289 --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/i420_buffer.cc @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/video/i420_buffer.h" + +#include <string.h> + +#include <algorithm> +#include <utility> + +#include "libyuv/convert.h" +#include "libyuv/planar_functions.h" +#include "libyuv/scale.h" +#include "rtc_base/checks.h" +#include "rtc_base/keep_ref_until_done.h" + +// Aligning pointer to 64 bytes for improved performance, e.g. use SIMD. +static const int kBufferAlignment = 64; + +namespace webrtc { + +namespace { + +int I420DataSize(int height, int stride_y, int stride_u, int stride_v) { + return stride_y * height + (stride_u + stride_v) * ((height + 1) / 2); +} + +} // namespace + +I420Buffer::I420Buffer(int width, int height) + : I420Buffer(width, height, width, (width + 1) / 2, (width + 1) / 2) { +} + +I420Buffer::I420Buffer(int width, + int height, + int stride_y, + int stride_u, + int stride_v) + : width_(width), + height_(height), + stride_y_(stride_y), + stride_u_(stride_u), + stride_v_(stride_v), + data_(static_cast<uint8_t*>(AlignedMalloc( + I420DataSize(height, stride_y, stride_u, stride_v), + kBufferAlignment))) { + RTC_DCHECK_GT(width, 0); + RTC_DCHECK_GT(height, 0); + RTC_DCHECK_GE(stride_y, width); + RTC_DCHECK_GE(stride_u, (width + 1) / 2); + RTC_DCHECK_GE(stride_v, (width + 1) / 2); +} + +I420Buffer::~I420Buffer() { +} + +// static +rtc::scoped_refptr<I420Buffer> I420Buffer::Create(int width, int height) { + return new rtc::RefCountedObject<I420Buffer>(width, height); +} + +// static +rtc::scoped_refptr<I420Buffer> I420Buffer::Create(int width, + int height, + int stride_y, + int stride_u, + int stride_v) { + return new rtc::RefCountedObject<I420Buffer>( + width, height, stride_y, stride_u, stride_v); +} + +// static +rtc::scoped_refptr<I420Buffer> I420Buffer::Copy( + const I420BufferInterface& source) { + return Copy(source.width(), source.height(), + source.DataY(), source.StrideY(), + source.DataU(), source.StrideU(), + source.DataV(), source.StrideV()); +} + +// static +rtc::scoped_refptr<I420Buffer> I420Buffer::Copy( + int width, int height, + const uint8_t* data_y, int stride_y, + const uint8_t* data_u, int stride_u, + const uint8_t* data_v, int stride_v) { + // Note: May use different strides than the input data. + rtc::scoped_refptr<I420Buffer> buffer = Create(width, height); + RTC_CHECK_EQ(0, libyuv::I420Copy(data_y, stride_y, + data_u, stride_u, + data_v, stride_v, + buffer->MutableDataY(), buffer->StrideY(), + buffer->MutableDataU(), buffer->StrideU(), + buffer->MutableDataV(), buffer->StrideV(), + width, height)); + return buffer; +} + +// static +rtc::scoped_refptr<I420Buffer> I420Buffer::Rotate( + const I420BufferInterface& src, + VideoRotation rotation) { + RTC_CHECK(src.DataY()); + RTC_CHECK(src.DataU()); + RTC_CHECK(src.DataV()); + + int rotated_width = src.width(); + int rotated_height = src.height(); + if (rotation == webrtc::kVideoRotation_90 || + rotation == webrtc::kVideoRotation_270) { + std::swap(rotated_width, rotated_height); + } + + rtc::scoped_refptr<webrtc::I420Buffer> buffer = + I420Buffer::Create(rotated_width, rotated_height); + + RTC_CHECK_EQ(0, libyuv::I420Rotate( + src.DataY(), src.StrideY(), + src.DataU(), src.StrideU(), + src.DataV(), src.StrideV(), + buffer->MutableDataY(), buffer->StrideY(), buffer->MutableDataU(), + buffer->StrideU(), buffer->MutableDataV(), buffer->StrideV(), + src.width(), src.height(), + static_cast<libyuv::RotationMode>(rotation))); + + return buffer; +} + +void I420Buffer::InitializeData() { + memset(data_.get(), 0, + I420DataSize(height_, stride_y_, stride_u_, stride_v_)); +} + +int I420Buffer::width() const { + return width_; +} + +int I420Buffer::height() const { + return height_; +} + +const uint8_t* I420Buffer::DataY() const { + return data_.get(); +} +const uint8_t* I420Buffer::DataU() const { + return data_.get() + stride_y_ * height_; +} +const uint8_t* I420Buffer::DataV() const { + return data_.get() + stride_y_ * height_ + stride_u_ * ((height_ + 1) / 2); +} + +int I420Buffer::StrideY() const { + return stride_y_; +} +int I420Buffer::StrideU() const { + return stride_u_; +} +int I420Buffer::StrideV() const { + return stride_v_; +} + +uint8_t* I420Buffer::MutableDataY() { + return const_cast<uint8_t*>(DataY()); +} +uint8_t* I420Buffer::MutableDataU() { + return const_cast<uint8_t*>(DataU()); +} +uint8_t* I420Buffer::MutableDataV() { + return const_cast<uint8_t*>(DataV()); +} + +// static +void I420Buffer::SetBlack(I420Buffer* buffer) { + RTC_CHECK(libyuv::I420Rect(buffer->MutableDataY(), buffer->StrideY(), + buffer->MutableDataU(), buffer->StrideU(), + buffer->MutableDataV(), buffer->StrideV(), + 0, 0, buffer->width(), buffer->height(), + 0, 128, 128) == 0); +} + +void I420Buffer::CropAndScaleFrom(const I420BufferInterface& src, + int offset_x, + int offset_y, + int crop_width, + int crop_height) { + RTC_CHECK_LE(crop_width, src.width()); + RTC_CHECK_LE(crop_height, src.height()); + RTC_CHECK_LE(crop_width + offset_x, src.width()); + RTC_CHECK_LE(crop_height + offset_y, src.height()); + RTC_CHECK_GE(offset_x, 0); + RTC_CHECK_GE(offset_y, 0); + + // Make sure offset is even so that u/v plane becomes aligned. + const int uv_offset_x = offset_x / 2; + const int uv_offset_y = offset_y / 2; + offset_x = uv_offset_x * 2; + offset_y = uv_offset_y * 2; + + const uint8_t* y_plane = + src.DataY() + src.StrideY() * offset_y + offset_x; + const uint8_t* u_plane = + src.DataU() + src.StrideU() * uv_offset_y + uv_offset_x; + const uint8_t* v_plane = + src.DataV() + src.StrideV() * uv_offset_y + uv_offset_x; + int res = libyuv::I420Scale(y_plane, src.StrideY(), + u_plane, src.StrideU(), + v_plane, src.StrideV(), + crop_width, crop_height, + MutableDataY(), StrideY(), + MutableDataU(), StrideU(), + MutableDataV(), StrideV(), + width(), height(), libyuv::kFilterBox); + + RTC_DCHECK_EQ(res, 0); +} + +void I420Buffer::CropAndScaleFrom(const I420BufferInterface& src) { + const int crop_width = height() ? + std::min(src.width(), width() * src.height() / height()) : src.width(); + const int crop_height = width() ? + std::min(src.height(), height() * src.width() / width()) : src.height(); + + CropAndScaleFrom( + src, + (src.width() - crop_width) / 2, (src.height() - crop_height) / 2, + crop_width, crop_height); +} + +void I420Buffer::ScaleFrom(const I420BufferInterface& src) { + CropAndScaleFrom(src, 0, 0, src.width(), src.height()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/api/video/i420_buffer.h b/third_party/libwebrtc/webrtc/api/video/i420_buffer.h new file mode 100644 index 0000000000..bdac80bbc1 --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/i420_buffer.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_I420_BUFFER_H_ +#define API_VIDEO_I420_BUFFER_H_ + +#include <memory> + +#include "api/video/video_rotation.h" +#include "api/video/video_frame_buffer.h" +#include "system_wrappers/include/aligned_malloc.h" + +namespace webrtc { + +// Plain I420 buffer in standard memory. +class I420Buffer : public I420BufferInterface { + public: + static rtc::scoped_refptr<I420Buffer> Create(int width, int height); + static rtc::scoped_refptr<I420Buffer> Create(int width, + int height, + int stride_y, + int stride_u, + int stride_v); + + // Create a new buffer and copy the pixel data. + static rtc::scoped_refptr<I420Buffer> Copy(const I420BufferInterface& buffer); + // Deprecated. + static rtc::scoped_refptr<I420Buffer> Copy(const VideoFrameBuffer& buffer) { + return Copy(*buffer.GetI420()); + } + + static rtc::scoped_refptr<I420Buffer> Copy( + int width, int height, + const uint8_t* data_y, int stride_y, + const uint8_t* data_u, int stride_u, + const uint8_t* data_v, int stride_v); + + // Returns a rotated copy of |src|. + static rtc::scoped_refptr<I420Buffer> Rotate(const I420BufferInterface& src, + VideoRotation rotation); + // Deprecated. + static rtc::scoped_refptr<I420Buffer> Rotate(const VideoFrameBuffer& src, + VideoRotation rotation) { + return Rotate(*src.GetI420(), rotation); + } + + // Sets the buffer to all black. + static void SetBlack(I420Buffer* buffer); + + // Sets all three planes to all zeros. Used to work around for + // quirks in memory checkers + // (https://bugs.chromium.org/p/libyuv/issues/detail?id=377) and + // ffmpeg (http://crbug.com/390941). + // TODO(nisse): Deprecated. Should be deleted if/when those issues + // are resolved in a better way. Or in the mean time, use SetBlack. + void InitializeData(); + + int width() const override; + int height() const override; + const uint8_t* DataY() const override; + const uint8_t* DataU() const override; + const uint8_t* DataV() const override; + + int StrideY() const override; + int StrideU() const override; + int StrideV() const override; + + uint8_t* MutableDataY(); + uint8_t* MutableDataU(); + uint8_t* MutableDataV(); + + // Scale the cropped area of |src| to the size of |this| buffer, and + // write the result into |this|. + void CropAndScaleFrom(const I420BufferInterface& src, + int offset_x, + int offset_y, + int crop_width, + int crop_height); + + // The common case of a center crop, when needed to adjust the + // aspect ratio without distorting the image. + void CropAndScaleFrom(const I420BufferInterface& src); + + // Scale all of |src| to the size of |this| buffer, with no cropping. + void ScaleFrom(const I420BufferInterface& src); + + protected: + I420Buffer(int width, int height); + I420Buffer(int width, int height, int stride_y, int stride_u, int stride_v); + + ~I420Buffer() override; + + private: + const int width_; + const int height_; + const int stride_y_; + const int stride_u_; + const int stride_v_; + const std::unique_ptr<uint8_t, AlignedFreeDeleter> data_; +}; + +} // namespace webrtc + +#endif // API_VIDEO_I420_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/api/video/video_content_type.cc b/third_party/libwebrtc/webrtc/api/video/video_content_type.cc new file mode 100644 index 0000000000..149b4f9926 --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/video_content_type.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/video_content_type.h" + +// VideoContentType stored as a single byte, which is sent over the network. +// Structure: +// +// 0 1 2 3 4 5 6 7 +// +---------------+ +// |r r e e e s s c| +// +// where: +// r - reserved bits. +// e - 3-bit number of an experiment group counted from 1. 0 means there's no +// experiment ongoing. +// s - 2-bit simulcast stream id or spatial layer, counted from 1. 0 means that +// no simulcast information is set. +// c - content type. 0 means real-time video, 1 means screenshare. +// + +namespace webrtc { +namespace videocontenttypehelpers { + +namespace { +static constexpr uint8_t kScreenshareBitsSize = 1; +static constexpr uint8_t kScreenshareBitsMask = + (1u << kScreenshareBitsSize) - 1; + +static constexpr uint8_t kSimulcastShift = 1; +static constexpr uint8_t kSimulcastBitsSize = 2; +static constexpr uint8_t kSimulcastBitsMask = ((1u << kSimulcastBitsSize) - 1) + << kSimulcastShift; // 0b00000110 + +static constexpr uint8_t kExperimentShift = 3; +static constexpr uint8_t kExperimentBitsSize = 3; +static constexpr uint8_t kExperimentBitsMask = + ((1u << kExperimentBitsSize) - 1) << kExperimentShift; // 0b00111000 + +static constexpr uint8_t kTotalBitsSize = + kScreenshareBitsSize + kSimulcastBitsSize + kExperimentBitsSize; +} // namespace + +bool SetExperimentId(VideoContentType* content_type, uint8_t experiment_id) { + // Store in bits 2-4. + if (experiment_id >= (1 << kExperimentBitsSize)) + return false; + *content_type = static_cast<VideoContentType>( + (static_cast<uint8_t>(*content_type) & ~kExperimentBitsMask) | + ((experiment_id << kExperimentShift) & kExperimentBitsMask)); + return true; +} + +bool SetSimulcastId(VideoContentType* content_type, uint8_t simulcast_id) { + // Store in bits 5-6. + if (simulcast_id >= (1 << kSimulcastBitsSize)) + return false; + *content_type = static_cast<VideoContentType>( + (static_cast<uint8_t>(*content_type) & ~kSimulcastBitsMask) | + ((simulcast_id << kSimulcastShift) & kSimulcastBitsMask)); + return true; +} + +uint8_t GetExperimentId( + const VideoContentType& content_type) { + return (static_cast<uint8_t>(content_type) & kExperimentBitsMask) >> + kExperimentShift; +} +uint8_t GetSimulcastId( + const VideoContentType& content_type) { + return (static_cast<uint8_t>(content_type) & kSimulcastBitsMask) >> + kSimulcastShift; +} + +bool IsScreenshare( + const VideoContentType& content_type) { + return (static_cast<uint8_t>(content_type) & kScreenshareBitsMask) > 0; +} + +bool IsValidContentType(uint8_t value) { + // Any 6-bit value is allowed. + return value < (1 << kTotalBitsSize); +} + +const char* ToString(const VideoContentType& content_type) { + return IsScreenshare(content_type) ? "screen" : "realtime"; +} +} // namespace videocontenttypehelpers +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/api/video/video_content_type.h b/third_party/libwebrtc/webrtc/api/video/video_content_type.h new file mode 100644 index 0000000000..8c6460288d --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/video_content_type.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_CONTENT_TYPE_H_ +#define API_VIDEO_VIDEO_CONTENT_TYPE_H_ + +#include <stdint.h> + +#include <string> + +namespace webrtc { + +enum class VideoContentType : uint8_t { + UNSPECIFIED = 0, + SCREENSHARE = 1, +}; + +namespace videocontenttypehelpers { +bool SetExperimentId(VideoContentType* content_type, uint8_t experiment_id); +bool SetSimulcastId(VideoContentType* content_type, uint8_t simulcast_id); + +uint8_t GetExperimentId(const VideoContentType& content_type); +uint8_t GetSimulcastId(const VideoContentType& content_type); + +bool IsScreenshare(const VideoContentType& content_type); + +bool IsValidContentType(uint8_t value); + +const char* ToString(const VideoContentType& content_type); +} // namespace videocontenttypehelpers + +} // namespace webrtc + +#endif // API_VIDEO_VIDEO_CONTENT_TYPE_H_ diff --git a/third_party/libwebrtc/webrtc/api/video/video_frame.cc b/third_party/libwebrtc/webrtc/api/video/video_frame.cc new file mode 100644 index 0000000000..93b3c9c6b9 --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/video_frame.cc @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/video_frame.h" + +#include "rtc_base/checks.h" +#include "rtc_base/timeutils.h" + +namespace webrtc { + +VideoFrame::VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer, + webrtc::VideoRotation rotation, + int64_t timestamp_us) + : video_frame_buffer_(buffer), + timestamp_rtp_(0), + ntp_time_ms_(0), + timestamp_us_(timestamp_us), + rotation_(rotation) {} + +VideoFrame::VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer, + uint32_t timestamp, + int64_t render_time_ms, + VideoRotation rotation) + : video_frame_buffer_(buffer), + timestamp_rtp_(timestamp), + ntp_time_ms_(0), + timestamp_us_(render_time_ms * rtc::kNumMicrosecsPerMillisec), + rotation_(rotation) { + RTC_DCHECK(buffer); +} + +VideoFrame::~VideoFrame() = default; + +VideoFrame::VideoFrame(const VideoFrame&) = default; +VideoFrame::VideoFrame(VideoFrame&&) = default; +VideoFrame& VideoFrame::operator=(const VideoFrame&) = default; +VideoFrame& VideoFrame::operator=(VideoFrame&&) = default; + +int VideoFrame::width() const { + return video_frame_buffer_ ? video_frame_buffer_->width() : 0; +} + +int VideoFrame::height() const { + return video_frame_buffer_ ? video_frame_buffer_->height() : 0; +} + +uint32_t VideoFrame::size() const { + return width() * height(); +} + +rtc::scoped_refptr<VideoFrameBuffer> VideoFrame::video_frame_buffer() const { + return video_frame_buffer_; +} + +int64_t VideoFrame::render_time_ms() const { + return timestamp_us() / rtc::kNumMicrosecsPerMillisec; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/api/video/video_frame.h b/third_party/libwebrtc/webrtc/api/video/video_frame.h new file mode 100644 index 0000000000..a72bef1d32 --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/video_frame.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_FRAME_H_ +#define API_VIDEO_VIDEO_FRAME_H_ + +#include <stdint.h> + +#include "api/video/video_rotation.h" +#include "api/video/video_frame_buffer.h" + +namespace webrtc { + +class VideoFrame { + public: + // TODO(nisse): This constructor is consistent with the now deleted + // cricket::WebRtcVideoFrame. We should consider whether or not we + // want to stick to this style and deprecate the other constructor. + VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer, + webrtc::VideoRotation rotation, + int64_t timestamp_us); + + // Preferred constructor. + VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer, + uint32_t timestamp, + int64_t render_time_ms, + VideoRotation rotation); + + ~VideoFrame(); + + // Support move and copy. + VideoFrame(const VideoFrame&); + VideoFrame(VideoFrame&&); + VideoFrame& operator=(const VideoFrame&); + VideoFrame& operator=(VideoFrame&&); + + // Get frame width. + int width() const; + // Get frame height. + int height() const; + // Get frame size in pixels. + uint32_t size() const; + + // System monotonic clock, same timebase as rtc::TimeMicros(). + int64_t timestamp_us() const { return timestamp_us_; } + void set_timestamp_us(int64_t timestamp_us) { timestamp_us_ = timestamp_us; } + + // TODO(nisse): After the cricket::VideoFrame and webrtc::VideoFrame + // merge, timestamps other than timestamp_us will likely be + // deprecated. + + // Set frame timestamp (90kHz). + void set_timestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; } + + // Get frame timestamp (90kHz). + uint32_t timestamp() const { return timestamp_rtp_; } + + // For now, transport_frame_id and rtp timestamp are the same. + // TODO(nisse): Must be handled differently for QUIC. + uint32_t transport_frame_id() const { return timestamp(); } + + // Set capture ntp time in milliseconds. + // TODO(nisse): Deprecated. Migrate all users to timestamp_us(). + void set_ntp_time_ms(int64_t ntp_time_ms) { ntp_time_ms_ = ntp_time_ms; } + + // Get capture ntp time in milliseconds. + // TODO(nisse): Deprecated. Migrate all users to timestamp_us(). + int64_t ntp_time_ms() const { return ntp_time_ms_; } + + // Naming convention for Coordination of Video Orientation. Please see + // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ts_126114v120700p.pdf + // + // "pending rotation" or "pending" = a frame that has a VideoRotation > 0. + // + // "not pending" = a frame that has a VideoRotation == 0. + // + // "apply rotation" = modify a frame from being "pending" to being "not + // pending" rotation (a no-op for "unrotated"). + // + VideoRotation rotation() const { return rotation_; } + void set_rotation(VideoRotation rotation) { rotation_ = rotation; } + + // Get render time in milliseconds. + // TODO(nisse): Deprecated. Migrate all users to timestamp_us(). + int64_t render_time_ms() const; + + // Return the underlying buffer. Never nullptr for a properly + // initialized VideoFrame. + rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer() const; + + // TODO(nisse): Deprecated. + // Return true if the frame is stored in a texture. + bool is_texture() const { + return video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative; + } + + private: + // An opaque reference counted handle that stores the pixel data. + rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_; + uint32_t timestamp_rtp_; + int64_t ntp_time_ms_; + int64_t timestamp_us_; + VideoRotation rotation_; +}; + +} // namespace webrtc + +#endif // API_VIDEO_VIDEO_FRAME_H_ diff --git a/third_party/libwebrtc/webrtc/api/video/video_frame_buffer.cc b/third_party/libwebrtc/webrtc/api/video/video_frame_buffer.cc new file mode 100644 index 0000000000..867f249fe6 --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/video_frame_buffer.cc @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/video_frame_buffer.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +rtc::scoped_refptr<I420BufferInterface> VideoFrameBuffer::GetI420() { + RTC_CHECK(type() == Type::kI420); + return static_cast<I420BufferInterface*>(this); +} + +rtc::scoped_refptr<const I420BufferInterface> VideoFrameBuffer::GetI420() + const { + RTC_CHECK(type() == Type::kI420); + return static_cast<const I420BufferInterface*>(this); +} + +I420ABufferInterface* VideoFrameBuffer::GetI420A() { + RTC_CHECK(type() == Type::kI420A); + return static_cast<I420ABufferInterface*>(this); +} + +const I420ABufferInterface* VideoFrameBuffer::GetI420A() const { + RTC_CHECK(type() == Type::kI420A); + return static_cast<const I420ABufferInterface*>(this); +} + +I444BufferInterface* VideoFrameBuffer::GetI444() { + RTC_CHECK(type() == Type::kI444); + return static_cast<I444BufferInterface*>(this); +} + +const I444BufferInterface* VideoFrameBuffer::GetI444() const { + RTC_CHECK(type() == Type::kI444); + return static_cast<const I444BufferInterface*>(this); +} + +VideoFrameBuffer::Type I420BufferInterface::type() const { + return Type::kI420; +} + +int I420BufferInterface::ChromaWidth() const { + return (width() + 1) / 2; +} + +int I420BufferInterface::ChromaHeight() const { + return (height() + 1) / 2; +} + +rtc::scoped_refptr<I420BufferInterface> I420BufferInterface::ToI420() { + return this; +} + +VideoFrameBuffer::Type I420ABufferInterface::type() const { + return Type::kI420A; +} + +VideoFrameBuffer::Type I444BufferInterface::type() const { + return Type::kI444; +} + +int I444BufferInterface::ChromaWidth() const { + return width(); +} + +int I444BufferInterface::ChromaHeight() const { + return height(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/api/video/video_frame_buffer.h b/third_party/libwebrtc/webrtc/api/video/video_frame_buffer.h new file mode 100644 index 0000000000..2be7e0bb9f --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/video_frame_buffer.h @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_FRAME_BUFFER_H_ +#define API_VIDEO_VIDEO_FRAME_BUFFER_H_ + +#include <stdint.h> + +#include "rtc_base/refcount.h" +#include "rtc_base/scoped_ref_ptr.h" + +namespace webrtc { + +class I420BufferInterface; +class I420ABufferInterface; +class I444BufferInterface; + +// Base class for frame buffers of different types of pixel format and storage. +// The tag in type() indicates how the data is represented, and each type is +// implemented as a subclass. To access the pixel data, call the appropriate +// GetXXX() function, where XXX represents the type. There is also a function +// ToI420() that returns a frame buffer in I420 format, converting from the +// underlying representation if necessary. I420 is the most widely accepted +// format and serves as a fallback for video sinks that can only handle I420, +// e.g. the internal WebRTC software encoders. A special enum value 'kNative' is +// provided for external clients to implement their own frame buffer +// representations, e.g. as textures. The external client can produce such +// native frame buffers from custom video sources, and then cast it back to the +// correct subclass in custom video sinks. The purpose of this is to improve +// performance by providing an optimized path without intermediate conversions. +// Frame metadata such as rotation and timestamp are stored in +// webrtc::VideoFrame, and not here. +class VideoFrameBuffer : public rtc::RefCountInterface { + public: + // New frame buffer types will be added conservatively when there is an + // opportunity to optimize the path between some pair of video source and + // video sink. + enum class Type { + kNative, + kI420, + kI420A, + kI444, + }; + + // This function specifies in what pixel format the data is stored in. + virtual Type type() const = 0; + + // The resolution of the frame in pixels. For formats where some planes are + // subsampled, this is the highest-resolution plane. + virtual int width() const = 0; + virtual int height() const = 0; + + // Returns a memory-backed frame buffer in I420 format. If the pixel data is + // in another format, a conversion will take place. All implementations must + // provide a fallback to I420 for compatibility with e.g. the internal WebRTC + // software encoders. + virtual rtc::scoped_refptr<I420BufferInterface> ToI420() = 0; + + // These functions should only be called if type() is of the correct type. + // Calling with a different type will result in a crash. + // TODO(magjed): Return raw pointers for GetI420 once deprecated interface is + // removed. + rtc::scoped_refptr<I420BufferInterface> GetI420(); + rtc::scoped_refptr<const I420BufferInterface> GetI420() const; + I420ABufferInterface* GetI420A(); + const I420ABufferInterface* GetI420A() const; + I444BufferInterface* GetI444(); + const I444BufferInterface* GetI444() const; + + protected: + ~VideoFrameBuffer() override {} +}; + +// This interface represents Type::kI420 and Type::kI444. +class PlanarYuvBuffer : public VideoFrameBuffer { + public: + virtual int ChromaWidth() const = 0; + virtual int ChromaHeight() const = 0; + + // Returns pointer to the pixel data for a given plane. The memory is owned by + // the VideoFrameBuffer object and must not be freed by the caller. + virtual const uint8_t* DataY() const = 0; + virtual const uint8_t* DataU() const = 0; + virtual const uint8_t* DataV() const = 0; + + // Returns the number of bytes between successive rows for a given plane. + virtual int StrideY() const = 0; + virtual int StrideU() const = 0; + virtual int StrideV() const = 0; + + protected: + ~PlanarYuvBuffer() override {} +}; + +class I420BufferInterface : public PlanarYuvBuffer { + public: + Type type() const override; + + int ChromaWidth() const final; + int ChromaHeight() const final; + + rtc::scoped_refptr<I420BufferInterface> ToI420() final; + + protected: + ~I420BufferInterface() override {} +}; + +class I420ABufferInterface : public I420BufferInterface { + public: + Type type() const final; + virtual const uint8_t* DataA() const = 0; + virtual int StrideA() const = 0; + + protected: + ~I420ABufferInterface() override {} +}; + +class I444BufferInterface : public PlanarYuvBuffer { + public: + Type type() const final; + + int ChromaWidth() const final; + int ChromaHeight() const final; + + protected: + ~I444BufferInterface() override {} +}; + +} // namespace webrtc + +#endif // API_VIDEO_VIDEO_FRAME_BUFFER_H_ diff --git a/third_party/libwebrtc/webrtc/api/video/video_rotation.h b/third_party/libwebrtc/webrtc/api/video/video_rotation.h new file mode 100644 index 0000000000..6a29588ee5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/video_rotation.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_ROTATION_H_ +#define API_VIDEO_VIDEO_ROTATION_H_ + +namespace webrtc { + +// enum for clockwise rotation. +enum VideoRotation { + kVideoRotation_0 = 0, + kVideoRotation_90 = 90, + kVideoRotation_180 = 180, + kVideoRotation_270 = 270 +}; + +} // namespace webrtc + +#endif // API_VIDEO_VIDEO_ROTATION_H_ diff --git a/third_party/libwebrtc/webrtc/api/video/video_timing.cc b/third_party/libwebrtc/webrtc/api/video/video_timing.cc new file mode 100644 index 0000000000..3ccbe4eae5 --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/video_timing.cc @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/video_timing.h" + +#include <sstream> + +namespace webrtc { + +TimingFrameInfo::TimingFrameInfo() + : rtp_timestamp(0), + capture_time_ms(-1), + encode_start_ms(-1), + encode_finish_ms(-1), + packetization_finish_ms(-1), + pacer_exit_ms(-1), + network_timestamp_ms(-1), + network2_timestamp_ms(-1), + receive_start_ms(-1), + receive_finish_ms(-1), + decode_start_ms(-1), + decode_finish_ms(-1), + render_time_ms(-1), + flags(TimingFrameFlags::kDefault) {} + +int64_t TimingFrameInfo::EndToEndDelay() const { + return capture_time_ms >= 0 ? decode_finish_ms - capture_time_ms : -1; +} + +bool TimingFrameInfo::IsLongerThan(const TimingFrameInfo& other) const { + int64_t other_delay = other.EndToEndDelay(); + return other_delay == -1 || EndToEndDelay() > other_delay; +} + +bool TimingFrameInfo::operator<(const TimingFrameInfo& other) const { + return other.IsLongerThan(*this); +} + +bool TimingFrameInfo::operator<=(const TimingFrameInfo& other) const { + return !IsLongerThan(other); +} + +bool TimingFrameInfo::IsOutlier() const { + return !IsInvalid() && (flags & TimingFrameFlags::kTriggeredBySize); +} + +bool TimingFrameInfo::IsTimerTriggered() const { + return !IsInvalid() && (flags & TimingFrameFlags::kTriggeredByTimer); +} + +bool TimingFrameInfo::IsInvalid() const { + return flags == TimingFrameFlags::kInvalid; +} + +std::string TimingFrameInfo::ToString() const { + std::stringstream out; + if (IsInvalid()) { + out << ""; + } else { + out << rtp_timestamp << ',' << capture_time_ms << ',' << encode_start_ms + << ',' << encode_finish_ms << ',' << packetization_finish_ms << ',' + << pacer_exit_ms << ',' << network_timestamp_ms << ',' + << network2_timestamp_ms << ',' << receive_start_ms << ',' + << receive_finish_ms << ',' << decode_start_ms << ',' + << decode_finish_ms << ',' << render_time_ms << ',' + << IsOutlier() << ',' << IsTimerTriggered(); + } + return out.str(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/webrtc/api/video/video_timing.h b/third_party/libwebrtc/webrtc/api/video/video_timing.h new file mode 100644 index 0000000000..ab8cd99136 --- /dev/null +++ b/third_party/libwebrtc/webrtc/api/video/video_timing.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_TIMING_H_ +#define API_VIDEO_VIDEO_TIMING_H_ + +#include <stdint.h> + +#include <limits> +#include <string> + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +enum TimingFrameFlags : uint8_t { + kNotTriggered = 0, // Timing info valid, but not to be transmitted. + // Used on send-side only. + // TODO(ilnik): Delete compatibility alias. + // Used to be sent over the wire, for the old protocol. + kDefault = 0, // Old name, for API compatibility. + kTriggeredByTimer = 1 << 0, // Frame marked for tracing by periodic timer. + kTriggeredBySize = 1 << 1, // Frame marked for tracing due to size. + kInvalid = std::numeric_limits<uint8_t>::max() // Invalid, ignore! +}; + +// Video timing timestamps in ms counted from capture_time_ms of a frame. +// This structure represents data sent in video-timing RTP header extension. +struct VideoSendTiming { + // Offsets of the fields in the RTP header extension, counting from the first + // byte after the one-byte header. + static constexpr uint8_t kFlagsOffset = 0; + static constexpr uint8_t kEncodeStartDeltaOffset = 1; + static constexpr uint8_t kEncodeFinishDeltaOffset = 3; + static constexpr uint8_t kPacketizationFinishDeltaOffset = 5; + static constexpr uint8_t kPacerExitDeltaOffset = 7; + static constexpr uint8_t kNetworkTimestampDeltaOffset = 9; + static constexpr uint8_t kNetwork2TimestampDeltaOffset = 11; + + // Returns |time_ms - base_ms| capped at max 16-bit value. + // Used to fill this data structure as per + // https://webrtc.org/experiments/rtp-hdrext/video-timing/ extension stores + // 16-bit deltas of timestamps from packet capture time. + static uint16_t GetDeltaCappedMs(int64_t base_ms, int64_t time_ms) { + RTC_DCHECK_GE(time_ms, base_ms); + return rtc::saturated_cast<uint16_t>(time_ms - base_ms); + } + + uint16_t encode_start_delta_ms; + uint16_t encode_finish_delta_ms; + uint16_t packetization_finish_delta_ms; + uint16_t pacer_exit_delta_ms; + uint16_t network_timestamp_delta_ms; + uint16_t network2_timestamp_delta_ms; + uint8_t flags; +}; + +// Used to report precise timings of a 'timing frames'. Contains all important +// timestamps for a lifetime of that specific frame. Reported as a string via +// GetStats(). Only frame which took the longest between two GetStats calls is +// reported. +struct TimingFrameInfo { + TimingFrameInfo(); + + // Returns end-to-end delay of a frame, if sender and receiver timestamps are + // synchronized, -1 otherwise. + int64_t EndToEndDelay() const; + + // Returns true if current frame took longer to process than |other| frame. + // If other frame's clocks are not synchronized, current frame is always + // preferred. + bool IsLongerThan(const TimingFrameInfo& other) const; + + // Returns true if flags are set to indicate this frame was marked for tracing + // due to the size being outside some limit. + bool IsOutlier() const; + + // Returns true if flags are set to indicate this frame was marked fro tracing + // due to cyclic timer. + bool IsTimerTriggered() const; + + // Returns true if the timing data is marked as invalid, in which case it + // should be ignored. + bool IsInvalid() const; + + std::string ToString() const; + + bool operator<(const TimingFrameInfo& other) const; + + bool operator<=(const TimingFrameInfo& other) const; + + uint32_t rtp_timestamp; // Identifier of a frame. + // All timestamps below are in local monotonous clock of a receiver. + // If sender clock is not yet estimated, sender timestamps + // (capture_time_ms ... pacer_exit_ms) are negative values, still + // relatively correct. + int64_t capture_time_ms; // Captrue time of a frame. + int64_t encode_start_ms; // Encode start time. + int64_t encode_finish_ms; // Encode completion time. + int64_t packetization_finish_ms; // Time when frame was passed to pacer. + int64_t pacer_exit_ms; // Time when last packet was pushed out of pacer. + // Two in-network RTP processor timestamps: meaning is application specific. + int64_t network_timestamp_ms; + int64_t network2_timestamp_ms; + int64_t receive_start_ms; // First received packet time. + int64_t receive_finish_ms; // Last received packet time. + int64_t decode_start_ms; // Decode start time. + int64_t decode_finish_ms; // Decode completion time. + int64_t render_time_ms; // Proposed render time to insure smooth playback. + + uint8_t flags; // Flags indicating validity and/or why tracing was triggered. +}; + +} // namespace webrtc + +#endif // API_VIDEO_VIDEO_TIMING_H_ |