summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/api/video/video_frame.h
blob: 2608f9aa427135c57fcfa8f5aa7b70e6411aefc1 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
/*
 *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef API_VIDEO_VIDEO_FRAME_H_
#define API_VIDEO_VIDEO_FRAME_H_

#include <stdint.h>

#include <utility>

#include "absl/types/optional.h"
#include "api/rtp_packet_infos.h"
#include "api/scoped_refptr.h"
#include "api/video/color_space.h"
#include "api/video/hdr_metadata.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_rotation.h"
#include "rtc_base/checks.h"
#include "rtc_base/system/rtc_export.h"

namespace webrtc {

class RTC_EXPORT VideoFrame {
 public:
  // Value used to signal that `VideoFrame::id()` is not set.
  static constexpr uint16_t kNotSetId = 0;

  struct RTC_EXPORT UpdateRect {
    int offset_x;
    int offset_y;
    int width;
    int height;

    // Makes this UpdateRect a bounding box of this and other rect.
    void Union(const UpdateRect& other);

    // Makes this UpdateRect an intersection of this and other rect.
    void Intersect(const UpdateRect& other);

    // Sets everything to 0, making this UpdateRect a zero-size (empty) update.
    void MakeEmptyUpdate();

    bool IsEmpty() const;

    // Per-member equality check. Empty rectangles with different offsets would
    // be considered different.
    bool operator==(const UpdateRect& other) const {
      return other.offset_x == offset_x && other.offset_y == offset_y &&
             other.width == width && other.height == height;
    }

    bool operator!=(const UpdateRect& other) const { return !(*this == other); }

    // Scales update_rect given original frame dimensions.
    // Cropping is applied first, then rect is scaled down.
    // Update rect is snapped to 2x2 grid due to possible UV subsampling and
    // then expanded by additional 2 pixels in each direction to accommodate any
    // possible scaling artifacts.
    // Note, close but not equal update_rects on original frame may result in
    // the same scaled update rects.
    UpdateRect ScaleWithFrame(int frame_width,
                              int frame_height,
                              int crop_x,
                              int crop_y,
                              int crop_width,
                              int crop_height,
                              int scaled_width,
                              int scaled_height) const;
  };

  struct RTC_EXPORT ProcessingTime {
    TimeDelta Elapsed() const { return finish - start; }
    Timestamp start;
    Timestamp finish;
  };

  struct RTC_EXPORT RenderParameters {
    bool use_low_latency_rendering = false;
    absl::optional<int32_t> max_composition_delay_in_frames;

    bool operator==(const RenderParameters& other) const {
      return other.use_low_latency_rendering == use_low_latency_rendering &&
             other.max_composition_delay_in_frames ==
                 max_composition_delay_in_frames;
    }

    bool operator!=(const RenderParameters& other) const {
      return !(*this == other);
    }
  };

  // Preferred way of building VideoFrame objects.
  class RTC_EXPORT Builder {
   public:
    Builder();
    ~Builder();

    VideoFrame build();
    Builder& set_video_frame_buffer(
        const rtc::scoped_refptr<VideoFrameBuffer>& buffer);
    Builder& set_timestamp_ms(int64_t timestamp_ms);
    Builder& set_timestamp_us(int64_t timestamp_us);
    Builder& set_capture_time_identifier(
        const absl::optional<Timestamp>& capture_time_identifier);
    Builder& set_reference_time(
        const absl::optional<Timestamp>& reference_time);
    Builder& set_timestamp_rtp(uint32_t timestamp_rtp);
    Builder& set_ntp_time_ms(int64_t ntp_time_ms);
    Builder& set_rotation(VideoRotation rotation);
    Builder& set_color_space(const absl::optional<ColorSpace>& color_space);
    Builder& set_color_space(const ColorSpace* color_space);
    Builder& set_id(uint16_t id);
    Builder& set_update_rect(const absl::optional<UpdateRect>& update_rect);
    Builder& set_packet_infos(RtpPacketInfos packet_infos);

   private:
    uint16_t id_ = kNotSetId;
    rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
    int64_t timestamp_us_ = 0;
    absl::optional<Timestamp> capture_time_identifier_;
    absl::optional<Timestamp> reference_time_;
    uint32_t timestamp_rtp_ = 0;
    int64_t ntp_time_ms_ = 0;
    VideoRotation rotation_ = kVideoRotation_0;
    absl::optional<ColorSpace> color_space_;
    RenderParameters render_parameters_;
    absl::optional<UpdateRect> update_rect_;
    RtpPacketInfos packet_infos_;
  };

  // To be deprecated. Migrate all use to Builder.
  VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
             webrtc::VideoRotation rotation,
             int64_t timestamp_us);
  VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
             uint32_t timestamp_rtp,
             int64_t render_time_ms,
             VideoRotation rotation);

  ~VideoFrame();

  // Support move and copy.
  VideoFrame(const VideoFrame&);
  VideoFrame(VideoFrame&&);
  VideoFrame& operator=(const VideoFrame&);
  VideoFrame& operator=(VideoFrame&&);

  // Get frame width.
  int width() const;
  // Get frame height.
  int height() const;
  // Get frame size in pixels.
  uint32_t size() const;

  // Get frame ID. Returns `kNotSetId` if ID is not set. Not guaranteed to be
  // transferred from the sender to the receiver, but preserved on the sender
  // side. The id should be propagated between all frame modifications during
  // its lifetime from capturing to sending as encoded image. It is intended to
  // be unique over a time window of a few minutes for the peer connection to
  // which the corresponding video stream belongs to.
  uint16_t id() const { return id_; }
  void set_id(uint16_t id) { id_ = id; }

  // System monotonic clock, same timebase as rtc::TimeMicros().
  int64_t timestamp_us() const { return timestamp_us_; }
  void set_timestamp_us(int64_t timestamp_us) { timestamp_us_ = timestamp_us; }

  const absl::optional<Timestamp>& capture_time_identifier() const {
    return capture_time_identifier_;
  }
  void set_capture_time_identifier(
      const absl::optional<Timestamp>& capture_time_identifier) {
    capture_time_identifier_ = capture_time_identifier;
  }

  const absl::optional<Timestamp>& reference_time() const {
    return reference_time_;
  }
  void set_reference_time(const absl::optional<Timestamp>& reference_time) {
    reference_time_ = reference_time;
  }

  // Set frame timestamp (90kHz).
  void set_timestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; }

  // Get frame timestamp (90kHz).
  uint32_t timestamp() const { return timestamp_rtp_; }

  // Set capture ntp time in milliseconds.
  void set_ntp_time_ms(int64_t ntp_time_ms) { ntp_time_ms_ = ntp_time_ms; }

  // Get capture ntp time in milliseconds.
  int64_t ntp_time_ms() const { return ntp_time_ms_; }

  // Naming convention for Coordination of Video Orientation. Please see
  // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ts_126114v120700p.pdf
  //
  // "pending rotation" or "pending" = a frame that has a VideoRotation > 0.
  //
  // "not pending" = a frame that has a VideoRotation == 0.
  //
  // "apply rotation" = modify a frame from being "pending" to being "not
  //                    pending" rotation (a no-op for "unrotated").
  //
  VideoRotation rotation() const { return rotation_; }
  void set_rotation(VideoRotation rotation) { rotation_ = rotation; }

  // Get color space when available.
  const absl::optional<ColorSpace>& color_space() const { return color_space_; }
  void set_color_space(const absl::optional<ColorSpace>& color_space) {
    color_space_ = color_space;
  }

  RenderParameters render_parameters() const { return render_parameters_; }
  void set_render_parameters(const RenderParameters& render_parameters) {
    render_parameters_ = render_parameters;
  }

  // Deprecated in favor of render_parameters, will be removed once Chromium is
  // updated. max_composition_delay_in_frames() is used in an experiment of a
  // low-latency renderer algorithm see crbug.com/1138888.
  [[deprecated("Use render_parameters() instead.")]] absl::optional<int32_t>
  max_composition_delay_in_frames() const {
    return render_parameters_.max_composition_delay_in_frames;
  }

  // Get render time in milliseconds.
  int64_t render_time_ms() const;

  // Return the underlying buffer. Never nullptr for a properly
  // initialized VideoFrame.
  rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer() const;

  void set_video_frame_buffer(
      const rtc::scoped_refptr<VideoFrameBuffer>& buffer);

  // Return true if the frame is stored in a texture.
  bool is_texture() const {
    return video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative;
  }

  bool has_update_rect() const { return update_rect_.has_value(); }

  // Returns update_rect set by the builder or set_update_rect() or whole frame
  // rect if no update rect is available.
  UpdateRect update_rect() const {
    return update_rect_.value_or(UpdateRect{0, 0, width(), height()});
  }

  // Rectangle must be within the frame dimensions.
  void set_update_rect(const VideoFrame::UpdateRect& update_rect) {
    RTC_DCHECK_GE(update_rect.offset_x, 0);
    RTC_DCHECK_GE(update_rect.offset_y, 0);
    RTC_DCHECK_LE(update_rect.offset_x + update_rect.width, width());
    RTC_DCHECK_LE(update_rect.offset_y + update_rect.height, height());
    update_rect_ = update_rect;
  }

  void clear_update_rect() { update_rect_ = absl::nullopt; }

  // Get information about packets used to assemble this video frame. Might be
  // empty if the information isn't available.
  const RtpPacketInfos& packet_infos() const { return packet_infos_; }
  void set_packet_infos(RtpPacketInfos value) {
    packet_infos_ = std::move(value);
  }

  const absl::optional<ProcessingTime> processing_time() const {
    return processing_time_;
  }
  void set_processing_time(const ProcessingTime& processing_time) {
    processing_time_ = processing_time;
  }

 private:
  VideoFrame(uint16_t id,
             const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
             int64_t timestamp_us,
             const absl::optional<Timestamp>& capture_time_identifier,
             const absl::optional<Timestamp>& reference_time,
             uint32_t timestamp_rtp,
             int64_t ntp_time_ms,
             VideoRotation rotation,
             const absl::optional<ColorSpace>& color_space,
             const RenderParameters& render_parameters,
             const absl::optional<UpdateRect>& update_rect,
             RtpPacketInfos packet_infos);

  uint16_t id_;
  // An opaque reference counted handle that stores the pixel data.
  rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
  uint32_t timestamp_rtp_;
  int64_t ntp_time_ms_;
  int64_t timestamp_us_;
  absl::optional<Timestamp> capture_time_identifier_;
  // Contains a monotonically increasing clock time and represents the time
  // when the frame was captured. Not all platforms provide the "true" sample
  // capture time in |reference_time| but might instead use a somewhat delayed
  // (by the time it took to capture the frame) version of it.
  absl::optional<Timestamp> reference_time_;
  VideoRotation rotation_;
  absl::optional<ColorSpace> color_space_;
  // Contains parameters that affect have the frame should be rendered.
  RenderParameters render_parameters_;
  // Updated since the last frame area. If present it means that the bounding
  // box of all the changes is within the rectangular area and is close to it.
  // If absent, it means that there's no information about the change at all and
  // update_rect() will return a rectangle corresponding to the entire frame.
  absl::optional<UpdateRect> update_rect_;
  // Information about packets used to assemble this video frame. This is needed
  // by `SourceTracker` when the frame is delivered to the RTCRtpReceiver's
  // MediaStreamTrack, in order to implement getContributingSources(). See:
  // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources
  RtpPacketInfos packet_infos_;
  // Processing timestamps of the frame. For received video frames these are the
  // timestamps when the frame is sent to the decoder and the decoded image
  // returned from the decoder.
  // Currently, not set for locally captured video frames.
  absl::optional<ProcessingTime> processing_time_;
};

}  // namespace webrtc

#endif  // API_VIDEO_VIDEO_FRAME_H_