summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/video/video_analyzer.h
blob: 8c8100c67e3aff256afea15573b484f0228b091c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
/*
 *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */
#ifndef VIDEO_VIDEO_ANALYZER_H_
#define VIDEO_VIDEO_ANALYZER_H_

#include <deque>
#include <map>
#include <memory>
#include <string>
#include <vector>

#include "absl/strings/string_view.h"
#include "api/numerics/samples_stats_counter.h"
#include "api/task_queue/task_queue_base.h"
#include "api/test/metrics/metric.h"
#include "api/video/video_source_interface.h"
#include "modules/rtp_rtcp/source/rtp_packet.h"
#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
#include "rtc_base/event.h"
#include "rtc_base/numerics/running_statistics.h"
#include "rtc_base/numerics/sequence_number_unwrapper.h"
#include "rtc_base/platform_thread.h"
#include "rtc_base/synchronization/mutex.h"
#include "test/layer_filtering_transport.h"
#include "test/rtp_file_writer.h"

namespace webrtc {

class VideoAnalyzer : public PacketReceiver,
                      public Transport,
                      public rtc::VideoSinkInterface<VideoFrame> {
 public:
  VideoAnalyzer(test::LayerFilteringTransport* transport,
                const std::string& test_label,
                double avg_psnr_threshold,
                double avg_ssim_threshold,
                int duration_frames,
                TimeDelta test_duration,
                FILE* graph_data_output_file,
                const std::string& graph_title,
                uint32_t ssrc_to_analyze,
                uint32_t rtx_ssrc_to_analyze,
                size_t selected_stream,
                int selected_sl,
                int selected_tl,
                bool is_quick_test_enabled,
                Clock* clock,
                std::string rtp_dump_name,
                TaskQueueBase* task_queue);
  ~VideoAnalyzer();

  virtual void SetReceiver(PacketReceiver* receiver);
  void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source,
                 bool respect_sink_wants);
  void SetCall(Call* call);
  void SetSendStream(VideoSendStream* stream);
  void SetReceiveStream(VideoReceiveStreamInterface* stream);
  void SetAudioReceiveStream(AudioReceiveStreamInterface* recv_stream);

  rtc::VideoSinkInterface<VideoFrame>* InputInterface();
  rtc::VideoSourceInterface<VideoFrame>* OutputInterface();

  void DeliverRtcpPacket(rtc::CopyOnWriteBuffer packet) override;
  void DeliverRtpPacket(MediaType media_type,
                        RtpPacketReceived packet,
                        PacketReceiver::OnUndemuxablePacketHandler
                            undemuxable_packet_handler) override;

  void PreEncodeOnFrame(const VideoFrame& video_frame);
  void PostEncodeOnFrame(size_t stream_id, uint32_t timestamp);

  bool SendRtp(rtc::ArrayView<const uint8_t> packet,
               const PacketOptions& options) override;

  bool SendRtcp(rtc::ArrayView<const uint8_t> packet) override;
  void OnFrame(const VideoFrame& video_frame) override;
  void Wait();

  void StartMeasuringCpuProcessTime();
  void StopMeasuringCpuProcessTime();
  void StartExcludingCpuThreadTime() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_);
  void StopExcludingCpuThreadTime() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_);
  double GetCpuUsagePercent() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_);

  test::LayerFilteringTransport* const transport_;
  PacketReceiver* receiver_;

 private:
  struct FrameComparison {
    FrameComparison();
    FrameComparison(const VideoFrame& reference,
                    const VideoFrame& render,
                    bool dropped,
                    int64_t input_time_ms,
                    int64_t send_time_ms,
                    int64_t recv_time_ms,
                    int64_t render_time_ms,
                    size_t encoded_frame_size);
    FrameComparison(bool dropped,
                    int64_t input_time_ms,
                    int64_t send_time_ms,
                    int64_t recv_time_ms,
                    int64_t render_time_ms,
                    size_t encoded_frame_size);

    absl::optional<VideoFrame> reference;
    absl::optional<VideoFrame> render;
    bool dropped;
    int64_t input_time_ms;
    int64_t send_time_ms;
    int64_t recv_time_ms;
    int64_t render_time_ms;
    size_t encoded_frame_size;
  };

  struct Sample {
    Sample(int dropped,
           int64_t input_time_ms,
           int64_t send_time_ms,
           int64_t recv_time_ms,
           int64_t render_time_ms,
           size_t encoded_frame_size,
           double psnr,
           double ssim);

    int dropped;
    int64_t input_time_ms;
    int64_t send_time_ms;
    int64_t recv_time_ms;
    int64_t render_time_ms;
    size_t encoded_frame_size;
    double psnr;
    double ssim;
  };

  // Implements VideoSinkInterface to receive captured frames from a
  // FrameGeneratorCapturer. Implements VideoSourceInterface to be able to act
  // as a source to VideoSendStream.
  // It forwards all input frames to the VideoAnalyzer for later comparison and
  // forwards the captured frames to the VideoSendStream.
  class CapturedFrameForwarder : public rtc::VideoSinkInterface<VideoFrame>,
                                 public rtc::VideoSourceInterface<VideoFrame> {
   public:
    CapturedFrameForwarder(VideoAnalyzer* analyzer,
                           Clock* clock,
                           int frames_to_capture,
                           TimeDelta test_duration);
    void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source);

   private:
    void OnFrame(const VideoFrame& video_frame)
        RTC_LOCKS_EXCLUDED(lock_) override;

    // Called when `send_stream_.SetSource()` is called.
    void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
                         const rtc::VideoSinkWants& wants)
        RTC_LOCKS_EXCLUDED(lock_) override;

    // Called by `send_stream_` when `send_stream_.SetSource()` is called.
    void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink)
        RTC_LOCKS_EXCLUDED(lock_) override;

    VideoAnalyzer* const analyzer_;
    Mutex lock_;
    rtc::VideoSinkInterface<VideoFrame>* send_stream_input_
        RTC_GUARDED_BY(lock_);
    VideoSourceInterface<VideoFrame>* video_source_;
    Clock* clock_;
    int captured_frames_ RTC_GUARDED_BY(lock_);
    const int frames_to_capture_;
    const Timestamp test_end_;
  };

  struct FrameWithPsnr {
    double psnr;
    VideoFrame frame;
  };

  bool IsInSelectedSpatialAndTemporalLayer(const RtpPacket& rtp_packet);

  void AddFrameComparison(const VideoFrame& reference,
                          const VideoFrame& render,
                          bool dropped,
                          int64_t render_time_ms)
      RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);

  void PollStats() RTC_LOCKS_EXCLUDED(comparison_lock_);
  static void FrameComparisonThread(void* obj);
  bool CompareFrames();
  bool PopComparison(FrameComparison* comparison);
  // Increment counter for number of frames received for comparison.
  void FrameRecorded() RTC_EXCLUSIVE_LOCKS_REQUIRED(comparison_lock_);
  // Returns true if all frames to be compared have been taken from the queue.
  bool AllFramesRecorded() RTC_LOCKS_EXCLUDED(comparison_lock_);
  bool AllFramesRecordedLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(comparison_lock_);
  // Increase count of number of frames processed. Returns true if this was the
  // last frame to be processed.
  bool FrameProcessed() RTC_LOCKS_EXCLUDED(comparison_lock_);
  void PrintResults() RTC_LOCKS_EXCLUDED(lock_, comparison_lock_);
  void PerformFrameComparison(const FrameComparison& comparison)
      RTC_LOCKS_EXCLUDED(comparison_lock_);
  void PrintResult(absl::string_view result_type,
                   const SamplesStatsCounter& stats,
                   webrtc::test::Unit unit,
                   webrtc::test::ImprovementDirection improvement_direction);
  void PrintResultWithExternalMean(
      absl::string_view result_type,
      double mean,
      const SamplesStatsCounter& stats,
      webrtc::test::Unit unit,
      webrtc::test::ImprovementDirection improvement_direction);
  void PrintSamplesToFile(void) RTC_LOCKS_EXCLUDED(comparison_lock_);
  void AddCapturedFrameForComparison(const VideoFrame& video_frame)
      RTC_LOCKS_EXCLUDED(lock_, comparison_lock_);

  Call* call_;
  VideoSendStream* send_stream_;
  VideoReceiveStreamInterface* receive_stream_;
  AudioReceiveStreamInterface* audio_receive_stream_;
  CapturedFrameForwarder captured_frame_forwarder_;
  const std::string test_label_;
  FILE* const graph_data_output_file_;
  const std::string graph_title_;
  const uint32_t ssrc_to_analyze_;
  const uint32_t rtx_ssrc_to_analyze_;
  const size_t selected_stream_;
  const int selected_sl_;
  const int selected_tl_;

  Mutex comparison_lock_;
  std::vector<Sample> samples_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter sender_time_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter receiver_time_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter network_time_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter psnr_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter ssim_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter end_to_end_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter rendered_delta_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter encoded_frame_size_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter encode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter encode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter encode_usage_percent_ RTC_GUARDED_BY(comparison_lock_);
  double mean_decode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter decode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter decode_time_max_ms_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter media_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter fec_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter memory_usage_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter audio_expand_rate_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter audio_accelerate_rate_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter audio_jitter_buffer_ms_ RTC_GUARDED_BY(comparison_lock_);
  SamplesStatsCounter pixels_ RTC_GUARDED_BY(comparison_lock_);
  // Rendered frame with worst PSNR is saved for further analysis.
  absl::optional<FrameWithPsnr> worst_frame_ RTC_GUARDED_BY(comparison_lock_);
  // Freeze metrics.
  SamplesStatsCounter time_between_freezes_ RTC_GUARDED_BY(comparison_lock_);
  uint32_t freeze_count_ RTC_GUARDED_BY(comparison_lock_);
  uint32_t total_freezes_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
  double total_inter_frame_delay_ RTC_GUARDED_BY(comparison_lock_);
  double total_squared_inter_frame_delay_ RTC_GUARDED_BY(comparison_lock_);

  double decode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
  double render_frame_rate_ RTC_GUARDED_BY(comparison_lock_);

  size_t last_fec_bytes_;

  Mutex lock_ RTC_ACQUIRED_BEFORE(comparison_lock_)
      RTC_ACQUIRED_BEFORE(cpu_measurement_lock_);
  const int frames_to_process_;
  const Timestamp test_end_;
  int frames_recorded_ RTC_GUARDED_BY(comparison_lock_);
  int frames_processed_ RTC_GUARDED_BY(comparison_lock_);
  int captured_frames_ RTC_GUARDED_BY(comparison_lock_);
  int dropped_frames_ RTC_GUARDED_BY(comparison_lock_);
  int dropped_frames_before_first_encode_ RTC_GUARDED_BY(lock_);
  int dropped_frames_before_rendering_ RTC_GUARDED_BY(lock_);
  int64_t last_render_time_ RTC_GUARDED_BY(comparison_lock_);
  int64_t last_render_delta_ms_ RTC_GUARDED_BY(comparison_lock_);
  int64_t last_unfreeze_time_ms_ RTC_GUARDED_BY(comparison_lock_);
  uint32_t rtp_timestamp_delta_ RTC_GUARDED_BY(lock_);

  Mutex cpu_measurement_lock_;
  int64_t cpu_time_ RTC_GUARDED_BY(cpu_measurement_lock_);
  int64_t wallclock_time_ RTC_GUARDED_BY(cpu_measurement_lock_);

  std::deque<VideoFrame> frames_ RTC_GUARDED_BY(lock_);
  absl::optional<VideoFrame> last_rendered_frame_ RTC_GUARDED_BY(lock_);
  RtpTimestampUnwrapper wrap_handler_ RTC_GUARDED_BY(lock_);
  std::map<int64_t, int64_t> send_times_ RTC_GUARDED_BY(lock_);
  std::map<int64_t, int64_t> recv_times_ RTC_GUARDED_BY(lock_);
  std::map<int64_t, size_t> encoded_frame_sizes_ RTC_GUARDED_BY(lock_);
  absl::optional<uint32_t> first_encoded_timestamp_ RTC_GUARDED_BY(lock_);
  absl::optional<uint32_t> first_sent_timestamp_ RTC_GUARDED_BY(lock_);
  const double avg_psnr_threshold_;
  const double avg_ssim_threshold_;
  bool is_quick_test_enabled_;

  std::vector<rtc::PlatformThread> comparison_thread_pool_;
  rtc::Event comparison_available_event_;
  std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_);
  bool quit_ RTC_GUARDED_BY(comparison_lock_);
  rtc::Event done_;

  std::unique_ptr<VideoRtpDepacketizer> vp8_depacketizer_;
  std::unique_ptr<VideoRtpDepacketizer> vp9_depacketizer_;
  std::unique_ptr<test::RtpFileWriter> rtp_file_writer_;
  Clock* const clock_;
  const int64_t start_ms_;
  TaskQueueBase* task_queue_;
};

}  // namespace webrtc
#endif  // VIDEO_VIDEO_ANALYZER_H_