From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- .../neteq/tools/neteq_delay_analyzer.cc | 307 +++++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc (limited to 'third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc') diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc new file mode 100644 index 0000000000..9e77457775 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_delay_analyzer.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/sequence_number_unwrapper.h" + +namespace webrtc { +namespace test { +namespace { +constexpr char kArrivalDelayX[] = "arrival_delay_x"; +constexpr char kArrivalDelayY[] = "arrival_delay_y"; +constexpr char kTargetDelayX[] = "target_delay_x"; +constexpr char kTargetDelayY[] = "target_delay_y"; +constexpr char kPlayoutDelayX[] = "playout_delay_x"; +constexpr char kPlayoutDelayY[] = "playout_delay_y"; + +// Helper function for NetEqDelayAnalyzer::CreateGraphs. Returns the +// interpolated value of a function at the point x. Vector x_vec contains the +// sample points, and y_vec contains the function values at these points. The +// return value is a linear interpolation between y_vec values. +double LinearInterpolate(double x, + const std::vector& x_vec, + const std::vector& y_vec) { + // Find first element which is larger than x. + auto it = std::upper_bound(x_vec.begin(), x_vec.end(), x); + if (it == x_vec.end()) { + --it; + } + const size_t upper_ix = it - x_vec.begin(); + + size_t lower_ix; + if (upper_ix == 0 || x_vec[upper_ix] <= x) { + lower_ix = upper_ix; + } else { + lower_ix = upper_ix - 1; + } + double y; + if (lower_ix == upper_ix) { + y = y_vec[lower_ix]; + } else { + RTC_DCHECK_NE(x_vec[lower_ix], x_vec[upper_ix]); + y = (x - x_vec[lower_ix]) * (y_vec[upper_ix] - y_vec[lower_ix]) / + (x_vec[upper_ix] - x_vec[lower_ix]) + + y_vec[lower_ix]; + } + return y; +} + +void PrintDelays(const NetEqDelayAnalyzer::Delays& delays, + int64_t ref_time_ms, + absl::string_view var_name_x, + absl::string_view var_name_y, + std::ofstream& output, + absl::string_view terminator = "") { + output << var_name_x << " = [ "; + for (const std::pair& delay : delays) { + output << (delay.first - ref_time_ms) / 1000.f << ", "; + } + output << "]" << terminator << std::endl; + + output << var_name_y << " = [ "; + for (const std::pair& delay : delays) { + output << delay.second << ", "; + } + output << "]" << terminator << std::endl; +} + +} // namespace + +void NetEqDelayAnalyzer::AfterInsertPacket( + const test::NetEqInput::PacketData& packet, + NetEq* neteq) { + data_.insert( + std::make_pair(packet.header.timestamp, TimingData(packet.time_ms))); + ssrcs_.insert(packet.header.ssrc); + payload_types_.insert(packet.header.payloadType); +} + +void NetEqDelayAnalyzer::BeforeGetAudio(NetEq* neteq) { + last_sync_buffer_ms_ = neteq->SyncBufferSizeMs(); +} + +void NetEqDelayAnalyzer::AfterGetAudio(int64_t time_now_ms, + const AudioFrame& audio_frame, + bool /*muted*/, + NetEq* neteq) { + get_audio_time_ms_.push_back(time_now_ms); + for (const RtpPacketInfo& info : audio_frame.packet_infos_) { + auto it = data_.find(info.rtp_timestamp()); + if (it == data_.end()) { + // This is a packet that was split out from another packet. Skip it. + continue; + } + auto& it_timing = it->second; + RTC_CHECK(!it_timing.decode_get_audio_count) + << "Decode time already written"; + it_timing.decode_get_audio_count = get_audio_count_; + RTC_CHECK(!it_timing.sync_delay_ms) << "Decode time already written"; + it_timing.sync_delay_ms = last_sync_buffer_ms_; + it_timing.target_delay_ms = neteq->TargetDelayMs(); + it_timing.current_delay_ms = neteq->FilteredCurrentDelayMs(); + } + last_sample_rate_hz_ = audio_frame.sample_rate_hz_; + ++get_audio_count_; +} + +void NetEqDelayAnalyzer::CreateGraphs(Delays* arrival_delay_ms, + Delays* corrected_arrival_delay_ms, + Delays* playout_delay_ms, + Delays* target_delay_ms) const { + if (get_audio_time_ms_.empty()) { + return; + } + // Create nominal_get_audio_time_ms, a vector starting at + // get_audio_time_ms_[0] and increasing by 10 for each element. + std::vector nominal_get_audio_time_ms(get_audio_time_ms_.size()); + nominal_get_audio_time_ms[0] = get_audio_time_ms_[0]; + std::transform( + nominal_get_audio_time_ms.begin(), nominal_get_audio_time_ms.end() - 1, + nominal_get_audio_time_ms.begin() + 1, [](int64_t& x) { return x + 10; }); + RTC_DCHECK( + std::is_sorted(get_audio_time_ms_.begin(), get_audio_time_ms_.end())); + + std::vector rtp_timestamps_ms; + double offset = std::numeric_limits::max(); + RtpTimestampUnwrapper unwrapper; + // This loop traverses data_ and populates rtp_timestamps_ms as well as + // calculates the base offset. + for (auto& d : data_) { + rtp_timestamps_ms.push_back( + static_cast(unwrapper.Unwrap(d.first)) / + rtc::CheckedDivExact(last_sample_rate_hz_, 1000)); + offset = + std::min(offset, d.second.arrival_time_ms - rtp_timestamps_ms.back()); + } + + // This loop traverses the data again and populates the graph vectors. The + // reason to have two loops and traverse twice is that the offset cannot be + // known until the first traversal is done. Meanwhile, the final offset must + // be known already at the start of this second loop. + size_t i = 0; + for (const auto& data : data_) { + const double offset_send_time_ms = rtp_timestamps_ms[i++] + offset; + const auto& timing = data.second; + corrected_arrival_delay_ms->push_back(std::make_pair( + timing.arrival_time_ms, + LinearInterpolate(timing.arrival_time_ms, get_audio_time_ms_, + nominal_get_audio_time_ms) - + offset_send_time_ms)); + arrival_delay_ms->push_back(std::make_pair( + timing.arrival_time_ms, timing.arrival_time_ms - offset_send_time_ms)); + + if (timing.decode_get_audio_count) { + // This packet was decoded. + RTC_DCHECK(timing.sync_delay_ms); + const int64_t get_audio_time = + *timing.decode_get_audio_count * 10 + get_audio_time_ms_[0]; + const float playout_ms = + get_audio_time + *timing.sync_delay_ms - offset_send_time_ms; + playout_delay_ms->push_back(std::make_pair(get_audio_time, playout_ms)); + RTC_DCHECK(timing.target_delay_ms); + RTC_DCHECK(timing.current_delay_ms); + const float target = + playout_ms - *timing.current_delay_ms + *timing.target_delay_ms; + target_delay_ms->push_back(std::make_pair(get_audio_time, target)); + } + } +} + +void NetEqDelayAnalyzer::CreateMatlabScript( + absl::string_view script_name) const { + Delays arrival_delay_ms; + Delays corrected_arrival_delay_ms; + Delays playout_delay_ms; + Delays target_delay_ms; + CreateGraphs(&arrival_delay_ms, &corrected_arrival_delay_ms, + &playout_delay_ms, &target_delay_ms); + + // Maybe better to find the actually smallest timestamp, to surely avoid + // x-axis starting from negative. + const int64_t ref_time_ms = arrival_delay_ms.front().first; + + // Create an output file stream to Matlab script file. + std::ofstream output(std::string{script_name}); + + PrintDelays(corrected_arrival_delay_ms, ref_time_ms, kArrivalDelayX, + kArrivalDelayY, output, ";"); + + // PrintDelays(corrected_arrival_delay_x, kCorrectedArrivalDelayX, + // kCorrectedArrivalDelayY, output); + + PrintDelays(playout_delay_ms, ref_time_ms, kPlayoutDelayX, kPlayoutDelayY, + output, ";"); + + PrintDelays(target_delay_ms, ref_time_ms, kTargetDelayX, kTargetDelayY, + output, ";"); + + output << "h=plot(" << kArrivalDelayX << ", " << kArrivalDelayY << ", " + << kTargetDelayX << ", " << kTargetDelayY << ", 'g.', " + << kPlayoutDelayX << ", " << kPlayoutDelayY << ");" << std::endl; + output << "set(h(1),'color',0.75*[1 1 1]);" << std::endl; + output << "set(h(2),'markersize',6);" << std::endl; + output << "set(h(3),'linew',1.5);" << std::endl; + output << "ax1=axis;" << std::endl; + output << "axis tight" << std::endl; + output << "ax2=axis;" << std::endl; + output << "axis([ax2(1:3) ax1(4)])" << std::endl; + output << "xlabel('time [s]');" << std::endl; + output << "ylabel('relative delay [ms]');" << std::endl; + if (!ssrcs_.empty()) { + auto ssrc_it = ssrcs_.cbegin(); + output << "title('SSRC: 0x" << std::hex << static_cast(*ssrc_it++); + while (ssrc_it != ssrcs_.end()) { + output << ", 0x" << std::hex << static_cast(*ssrc_it++); + } + output << std::dec; + auto pt_it = payload_types_.cbegin(); + output << "; Payload Types: " << *pt_it++; + while (pt_it != payload_types_.end()) { + output << ", " << *pt_it++; + } + output << "');" << std::endl; + } +} + +void NetEqDelayAnalyzer::CreatePythonScript( + absl::string_view script_name) const { + Delays arrival_delay_ms; + Delays corrected_arrival_delay_ms; + Delays playout_delay_ms; + Delays target_delay_ms; + CreateGraphs(&arrival_delay_ms, &corrected_arrival_delay_ms, + &playout_delay_ms, &target_delay_ms); + + // Maybe better to find the actually smallest timestamp, to surely avoid + // x-axis starting from negative. + const int64_t ref_time_ms = arrival_delay_ms.front().first; + + // Create an output file stream to the python script file. + std::ofstream output(std::string{script_name}); + + // Necessary includes + output << "import numpy as np" << std::endl; + output << "import matplotlib.pyplot as plt" << std::endl; + + PrintDelays(corrected_arrival_delay_ms, ref_time_ms, kArrivalDelayX, + kArrivalDelayY, output); + + // PrintDelays(corrected_arrival_delay_x, kCorrectedArrivalDelayX, + // kCorrectedArrivalDelayY, output); + + PrintDelays(playout_delay_ms, ref_time_ms, kPlayoutDelayX, kPlayoutDelayY, + output); + + PrintDelays(target_delay_ms, ref_time_ms, kTargetDelayX, kTargetDelayY, + output); + + output << "if __name__ == '__main__':" << std::endl; + output << " h=plt.plot(" << kArrivalDelayX << ", " << kArrivalDelayY << ", " + << kTargetDelayX << ", " << kTargetDelayY << ", 'g.', " + << kPlayoutDelayX << ", " << kPlayoutDelayY << ")" << std::endl; + output << " plt.setp(h[0],'color',[.75, .75, .75])" << std::endl; + output << " plt.setp(h[1],'markersize',6)" << std::endl; + output << " plt.setp(h[2],'linewidth',1.5)" << std::endl; + output << " plt.axis('tight')" << std::endl; + output << " plt.xlabel('time [s]')" << std::endl; + output << " plt.ylabel('relative delay [ms]')" << std::endl; + if (!ssrcs_.empty()) { + auto ssrc_it = ssrcs_.cbegin(); + output << " plt.legend((\"arrival delay\", \"target delay\", \"playout " + "delay\"))" + << std::endl; + output << " plt.title('SSRC: 0x" << std::hex + << static_cast(*ssrc_it++); + while (ssrc_it != ssrcs_.end()) { + output << ", 0x" << std::hex << static_cast(*ssrc_it++); + } + output << std::dec; + auto pt_it = payload_types_.cbegin(); + output << "; Payload Types: " << *pt_it++; + while (pt_it != payload_types_.end()) { + output << ", " << *pt_it++; + } + output << "')" << std::endl; + } + output << " plt.show()" << std::endl; +} + +} // namespace test +} // namespace webrtc -- cgit v1.2.3