summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/api/neteq/neteq.h
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
commit6bf0a5cb5034a7e684dcc3500e841785237ce2dd (patch)
treea68f146d7fa01f0134297619fbe7e33db084e0aa /third_party/libwebrtc/api/neteq/neteq.h
parentInitial commit. (diff)
downloadthunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.tar.xz
thunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.zip
Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/api/neteq/neteq.h')
-rw-r--r--third_party/libwebrtc/api/neteq/neteq.h322
1 files changed, 322 insertions, 0 deletions
diff --git a/third_party/libwebrtc/api/neteq/neteq.h b/third_party/libwebrtc/api/neteq/neteq.h
new file mode 100644
index 0000000000..5300c5601e
--- /dev/null
+++ b/third_party/libwebrtc/api/neteq/neteq.h
@@ -0,0 +1,322 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_NETEQ_NETEQ_H_
+#define API_NETEQ_NETEQ_H_
+
+#include <stddef.h> // Provide access to size_t.
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/audio_codecs/audio_codec_pair_id.h"
+#include "api/audio_codecs/audio_decoder.h"
+#include "api/audio_codecs/audio_format.h"
+#include "api/rtp_headers.h"
+#include "api/scoped_refptr.h"
+
+namespace webrtc {
+
+// Forward declarations.
+class AudioFrame;
+class AudioDecoderFactory;
+class Clock;
+
+struct NetEqNetworkStatistics {
+ uint16_t current_buffer_size_ms; // Current jitter buffer size in ms.
+ uint16_t preferred_buffer_size_ms; // Target buffer size in ms.
+ uint16_t jitter_peaks_found; // 1 if adding extra delay due to peaky
+ // jitter; 0 otherwise.
+ uint16_t expand_rate; // Fraction (of original stream) of synthesized
+ // audio inserted through expansion (in Q14).
+ uint16_t speech_expand_rate; // Fraction (of original stream) of synthesized
+ // speech inserted through expansion (in Q14).
+ uint16_t preemptive_rate; // Fraction of data inserted through pre-emptive
+ // expansion (in Q14).
+ uint16_t accelerate_rate; // Fraction of data removed through acceleration
+ // (in Q14).
+ uint16_t secondary_decoded_rate; // Fraction of data coming from FEC/RED
+ // decoding (in Q14).
+ uint16_t secondary_discarded_rate; // Fraction of discarded FEC/RED data (in
+ // Q14).
+ // Statistics for packet waiting times, i.e., the time between a packet
+ // arrives until it is decoded.
+ int mean_waiting_time_ms;
+ int median_waiting_time_ms;
+ int min_waiting_time_ms;
+ int max_waiting_time_ms;
+};
+
+// NetEq statistics that persist over the lifetime of the class.
+// These metrics are never reset.
+struct NetEqLifetimeStatistics {
+ // Stats below correspond to similarly-named fields in the WebRTC stats spec.
+ // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats
+ uint64_t total_samples_received = 0;
+ uint64_t concealed_samples = 0;
+ uint64_t concealment_events = 0;
+ uint64_t jitter_buffer_delay_ms = 0;
+ uint64_t jitter_buffer_emitted_count = 0;
+ uint64_t jitter_buffer_target_delay_ms = 0;
+ uint64_t jitter_buffer_minimum_delay_ms = 0;
+ uint64_t inserted_samples_for_deceleration = 0;
+ uint64_t removed_samples_for_acceleration = 0;
+ uint64_t silent_concealed_samples = 0;
+ uint64_t fec_packets_received = 0;
+ uint64_t fec_packets_discarded = 0;
+ uint64_t packets_discarded = 0;
+ // Below stats are not part of the spec.
+ uint64_t delayed_packet_outage_samples = 0;
+ // This is sum of relative packet arrival delays of received packets so far.
+ // Since end-to-end delay of a packet is difficult to measure and is not
+ // necessarily useful for measuring jitter buffer performance, we report a
+ // relative packet arrival delay. The relative packet arrival delay of a
+ // packet is defined as the arrival delay compared to the first packet
+ // received, given that it had zero delay. To avoid clock drift, the "first"
+ // packet can be made dynamic.
+ uint64_t relative_packet_arrival_delay_ms = 0;
+ uint64_t jitter_buffer_packets_received = 0;
+ // An interruption is a loss-concealment event lasting at least 150 ms. The
+ // two stats below count the number os such events and the total duration of
+ // these events.
+ int32_t interruption_count = 0;
+ int32_t total_interruption_duration_ms = 0;
+ // Total number of comfort noise samples generated during DTX.
+ uint64_t generated_noise_samples = 0;
+};
+
+// Metrics that describe the operations performed in NetEq, and the internal
+// state.
+struct NetEqOperationsAndState {
+ // These sample counters are cumulative, and don't reset. As a reference, the
+ // total number of output samples can be found in
+ // NetEqLifetimeStatistics::total_samples_received.
+ uint64_t preemptive_samples = 0;
+ uint64_t accelerate_samples = 0;
+ // Count of the number of buffer flushes.
+ uint64_t packet_buffer_flushes = 0;
+ // The statistics below are not cumulative.
+ // The waiting time of the last decoded packet.
+ uint64_t last_waiting_time_ms = 0;
+ // The sum of the packet and jitter buffer size in ms.
+ uint64_t current_buffer_size_ms = 0;
+ // The current frame size in ms.
+ uint64_t current_frame_size_ms = 0;
+ // Flag to indicate that the next packet is available.
+ bool next_packet_available = false;
+};
+
+// This is the interface class for NetEq.
+class NetEq {
+ public:
+ struct Config {
+ Config();
+ Config(const Config&);
+ Config(Config&&);
+ ~Config();
+ Config& operator=(const Config&);
+ Config& operator=(Config&&);
+
+ std::string ToString() const;
+
+ int sample_rate_hz = 48000; // Initial value. Will change with input data.
+ bool enable_post_decode_vad = false;
+ size_t max_packets_in_buffer = 200;
+ int max_delay_ms = 0;
+ int min_delay_ms = 0;
+ bool enable_fast_accelerate = false;
+ bool enable_muted_state = false;
+ bool enable_rtx_handling = false;
+ absl::optional<AudioCodecPairId> codec_pair_id;
+ bool for_test_no_time_stretching = false; // Use only for testing.
+ };
+
+ enum ReturnCodes { kOK = 0, kFail = -1 };
+
+ enum class Operation {
+ kNormal,
+ kMerge,
+ kExpand,
+ kAccelerate,
+ kFastAccelerate,
+ kPreemptiveExpand,
+ kRfc3389Cng,
+ kRfc3389CngNoPacket,
+ kCodecInternalCng,
+ kDtmf,
+ kUndefined,
+ };
+
+ enum class Mode {
+ kNormal,
+ kExpand,
+ kMerge,
+ kAccelerateSuccess,
+ kAccelerateLowEnergy,
+ kAccelerateFail,
+ kPreemptiveExpandSuccess,
+ kPreemptiveExpandLowEnergy,
+ kPreemptiveExpandFail,
+ kRfc3389Cng,
+ kCodecInternalCng,
+ kCodecPlc,
+ kDtmf,
+ kError,
+ kUndefined,
+ };
+
+ // Return type for GetDecoderFormat.
+ struct DecoderFormat {
+ int sample_rate_hz;
+ int num_channels;
+ SdpAudioFormat sdp_format;
+ };
+
+ virtual ~NetEq() {}
+
+ // Inserts a new packet into NetEq.
+ // Returns 0 on success, -1 on failure.
+ virtual int InsertPacket(const RTPHeader& rtp_header,
+ rtc::ArrayView<const uint8_t> payload) = 0;
+
+ // Lets NetEq know that a packet arrived with an empty payload. This typically
+ // happens when empty packets are used for probing the network channel, and
+ // these packets use RTP sequence numbers from the same series as the actual
+ // audio packets.
+ virtual void InsertEmptyPacket(const RTPHeader& rtp_header) = 0;
+
+ // Instructs NetEq to deliver 10 ms of audio data. The data is written to
+ // `audio_frame`. All data in `audio_frame` is wiped; `data_`, `speech_type_`,
+ // `num_channels_`, `sample_rate_hz_`, `samples_per_channel_`, and
+ // `vad_activity_` are updated upon success. If an error is returned, some
+ // fields may not have been updated, or may contain inconsistent values.
+ // If muted state is enabled (through Config::enable_muted_state), `muted`
+ // may be set to true after a prolonged expand period. When this happens, the
+ // `data_` in `audio_frame` is not written, but should be interpreted as being
+ // all zeros. For testing purposes, an override can be supplied in the
+ // `action_override` argument, which will cause NetEq to take this action
+ // next, instead of the action it would normally choose. An optional output
+ // argument for fetching the current sample rate can be provided, which
+ // will return the same value as last_output_sample_rate_hz() but will avoid
+ // additional synchronization.
+ // Returns kOK on success, or kFail in case of an error.
+ virtual int GetAudio(
+ AudioFrame* audio_frame,
+ bool* muted,
+ int* current_sample_rate_hz = nullptr,
+ absl::optional<Operation> action_override = absl::nullopt) = 0;
+
+ // Replaces the current set of decoders with the given one.
+ virtual void SetCodecs(const std::map<int, SdpAudioFormat>& codecs) = 0;
+
+ // Associates `rtp_payload_type` with the given codec, which NetEq will
+ // instantiate when it needs it. Returns true iff successful.
+ virtual bool RegisterPayloadType(int rtp_payload_type,
+ const SdpAudioFormat& audio_format) = 0;
+
+ // Removes `rtp_payload_type` from the codec database. Returns 0 on success,
+ // -1 on failure. Removing a payload type that is not registered is ok and
+ // will not result in an error.
+ virtual int RemovePayloadType(uint8_t rtp_payload_type) = 0;
+
+ // Removes all payload types from the codec database.
+ virtual void RemoveAllPayloadTypes() = 0;
+
+ // Sets a minimum delay in millisecond for packet buffer. The minimum is
+ // maintained unless a higher latency is dictated by channel condition.
+ // Returns true if the minimum is successfully applied, otherwise false is
+ // returned.
+ virtual bool SetMinimumDelay(int delay_ms) = 0;
+
+ // Sets a maximum delay in milliseconds for packet buffer. The latency will
+ // not exceed the given value, even required delay (given the channel
+ // conditions) is higher. Calling this method has the same effect as setting
+ // the `max_delay_ms` value in the NetEq::Config struct.
+ virtual bool SetMaximumDelay(int delay_ms) = 0;
+
+ // Sets a base minimum delay in milliseconds for packet buffer. The minimum
+ // delay which is set via `SetMinimumDelay` can't be lower than base minimum
+ // delay. Calling this method is similar to setting the `min_delay_ms` value
+ // in the NetEq::Config struct. Returns true if the base minimum is
+ // successfully applied, otherwise false is returned.
+ virtual bool SetBaseMinimumDelayMs(int delay_ms) = 0;
+
+ // Returns current value of base minimum delay in milliseconds.
+ virtual int GetBaseMinimumDelayMs() const = 0;
+
+ // Returns the current target delay in ms. This includes any extra delay
+ // requested through SetMinimumDelay.
+ virtual int TargetDelayMs() const = 0;
+
+ // Returns the current total delay (packet buffer and sync buffer) in ms,
+ // with smoothing applied to even out short-time fluctuations due to jitter.
+ // The packet buffer part of the delay is not updated during DTX/CNG periods.
+ virtual int FilteredCurrentDelayMs() const = 0;
+
+ // Writes the current network statistics to `stats`. The statistics are reset
+ // after the call.
+ virtual int NetworkStatistics(NetEqNetworkStatistics* stats) = 0;
+
+ // Current values only, not resetting any state.
+ virtual NetEqNetworkStatistics CurrentNetworkStatistics() const = 0;
+
+ // Returns a copy of this class's lifetime statistics. These statistics are
+ // never reset.
+ virtual NetEqLifetimeStatistics GetLifetimeStatistics() const = 0;
+
+ // Returns statistics about the performed operations and internal state. These
+ // statistics are never reset.
+ virtual NetEqOperationsAndState GetOperationsAndState() const = 0;
+
+ // Enables post-decode VAD. When enabled, GetAudio() will return
+ // kOutputVADPassive when the signal contains no speech.
+ virtual void EnableVad() = 0;
+
+ // Disables post-decode VAD.
+ virtual void DisableVad() = 0;
+
+ // Returns the RTP timestamp for the last sample delivered by GetAudio().
+ // The return value will be empty if no valid timestamp is available.
+ virtual absl::optional<uint32_t> GetPlayoutTimestamp() const = 0;
+
+ // Returns the sample rate in Hz of the audio produced in the last GetAudio
+ // call. If GetAudio has not been called yet, the configured sample rate
+ // (Config::sample_rate_hz) is returned.
+ virtual int last_output_sample_rate_hz() const = 0;
+
+ // Returns the decoder info for the given payload type. Returns empty if no
+ // such payload type was registered.
+ virtual absl::optional<DecoderFormat> GetDecoderFormat(
+ int payload_type) const = 0;
+
+ // Flushes both the packet buffer and the sync buffer.
+ virtual void FlushBuffers() = 0;
+
+ // Enables NACK and sets the maximum size of the NACK list, which should be
+ // positive and no larger than Nack::kNackListSizeLimit. If NACK is already
+ // enabled then the maximum NACK list size is modified accordingly.
+ virtual void EnableNack(size_t max_nack_list_size) = 0;
+
+ virtual void DisableNack() = 0;
+
+ // Returns a list of RTP sequence numbers corresponding to packets to be
+ // retransmitted, given an estimate of the round-trip time in milliseconds.
+ virtual std::vector<uint16_t> GetNackList(
+ int64_t round_trip_time_ms) const = 0;
+
+ // Returns the length of the audio yet to play in the sync buffer.
+ // Mainly intended for testing.
+ virtual int SyncBufferSizeMs() const = 0;
+};
+
+} // namespace webrtc
+#endif // API_NETEQ_NETEQ_H_